Initial commit: CloudOps infrastructure platform

This commit is contained in:
root
2026-04-09 19:58:57 +02:00
commit 1166a52f26
7762 changed files with 839452 additions and 0 deletions

View File

@@ -0,0 +1,194 @@
<?php
declare(strict_types=1);
namespace Mautic\IntegrationsBundle\Entity;
use Doctrine\DBAL\Types\Types;
use Doctrine\ORM\Mapping as ORM;
use Mautic\CoreBundle\Doctrine\Mapping\ClassMetadataBuilder;
class FieldChange
{
/**
* @var int
*/
private $id;
/**
* @var string
*/
private $integration;
/**
* @var int|string
*/
private $objectId;
/**
* @var string
*/
private $objectType;
/**
* @var \DateTimeInterface
*/
private $modifiedAt;
/**
* @var string
*/
private $columnName;
/**
* @var string
*/
private $columnType;
/**
* @var string
*/
private $columnValue;
public static function loadMetadata(ORM\ClassMetadata $metadata): void
{
$builder = new ClassMetadataBuilder($metadata);
$builder
->setTable('sync_object_field_change_report')
->setCustomRepositoryClass(FieldChangeRepository::class)
->addIndex(['object_type', 'object_id', 'column_name'], 'object_composite_key')
->addIndex(['integration', 'object_type', 'object_id', 'column_name'], 'integration_object_composite_key')
->addIndex(['integration', 'object_type', 'modified_at'], 'integration_object_type_modification_composite_key');
$builder->addId();
$builder
->createField('integration', Types::STRING)
->build();
$builder->addBigIntIdField('objectId', 'object_id', false);
$builder
->createField('objectType', Types::STRING)
->columnName('object_type')
->build();
$builder
->createField('modifiedAt', Types::DATETIME_MUTABLE)
->columnName('modified_at')
->build();
$builder
->createField('columnName', Types::STRING)
->columnName('column_name')
->build();
$builder
->createField('columnType', Types::STRING)
->columnName('column_type')
->build();
$builder
->createField('columnValue', Types::TEXT)
->columnName('column_value')
->build();
}
/**
* @return int
*/
public function getId()
{
return $this->id;
}
public function getIntegration(): string
{
return $this->integration;
}
/**
* @param string $integration
*
* @return FieldChange
*/
public function setIntegration($integration)
{
$this->integration = $integration;
return $this;
}
public function setObjectId(int $id): self
{
$this->objectId = (string) $id;
return $this;
}
public function getObjectId(): int
{
return (int) $this->objectId;
}
public function setObjectType(string $type): self
{
$this->objectType = $type;
return $this;
}
public function getObjectType(): string
{
return $this->objectType;
}
public function setModifiedAt(\DateTime $time): self
{
$this->modifiedAt = $time;
return $this;
}
public function getModifiedAt(): \DateTimeInterface
{
return $this->modifiedAt;
}
public function setColumnName(string $name): self
{
$this->columnName = $name;
return $this;
}
public function getColumnName(): string
{
return $this->columnName;
}
public function setColumnType(string $type): self
{
$this->columnType = $type;
return $this;
}
public function getColumnType(): string
{
return $this->columnType;
}
public function setColumnValue(string $value): self
{
$this->columnValue = $value;
return $this;
}
public function getColumnValue(): string
{
return $this->columnValue;
}
}

View File

@@ -0,0 +1,201 @@
<?php
declare(strict_types=1);
namespace Mautic\IntegrationsBundle\Entity;
use Doctrine\DBAL\ArrayParameterType;
use Doctrine\DBAL\Query\Expression\CompositeExpression;
use Mautic\CoreBundle\Entity\CommonRepository;
use Mautic\LeadBundle\Entity\Lead;
/**
* @extends CommonRepository<FieldChange>
*/
class FieldChangeRepository extends CommonRepository
{
/**
* Takes an object id & type and deletes all entities
* that match the given column names.
*/
public function deleteEntitiesForObjectByColumnName(int $objectId, string $objectType, array $columnNames): void
{
$qb = $this->getEntityManager()->getConnection()->createQueryBuilder();
$qb
->delete(MAUTIC_TABLE_PREFIX.'sync_object_field_change_report')
->where(
$qb->expr()->and(
$qb->expr()->eq('object_type', ':objectType'),
$qb->expr()->eq('object_id', ':objectId'),
$qb->expr()->in('column_name', ':columnNames')
)
)
->setParameter('objectType', $objectType)
->setParameter('objectId', $objectId)
->setParameter('columnNames', $columnNames, ArrayParameterType::STRING)
->executeStatement();
}
/**
* Takes an object id & type and deletes all entities that match.
*/
public function deleteEntitiesForObject(int $objectId, string $objectType, ?string $integration = null, ?\DateTimeInterface $toDateTime = null): void
{
$qb = $this->getEntityManager()->getConnection()->createQueryBuilder();
$expr = CompositeExpression::and($qb->expr()->eq('object_type', ':objectType'), $qb->expr()->eq('object_id', ':objectId'));
if ($integration) {
$expr = $expr->with(
$qb->expr()->eq('integration', ':integration')
);
$qb->setParameter('integration', $integration);
}
if (null !== $toDateTime) {
$expr = $expr->with($qb->expr()->lte('modified_at', ':toDateTime'));
$qb->setParameter('toDateTime', $toDateTime->format('Y-m-d H:i:s'));
}
$qb->setParameter('objectType', $objectType)
->setParameter('objectId', $objectId);
$qb
->delete(MAUTIC_TABLE_PREFIX.'sync_object_field_change_report')
->where($expr)
->executeStatement();
}
/**
* @param int|null $afterObjectId
* @param int $objectCount
*/
public function findChangesBefore(string $integration, string $objectType, \DateTimeInterface $toDateTime, $afterObjectId = null, $objectCount = 100): array
{
// Get a list of object IDs so that we can get complete snapshots of the objects
$qb = $this->getEntityManager()->getConnection()->createQueryBuilder();
$qb
->select('f.object_id')
->from(MAUTIC_TABLE_PREFIX.'sync_object_field_change_report', 'f')
->where(
$qb->expr()->and(
$qb->expr()->eq('f.integration', ':integration'),
$qb->expr()->eq('f.object_type', ':objectType'),
$qb->expr()->lte('f.modified_at', ':toDateTime')
)
);
if (Lead::class === $objectType) {
$qb->join('f', MAUTIC_TABLE_PREFIX.'leads', 'l', 'l.id = f.object_id');
}
$qb->setParameter('integration', $integration)
->setParameter('objectType', $objectType)
->setParameter('toDateTime', $toDateTime->format('Y-m-d H:i:s'))
->orderBy('f.object_id')
->groupBy('f.object_id')
->setMaxResults($objectCount);
if ($afterObjectId) {
$qb->andWhere(
$qb->expr()->gt('f.object_id', (int) $afterObjectId)
);
}
$objectIds = $qb->executeQuery()->fetchFirstColumn();
if (!$objectIds) {
return [];
}
// Get all the field changes for the requested objects
$qb
->resetQueryParts()
->select('*')
->from(MAUTIC_TABLE_PREFIX.'sync_object_field_change_report', 'f')
->where(
$qb->expr()->and(
$qb->expr()->eq('f.integration', ':integration'),
$qb->expr()->eq('f.object_type', ':objectType'),
$qb->expr()->in('f.object_id', $objectIds)
)
)
->setParameter('integration', $integration)
->setParameter('objectType', $objectType)
// 1. We must sort by f.object_id. Otherwise values stored in PartialObjectReportBuilder::lastProcessedTrackedId will be incorrect.
// 2. Newer updated fields must override older updated fields
->orderBy('f.object_id, f.modified_at', 'ASC');
return $qb->executeQuery()->fetchAllAssociative();
}
/**
* @param int $objectId
*/
public function findChangesForObject(string $integration, string $objectType, $objectId): array
{
// Get a list of object IDs so that we can get complete snapshots of the objects
$qb = $this->getEntityManager()->getConnection()->createQueryBuilder();
$qb
->select('*')
->from(MAUTIC_TABLE_PREFIX.'sync_object_field_change_report', 'f')
->where(
$qb->expr()->and(
$qb->expr()->eq('f.integration', ':integration'),
$qb->expr()->eq('f.object_type', ':objectType'),
$qb->expr()->eq('f.object_id', ':objectId')
)
)
->setParameter('integration', $integration)
->setParameter('objectType', $objectType)
->setParameter('objectId', (int) $objectId)
->orderBy('f.modified_at'); // Newer updated fields must override older updated fields
return $qb->executeQuery()->fetchAllAssociative();
}
public function deleteOrphanLeadChanges(): int
{
$totalDeleted = 0;
$limit = 1000;
$totalLimit = 100000;
$deletedInLastLoop = $limit;
while ($totalDeleted < $totalLimit && $deletedInLastLoop === $limit && $deleted = $this->doDeleteOrphanLeadChanges($limit)) {
$deletedInLastLoop = $deleted;
$totalDeleted += $deleted;
}
return $totalDeleted;
}
private function doDeleteOrphanLeadChanges(int $limit): int
{
$qb = $this->getEntityManager()->getConnection()->createQueryBuilder();
$qb->select('f.id')
->from(MAUTIC_TABLE_PREFIX.'sync_object_field_change_report', 'f')
->leftJoin('f', MAUTIC_TABLE_PREFIX.'leads', 'l', 'l.id = f.object_id')
->where(
$qb->expr()->and(
$qb->expr()->eq('object_type', ':objectType'),
$qb->expr()->isNull('l.id')
)
)
->setMaxResults($limit)
->setParameter('objectType', Lead::class);
$objectIds = $qb->executeQuery()->fetchFirstColumn();
if (!$objectIds) {
return 0;
}
$qb2 = $this->getEntityManager()->getConnection()->createQueryBuilder();
$qb2->delete(MAUTIC_TABLE_PREFIX.'sync_object_field_change_report')
->where(
$qb2->expr()->in('id', ':ids')
)
->setParameter('ids', $objectIds, ArrayParameterType::INTEGER);
return $qb2->executeStatement();
}
}

View File

@@ -0,0 +1,354 @@
<?php
declare(strict_types=1);
namespace Mautic\IntegrationsBundle\Entity;
use Doctrine\DBAL\Types\Types;
use Doctrine\ORM\Mapping as ORM;
use Mautic\CoreBundle\Doctrine\Mapping\ClassMetadataBuilder;
class ObjectMapping
{
/**
* @var int
*/
private $id;
private ?\DateTimeInterface $dateCreated;
/**
* @var string
*/
private $integration;
/**
* @var string
*/
private $internalObjectName;
/**
* @var string
*/
private $internalObjectId;
/**
* @var string
*/
private $integrationObjectName;
/**
* @var string
*/
private $integrationObjectId;
private ?\DateTimeInterface $lastSyncDate;
/**
* @var array
*/
private $internalStorage = [];
/**
* @var bool
*/
private $isDeleted = false;
/**
* @var string|null
*/
private $integrationReferenceId;
public static function loadMetadata(ORM\ClassMetadata $metadata): void
{
$builder = new ClassMetadataBuilder($metadata);
$builder
->setTable('sync_object_mapping')
->setCustomRepositoryClass(ObjectMappingRepository::class)
->addIndex(['internal_object_id'], 'internal_object_id_idx')
->addIndex(['integration', 'integration_object_name', 'integration_object_id', 'integration_reference_id'], 'integration_object')
->addIndex(['integration', 'integration_object_name', 'integration_reference_id', 'integration_object_id'], 'integration_reference')
->addIndex(['integration', 'internal_object_name', 'last_sync_date'], 'integration_integration_object_name_last_sync_date')
->addIndex(['integration', 'last_sync_date'], 'integration_last_sync_date');
$builder->addId();
$builder
->createField('dateCreated', Types::DATETIME_MUTABLE)
->columnName('date_created')
->build();
$builder
->createField('integration', Types::STRING)
->build();
$builder
->createField('internalObjectName', Types::STRING)
->columnName('internal_object_name')
->build();
$builder->addBigIntIdField('internalObjectId', 'internal_object_id', false);
$builder
->createField('integrationObjectName', Types::STRING)
->columnName('integration_object_name')
->build();
// Must be a string as not all IDs are integer based
$builder
->createField('integrationObjectId', Types::STRING)
->columnName('integration_object_id')
->build();
$builder
->createField('lastSyncDate', Types::DATETIME_MUTABLE)
->columnName('last_sync_date')
->build();
$builder
->createField('internalStorage', Types::JSON)
->columnName('internal_storage')
->build();
$builder
->createField('isDeleted', Types::BOOLEAN)
->columnName('is_deleted')
->build();
$builder
->createField('integrationReferenceId', Types::STRING)
->columnName('integration_reference_id')
->nullable()
->build();
}
public function __construct(?\DateTime $dateCreated = null)
{
if (null === $dateCreated) {
$dateCreated = new \DateTime();
}
$this->dateCreated = $dateCreated;
$this->lastSyncDate = $dateCreated;
}
/**
* @return int|null ?int
*/
public function getId()
{
return $this->id;
}
/**
* @param int $id
*
* @return ObjectMapping
*/
public function setId($id)
{
$this->id = $id;
return $this;
}
/**
* @return \DateTimeInterface|null
*/
public function getDateCreated()
{
return $this->dateCreated;
}
/**
* @return string
*/
public function getIntegration()
{
return $this->integration;
}
/**
* @param string $integration
*
* @return ObjectMapping
*/
public function setIntegration($integration)
{
$this->integration = $integration;
return $this;
}
/**
* @return string
*/
public function getInternalObjectName()
{
return $this->internalObjectName;
}
/**
* @param string $internalObjectName
*
* @return ObjectMapping
*/
public function setInternalObjectName($internalObjectName)
{
$this->internalObjectName = $internalObjectName;
return $this;
}
public function getInternalObjectId(): int
{
return (int) $this->internalObjectId;
}
/**
* @param int $internalObjectId
*
* @return ObjectMapping
*/
public function setInternalObjectId($internalObjectId)
{
$this->internalObjectId = (string) $internalObjectId;
return $this;
}
/**
* @return string
*/
public function getIntegrationObjectName()
{
return $this->integrationObjectName;
}
/**
* @param string $integrationObjectName
*
* @return ObjectMapping
*/
public function setIntegrationObjectName($integrationObjectName)
{
$this->integrationObjectName = $integrationObjectName;
return $this;
}
/**
* @return string
*/
public function getIntegrationObjectId()
{
return $this->integrationObjectId;
}
/**
* @param string $integrationObjectId
*
* @return ObjectMapping
*/
public function setIntegrationObjectId($integrationObjectId)
{
$this->integrationObjectId = $integrationObjectId;
return $this;
}
/**
* @return \DateTimeInterface
*/
public function getLastSyncDate()
{
return $this->lastSyncDate;
}
/**
* @param \DateTimeInterface|null $lastSyncDate
*
* @return ObjectMapping
*/
public function setLastSyncDate($lastSyncDate)
{
if (null === $lastSyncDate) {
$lastSyncDate = new \DateTime();
}
$this->lastSyncDate = $lastSyncDate;
return $this;
}
/**
* @return array
*/
public function getInternalStorage()
{
return $this->internalStorage;
}
/**
* @param array $internalStorage
*
* @return ObjectMapping
*/
public function setInternalStorage($internalStorage)
{
$this->internalStorage = $internalStorage;
return $this;
}
/**
* @return $this
*/
public function appendToInternalStorage($key, $value)
{
$this->internalStorage[$key] = $value;
return $this;
}
/**
* @return bool
*/
public function isDeleted()
{
return $this->isDeleted;
}
/**
* @param bool $isDeleted
*
* @return ObjectMapping
*/
public function setIsDeleted($isDeleted)
{
$this->isDeleted = $isDeleted;
return $this;
}
/**
* @return string|null
*/
public function getIntegrationReferenceId()
{
return $this->integrationReferenceId;
}
/**
* @param string|null $integrationReferenceId
*
* @return ObjectMapping
*/
public function setIntegrationReferenceId($integrationReferenceId)
{
$this->integrationReferenceId = $integrationReferenceId;
return $this;
}
}

View File

@@ -0,0 +1,225 @@
<?php
declare(strict_types=1);
namespace Mautic\IntegrationsBundle\Entity;
use Doctrine\DBAL\ArrayParameterType;
use Doctrine\DBAL\Types\Types;
use Mautic\CoreBundle\Entity\CommonRepository;
use Mautic\CoreBundle\Helper\DateTimeHelper;
/**
* @extends CommonRepository<ObjectMapping>
*/
class ObjectMappingRepository extends CommonRepository
{
public function getInternalObject($integration, $integrationObjectName, $integrationObjectId, $internalObjectName): ?array
{
return $this->doGetInternalObject($integration, $integrationObjectName, $integrationObjectId, $internalObjectName);
}
/**
* @return array<string,mixed>|null
*/
public function getInternalObjectWithLock(string $integration, string $integrationObjectName, string $integrationObjectId, string $internalObjectName, string $lock = 'LOCK IN SHARE MODE'): ?array
{
return $this->doGetInternalObject($integration, $integrationObjectName, $integrationObjectId, $internalObjectName, $lock);
}
/**
* @return array|null
*/
public function getIntegrationObject($integration, $internalObjectName, $internalObjectId, $integrationObjectName)
{
$qb = $this->getEntityManager()->getConnection()->createQueryBuilder();
$qb->select('*')
->from(MAUTIC_TABLE_PREFIX.'sync_object_mapping', 'i')
->where(
$qb->expr()->and(
$qb->expr()->eq('i.integration', ':integration'),
$qb->expr()->eq('i.internal_object_name', ':internalObjectName'),
$qb->expr()->eq('i.internal_object_id', ':internalObjectId'),
$qb->expr()->eq('i.integration_object_name', ':integrationObjectName')
)
)
->setParameter('integration', $integration)
->setParameter('internalObjectName', $internalObjectName)
->setParameter('internalObjectId', $internalObjectId)
->setParameter('integrationObjectName', $integrationObjectName);
$result = $qb->executeQuery()->fetchAssociative();
return $result ?: null;
}
/**
* @param string $integration
* @param string $oldObjectName
* @param mixed $oldObjectId
* @param string $newObjectName
* @param mixed $newObjectId
*/
public function updateIntegrationObject($integration, $oldObjectName, $oldObjectId, $newObjectName, $newObjectId): int
{
$qb = $this->getEntityManager()->getConnection()->createQueryBuilder();
$qb->update(MAUTIC_TABLE_PREFIX.'sync_object_mapping', 'i')
->set('integration_object_name', ':newObjectName')
->set('integration_object_id', ':newObjectId')
->where(
$qb->expr()->and(
$qb->expr()->eq('i.integration', ':integration'),
$qb->expr()->eq('i.integration_object_name', ':oldObjectName'),
$qb->expr()->eq('i.integration_object_id', ':oldObjectId')
)
)
->setParameter('newObjectName', $newObjectName)
->setParameter('newObjectId', $newObjectId)
->setParameter('integration', $integration)
->setParameter('oldObjectName', $oldObjectName)
->setParameter('oldObjectId', $oldObjectId);
return $qb->executeStatement();
}
public function updateInternalObjectId(int $internalObjectId, int $id): int
{
$qb = $this->getEntityManager()->getConnection()->createQueryBuilder();
$qb->update(MAUTIC_TABLE_PREFIX.'sync_object_mapping')
->set('internal_object_id', ':internalObjectId')
->where($qb->expr()->eq('id', ':id'))
->setParameter('internalObjectId', $internalObjectId)
->setParameter('id', $id);
return $qb->executeStatement();
}
/**
* This method allows inserting a new record when an ORM way is not possible.
* For example, when coping with \Doctrine\DBAL\Exception\RetryableException.
*/
public function insert(string $integration, string $integrationObjectName, string $integrationObjectId, string $internalObjectName, int $internalObjectId, ?\DateTimeInterface $createdAt = null): int
{
$createdAt = $createdAt ?: new \DateTimeImmutable();
$qb = $this->getEntityManager()->getConnection()->createQueryBuilder();
$qb->insert(MAUTIC_TABLE_PREFIX.'sync_object_mapping')
->values([
'integration' => ':integration',
'integration_object_name' => ':integrationObjectName',
'integration_object_id' => ':integrationObjectId',
'internal_object_name' => ':internalObjectName',
'internal_object_id' => ':internalObjectId',
'date_created' => ':date',
'last_sync_date' => ':date',
'is_deleted' => ':isDeleted',
'internal_storage' => ':internalStorage',
])
->setParameter('integration', $integration)
->setParameter('integrationObjectName', $integrationObjectName)
->setParameter('integrationObjectId', $integrationObjectId)
->setParameter('internalObjectName', $internalObjectName)
->setParameter('internalObjectId', $internalObjectId)
->setParameter('date', $createdAt->format(DateTimeHelper::FORMAT_DB))
->setParameter('isDeleted', false, Types::BOOLEAN)
->setParameter('internalStorage', [], Types::JSON);
return $qb->executeStatement();
}
/**
* @param string[]|string $objectIds
*
* @return \Doctrine\DBAL\Driver\Statement|int
*/
public function markAsDeleted(string $integration, string $objectName, $objectIds): int
{
$qb = $this->getEntityManager()->getConnection()->createQueryBuilder();
$qb->update(MAUTIC_TABLE_PREFIX.'sync_object_mapping', 'm')
->set('is_deleted', 1)
->where(
$qb->expr()->and(
$qb->expr()->eq('m.integration', ':integration'),
$qb->expr()->eq('m.integration_object_name', ':objectName')
)
)
->setParameter('integration', $integration)
->setParameter('objectName', $objectName);
if (is_array($objectIds)) {
$qb->setParameter('objectId', $objectIds, ArrayParameterType::STRING);
$qb->andWhere($qb->expr()->in('m.integration_object_id', ':objectId'));
} else {
$qb->setParameter('objectId', $objectIds);
$qb->andWhere($qb->expr()->eq('m.integration_object_id', ':objectId'));
}
return $qb->executeStatement();
}
public function deleteEntitiesForObject(int $internalObjectId, string $internalObject): void
{
$qb = $this->_em->createQueryBuilder();
$qb->delete(ObjectMapping::class, 'm');
$qb->where('m.internalObjectName = :internalObject');
$qb->andWhere('m.internalObjectId = :internalObjectId');
$qb->setParameter('internalObject', $internalObject);
$qb->setParameter('internalObjectId', $internalObjectId);
$qb->getQuery()->execute();
}
/**
* @return ObjectMapping[]
*/
public function getIntegrationMappingsForInternalObject(string $internalObject, int $internalObjectId): array
{
$qb = $this->createQueryBuilder('m');
$qb->select('m')
->where(
$qb->expr()->andX(
$qb->expr()->eq('m.internalObjectName', ':internalObject'),
$qb->expr()->eq('m.internalObjectId', ':internalObjectId')
)
)
->setParameter('internalObject', $internalObject)
->setParameter('internalObjectId', $internalObjectId);
return $qb->getQuery()->getResult();
}
/**
* @param string $integration
* @param string $integrationObjectName
* @param string $integrationObjectId
* @param string $internalObjectName
*
* @return mixed[]|null
*/
private function doGetInternalObject($integration, $integrationObjectName, $integrationObjectId, $internalObjectName, ?string $lock = null): ?array
{
$connection = $this->getEntityManager()->getConnection();
$qb = $connection->createQueryBuilder();
$qb->select('*')
->from(MAUTIC_TABLE_PREFIX.'sync_object_mapping', 'i')
->where(
$qb->expr()->and(
$qb->expr()->eq('i.integration', ':integration'),
$qb->expr()->eq('i.integration_object_name', ':integrationObjectName'),
$qb->expr()->eq('i.integration_object_id', ':integrationObjectId'),
$qb->expr()->eq('i.internal_object_name', ':internalObjectName')
)
)
->setParameter('integration', $integration)
->setParameter('integrationObjectName', $integrationObjectName)
->setParameter('integrationObjectId', $integrationObjectId)
->setParameter('internalObjectName', $internalObjectName);
$lock = $lock ? ' '.$lock : '';
$result = $connection->executeQuery($qb->getSQL().$lock, $qb->getParameters(), $qb->getParameterTypes())->fetchAssociative();
return $result ?: null;
}
}