Initial commit: CloudOps infrastructure platform

docker-compose/mautic-setup/mautic-backup-files/docroot/app/bundles/IntegrationsBundle/Sync/Helper/MappingHelper.php

@@ -0,0 +1,236 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mautic\IntegrationsBundle\Sync\Helper;
+
+use Mautic\IntegrationsBundle\Entity\ObjectMapping;
+use Mautic\IntegrationsBundle\Entity\ObjectMappingRepository;
+use Mautic\IntegrationsBundle\Event\InternalObjectFindEvent;
+use Mautic\IntegrationsBundle\IntegrationEvents;
+use Mautic\IntegrationsBundle\Sync\DAO\Mapping\MappingManualDAO;
+use Mautic\IntegrationsBundle\Sync\DAO\Mapping\RemappedObjectDAO;
+use Mautic\IntegrationsBundle\Sync\DAO\Mapping\UpdatedObjectMappingDAO;
+use Mautic\IntegrationsBundle\Sync\DAO\Sync\Order\ObjectChangeDAO;
+use Mautic\IntegrationsBundle\Sync\DAO\Sync\Report\ObjectDAO;
+use Mautic\IntegrationsBundle\Sync\Exception\FieldNotFoundException;
+use Mautic\IntegrationsBundle\Sync\Exception\ObjectDeletedException;
+use Mautic\IntegrationsBundle\Sync\Exception\ObjectNotFoundException;
+use Mautic\IntegrationsBundle\Sync\Exception\ObjectNotSupportedException;
+use Mautic\IntegrationsBundle\Sync\SyncDataExchange\Internal\ObjectProvider;
+use Mautic\IntegrationsBundle\Sync\SyncDataExchange\MauticSyncDataExchange;
+use Mautic\LeadBundle\Field\FieldsWithUniqueIdentifier;
+use Symfony\Component\EventDispatcher\EventDispatcherInterface;
+
+class MappingHelper
+{
+    public function __construct(
+        private FieldsWithUniqueIdentifier $fieldsWithUniqueIdentifier,
+        private ObjectMappingRepository $objectMappingRepository,
+        private ObjectProvider $objectProvider,
+        private EventDispatcherInterface $dispatcher,
+    ) {
+    }
+
+    /**
+     * @throws ObjectDeletedException
+     * @throws ObjectNotFoundException
+     * @throws ObjectNotSupportedException
+     */
+    public function findMauticObject(MappingManualDAO $mappingManualDAO, string $internalObjectName, ObjectDAO $integrationObjectDAO): ObjectDAO
+    {
+        // Check if this contact is already tracked
+        if ($internalObject = $this->objectMappingRepository->getInternalObject(
+            $mappingManualDAO->getIntegration(),
+            $integrationObjectDAO->getObject(),
+            $integrationObjectDAO->getObjectId(),
+            $internalObjectName
+        )) {
+            if ($internalObject['is_deleted']) {
+                throw new ObjectDeletedException();
+            }
+
+            return new ObjectDAO(
+                $internalObjectName,
+                $internalObject['internal_object_id'],
+                new \DateTime($internalObject['last_sync_date'], new \DateTimeZone('UTC'))
+            );
+        }
+
+        // We don't know who this is so search Mautic
+        $uniqueIdentifierFields = $this->fieldsWithUniqueIdentifier->getFieldsWithUniqueIdentifier(['object' => $internalObjectName]);
+        $identifiers            = [];
+
+        foreach ($uniqueIdentifierFields as $field => $fieldLabel) {
+            try {
+                $integrationField = $mappingManualDAO->getIntegrationMappedField($integrationObjectDAO->getObject(), $internalObjectName, $field);
+                if ($integrationValue = $integrationObjectDAO->getField($integrationField)) {
+                    $identifiers[$field] = $integrationValue->getValue()->getNormalizedValue();
+                }
+            } catch (FieldNotFoundException) {
+            }
+        }
+
+        if (empty($identifiers)) {
+            // No fields found to search for contact so return null
+            return new ObjectDAO($internalObjectName, null);
+        }
+
+        try {
+            $event = new InternalObjectFindEvent(
+                $this->objectProvider->getObjectByName($internalObjectName)
+            );
+        } catch (ObjectNotFoundException) {
+            // Throw this exception for BC.
+            throw new ObjectNotSupportedException(MauticSyncDataExchange::NAME, $internalObjectName);
+        }
+
+        $event->setFieldValues($identifiers);
+
+        $this->dispatcher->dispatch(
+            $event,
+            IntegrationEvents::INTEGRATION_FIND_INTERNAL_RECORDS,
+        );
+
+        $foundObjects = $event->getFoundObjects();
+
+        if (!$foundObjects) {
+            // No contacts were found
+            return new ObjectDAO($internalObjectName, null);
+        }
+
+        // Match found!
+        $objectId = $foundObjects[0]['id'];
+
+        // Let's store the relationship since we know it
+        $objectMapping = new ObjectMapping();
+        $objectMapping->setLastSyncDate($integrationObjectDAO->getChangeDateTime())
+            ->setIntegration($mappingManualDAO->getIntegration())
+            ->setIntegrationObjectName($integrationObjectDAO->getObject())
+            ->setIntegrationObjectId($integrationObjectDAO->getObjectId())
+            ->setInternalObjectName($internalObjectName)
+            ->setInternalObjectId($objectId);
+        $this->saveObjectMapping($objectMapping);
+
+        return new ObjectDAO($internalObjectName, $objectId);
+    }
+
+    /**
+     * Returns corresponding Mautic entity class name for the given Mautic object.
+     *
+     * @throws ObjectNotSupportedException
+     */
+    public function getMauticEntityClassName(string $internalObject): string
+    {
+        try {
+            return $this->objectProvider->getObjectByName($internalObject)->getEntityName();
+        } catch (ObjectNotFoundException) {
+            // Throw this exception instead to keep BC.
+            throw new ObjectNotSupportedException(MauticSyncDataExchange::NAME, $internalObject);
+        }
+    }
+
+    /**
+     * @throws ObjectDeletedException
+     */
+    public function findIntegrationObject(string $integration, string $integrationObjectName, ObjectDAO $internalObjectDAO): ObjectDAO
+    {
+        if ($integrationObject = $this->objectMappingRepository->getIntegrationObject(
+            $integration,
+            $internalObjectDAO->getObject(),
+            $internalObjectDAO->getObjectId(),
+            $integrationObjectName
+        )) {
+            if ($integrationObject['is_deleted']) {
+                throw new ObjectDeletedException();
+            }
+
+            return new ObjectDAO(
+                $integrationObjectName,
+                $integrationObject['integration_object_id'],
+                new \DateTime($integrationObject['last_sync_date'], new \DateTimeZone('UTC'))
+            );
+        }
+
+        return new ObjectDAO($integrationObjectName, null);
+    }
+
+    /**
+     * @param ObjectMapping[] $mappings
+     */
+    public function saveObjectMappings(array $mappings): void
+    {
+        foreach ($mappings as $mapping) {
+            $this->saveObjectMapping($mapping);
+        }
+    }
+
+    public function updateObjectMappings(array $mappings): void
+    {
+        foreach ($mappings as $mapping) {
+            try {
+                $this->updateObjectMapping($mapping);
+            } catch (ObjectNotFoundException) {
+                continue;
+            }
+        }
+    }
+
+    /**
+     * @param RemappedObjectDAO[] $mappings
+     */
+    public function remapIntegrationObjects(array $mappings): void
+    {
+        foreach ($mappings as $mapping) {
+            $this->objectMappingRepository->updateIntegrationObject(
+                $mapping->getIntegration(),
+                $mapping->getOldObjectName(),
+                $mapping->getOldObjectId(),
+                $mapping->getNewObjectName(),
+                $mapping->getNewObjectId()
+            );
+        }
+    }
+
+    /**
+     * @param ObjectChangeDAO[] $objects
+     */
+    public function markAsDeleted(array $objects): void
+    {
+        foreach ($objects as $object) {
+            $this->objectMappingRepository->markAsDeleted($object->getIntegration(), $object->getObject(), $object->getObjectId());
+        }
+    }
+
+    private function saveObjectMapping(ObjectMapping $objectMapping): void
+    {
+        $this->objectMappingRepository->saveEntity($objectMapping);
+        $this->objectMappingRepository->detachEntity($objectMapping);
+    }
+
+    /**
+     * @throws ObjectNotFoundException
+     */
+    private function updateObjectMapping(UpdatedObjectMappingDAO $updatedObjectMappingDAO): void
+    {
+        /** @var ObjectMapping $objectMapping */
+        $objectMapping = $this->objectMappingRepository->findOneBy(
+            [
+                'integration'           => $updatedObjectMappingDAO->getIntegration(),
+                'integrationObjectName' => $updatedObjectMappingDAO->getIntegrationObjectName(),
+                'integrationObjectId'   => $updatedObjectMappingDAO->getIntegrationObjectId(),
+            ]
+        );
+
+        if (!$objectMapping) {
+            throw new ObjectNotFoundException($updatedObjectMappingDAO->getIntegrationObjectName().':'.$updatedObjectMappingDAO->getIntegrationObjectId());
+        }
+
+        $objectMapping->setLastSyncDate($updatedObjectMappingDAO->getObjectModifiedDate());
+
+        $this->saveObjectMapping($objectMapping);
+
+        // Make the ObjectMapping available to the IntegrationEvents::INTEGRATION_BATCH_SYNC_COMPLETED_* events
+        $updatedObjectMappingDAO->setObjectMapping($objectMapping);
+    }
+}

docker-compose/mautic-setup/mautic-backup-files/docroot/app/bundles/IntegrationsBundle/Sync/Helper/RelationsHelper.php

@@ -0,0 +1,113 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mautic\IntegrationsBundle\Sync\Helper;
+
+use Mautic\IntegrationsBundle\Sync\DAO\Mapping\MappingManualDAO;
+use Mautic\IntegrationsBundle\Sync\DAO\Sync\Report\ObjectDAO;
+use Mautic\IntegrationsBundle\Sync\DAO\Sync\Report\RelationDAO;
+use Mautic\IntegrationsBundle\Sync\DAO\Sync\Report\ReportDAO;
+use Mautic\IntegrationsBundle\Sync\Exception\InternalIdNotFoundException;
+use Mautic\IntegrationsBundle\Sync\Exception\ObjectNotFoundException;
+
+class RelationsHelper
+{
+    /**
+     * @var ObjectDAO[]
+     */
+    private array $objectsToSynchronize = [];
+
+    public function __construct(
+        private MappingHelper $mappingHelper,
+    ) {
+    }
+
+    public function processRelations(MappingManualDAO $mappingManualDao, ReportDAO $syncReport): void
+    {
+        $this->objectsToSynchronize = [];
+        foreach ($syncReport->getRelations() as $relationObject) {
+            if (0 < $relationObject->getRelObjectInternalId()) {
+                continue;
+            }
+
+            $this->processRelation($mappingManualDao, $syncReport, $relationObject);
+        }
+    }
+
+    public function getObjectsToSynchronize(): array
+    {
+        return $this->objectsToSynchronize;
+    }
+
+    /**
+     * @throws \Mautic\IntegrationsBundle\Sync\Exception\FieldNotFoundException
+     * @throws \Mautic\IntegrationsBundle\Sync\Exception\ObjectDeletedException
+     * @throws \Mautic\IntegrationsBundle\Sync\Exception\ObjectNotSupportedException
+     */
+    private function processRelation(MappingManualDAO $mappingManualDao, ReportDAO $syncReport, RelationDAO $relationObject): void
+    {
+        $relObjectDao = new ObjectDAO($relationObject->getRelObjectName(), $relationObject->getRelObjectIntegrationId());
+
+        try {
+            $internalObjectName = $this->getInternalObjectName($mappingManualDao, $relationObject->getRelObjectName());
+            $internalObjectId   = $this->getInternalObjectId($mappingManualDao, $relationObject, $relObjectDao);
+            $this->addObjectInternalId($internalObjectId, $internalObjectName, $relationObject, $syncReport);
+        } catch (ObjectNotFoundException) {
+            return; // We are not mapping this object
+        } catch (InternalIdNotFoundException) {
+            $this->objectsToSynchronize[] = $relObjectDao;
+        }
+    }
+
+    /**
+     * @throws InternalIdNotFoundException
+     */
+    private function getInternalObjectId(MappingManualDAO $mappingManualDao, RelationDAO $relationObject, ObjectDAO $relObjectDao): int
+    {
+        $relObject        = $this->findInternalObject($mappingManualDao, $relationObject->getRelObjectName(), $relObjectDao);
+        $internalObjectId = (int) $relObject->getObjectId();
+
+        if ($internalObjectId) {
+            return $internalObjectId;
+        }
+
+        throw new InternalIdNotFoundException($relationObject->getRelObjectName());
+    }
+
+    /**
+     * @throws ObjectNotFoundException
+     * @throws \Mautic\IntegrationsBundle\Sync\Exception\ObjectDeletedException
+     * @throws \Mautic\IntegrationsBundle\Sync\Exception\ObjectNotSupportedException
+     */
+    private function findInternalObject(MappingManualDAO $mappingManualDao, string $relObjectName, ObjectDAO $objectDao): ObjectDAO
+    {
+        $internalObjectsName = $this->getInternalObjectName($mappingManualDao, $relObjectName);
+
+        return $this->mappingHelper->findMauticObject($mappingManualDao, $internalObjectsName, $objectDao);
+    }
+
+    /**
+     * @throws \Mautic\IntegrationsBundle\Sync\Exception\FieldNotFoundException
+     */
+    private function addObjectInternalId(int $relObjectId, string $relInternalType, RelationDAO $relationObject, ReportDAO $syncReport): void
+    {
+        $relationObject->setRelObjectInternalId($relObjectId);
+        $objectDAO      = $syncReport->getObject($relationObject->getObjectName(), $relationObject->getObjectIntegrationId());
+        $referenceValue = $objectDAO->getField($relationObject->getRelFieldName())->getValue()->getNormalizedValue();
+        $referenceValue->setType($relInternalType);
+        $referenceValue->setValue($relObjectId);
+    }
+
+    /**
+     * @return mixed
+     *
+     * @throws ObjectNotFoundException
+     */
+    private function getInternalObjectName(MappingManualDAO $mappingManualDao, string $relObjectName)
+    {
+        $internalObjectsNames = $mappingManualDao->getMappedInternalObjectsNames($relObjectName);
+
+        return $internalObjectsNames[0];
+    }
+}

docker-compose/mautic-setup/mautic-backup-files/docroot/app/bundles/IntegrationsBundle/Sync/Helper/SyncDateHelper.php

@@ -0,0 +1,150 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mautic\IntegrationsBundle\Sync\Helper;
+
+use Doctrine\DBAL\Connection;
+use Mautic\IntegrationsBundle\Sync\SyncDataExchange\MauticSyncDataExchange;
+
+class SyncDateHelper
+{
+    private ?\DateTimeInterface $syncFromDateTime = null;
+
+    private ?\DateTimeInterface $syncToDateTime = null;
+
+    private ?\DateTimeImmutable $syncDateTime = null;
+
+    /**
+     * @var \DateTimeInterface[]
+     */
+    private array $lastObjectSyncDates = [];
+
+    private ?\DateTimeInterface $internalSyncStartDateTime = null;
+
+    public function __construct(
+        private Connection $connection,
+    ) {
+    }
+
+    public function setSyncDateTimes(?\DateTimeInterface $fromDateTime = null, ?\DateTimeInterface $toDateTime = null): void
+    {
+        $this->syncFromDateTime    = $fromDateTime;
+        $this->syncToDateTime      = $toDateTime;
+        $this->syncDateTime        = new \DateTimeImmutable('now', new \DateTimeZone('UTC'));
+        $this->lastObjectSyncDates = [];
+    }
+
+    public function getSyncFromDateTime(string $integration, string $object): \DateTimeInterface
+    {
+        if ($this->syncFromDateTime) {
+            // The command requested a specific start date so use it
+
+            return $this->syncFromDateTime;
+        }
+
+        $key = $integration.$object;
+        if (isset($this->lastObjectSyncDates[$key])) {
+            // Use the same sync date for integrations to paginate properly
+
+            return $this->lastObjectSyncDates[$key];
+        }
+
+        if (MauticSyncDataExchange::NAME !== $integration && $lastSync = $this->getLastSyncDateForObject($integration, $object)) {
+            // Use the latest sync date recorded
+            $this->lastObjectSyncDates[$key] = $lastSync;
+        } else {
+            // Otherwise, just sync the last 24 hours
+            $this->lastObjectSyncDates[$key] = new \DateTimeImmutable('-24 hours', new \DateTimeZone('UTC'));
+        }
+
+        return $this->lastObjectSyncDates[$key];
+    }
+
+    public function getSyncToDateTime(): ?\DateTimeInterface
+    {
+        if ($this->syncToDateTime) {
+            return $this->syncToDateTime;
+        }
+
+        return $this->syncDateTime;
+    }
+
+    public function getSyncDateTime(): ?\DateTimeInterface
+    {
+        return $this->syncDateTime;
+    }
+
+    /**
+     * @return \DateTimeImmutable|null
+     */
+    public function getLastSyncDateForObject(string $integration, string $object): ?\DateTimeInterface
+    {
+        $qb = $this->connection->createQueryBuilder();
+
+        $result = $qb
+            ->select('max(m.last_sync_date)')
+            ->from(MAUTIC_TABLE_PREFIX.'sync_object_mapping', 'm')
+            ->where(
+                $qb->expr()->eq('m.integration', ':integration'),
+                $qb->expr()->eq('m.integration_object_name', ':object')
+            )
+            ->setParameter('integration', $integration)
+            ->setParameter('object', $object)
+            ->executeQuery()
+            ->fetchOne();
+
+        if (!$result) {
+            return null;
+        }
+
+        $lastSync = new \DateTimeImmutable($result, new \DateTimeZone('UTC'));
+
+        // The last sync is out of the requested sync date/time range
+        if ($this->syncFromDateTime && $lastSync < $this->syncFromDateTime) {
+            return null;
+        }
+
+        // The last sync is out of the requested sync date/time range
+        if ($lastSync > $this->getSyncToDateTime()) {
+            return null;
+        }
+
+        return $lastSync;
+    }
+
+    public function getInternalSyncStartDateTime(): ?\DateTimeInterface
+    {
+        return $this->internalSyncStartDateTime;
+    }
+
+    public function setInternalSyncStartDateTime(): void
+    {
+        if ($this->internalSyncStartDateTime) {
+            return;
+        }
+
+        $this->internalSyncStartDateTime = $this->calculateInternalSyncStartDateTime();
+    }
+
+    private function calculateInternalSyncStartDateTime(): \DateTimeInterface
+    {
+        $now = new \DateTimeImmutable('now', new \DateTimeZone('UTC'));
+        // If there is no syncToDateTime value use "now"
+        if (!$this->getSyncToDateTime()) {
+            return $now;
+        }
+
+        // Clone it so that we don't modify the initial object
+        $syncToDateTime = clone $this->getSyncToDateTime();
+
+        // We should compare in UTC timezone
+        if (method_exists($syncToDateTime, 'setTimezone')) {
+            $syncToDateTime->setTimezone(new \DateTimeZone('UTC'));
+        }
+
+        // If syncToDate is less than now then use syncToDate, because otherwise we may delete
+        // changes that aren't supposed to be deleted from the sync_object_field_change_report table
+        return min($now, $syncToDateTime);
+    }
+}