Initial commit: CloudOps infrastructure platform
This commit is contained in:
root
@@ -0,0 +1,62 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Mautic\LeadBundle\Command;
|
||||
|
||||
use Mautic\CoreBundle\Helper\CoreParametersHelper;
|
||||
use Mautic\CoreBundle\Helper\DateTimeHelper;
|
||||
use Mautic\CoreBundle\Helper\ExitCode;
|
||||
use Symfony\Component\Console\Command\Command;
|
||||
use Symfony\Component\Console\Input\InputArgument;
|
||||
use Symfony\Component\Console\Input\InputInterface;
|
||||
use Symfony\Component\Console\Output\OutputInterface;
|
||||
|
||||
class CleanupExportedFilesCommand extends Command
|
||||
{
|
||||
public const COMMAND_NAME = 'mautic:contacts:cleanup_exported_files';
|
||||
|
||||
/**
|
||||
* @var string
|
||||
*/
|
||||
private const CLEANUP_DAYS = 'cleanupAfterDays';
|
||||
|
||||
public function __construct(
|
||||
private CoreParametersHelper $coreParametersHelper,
|
||||
) {
|
||||
parent::__construct();
|
||||
}
|
||||
|
||||
protected function configure(): void
|
||||
{
|
||||
$this
|
||||
->setName(self::COMMAND_NAME)
|
||||
->setDescription('Remove contact export cache files from `contacts_export` directory if file is older than the week/7 days')
|
||||
->addArgument(self::CLEANUP_DAYS, InputArgument::OPTIONAL, 'Remove exported files after days');
|
||||
|
||||
parent::configure();
|
||||
}
|
||||
|
||||
protected function execute(InputInterface $input, OutputInterface $output): int
|
||||
{
|
||||
$days = $input->getArgument(self::CLEANUP_DAYS);
|
||||
if (!$days) {
|
||||
$days = $this->coreParametersHelper->get('clear_export_files_after_days');
|
||||
}
|
||||
|
||||
$dateHelper = new DateTimeHelper();
|
||||
$date = $dateHelper->getUtcDateTime()->modify('-'.(int) $days.' days');
|
||||
$cleanUpTimestamp = $date->getTimestamp();
|
||||
|
||||
$downloadFolder = $this->coreParametersHelper->get('contact_export_dir');
|
||||
$contactExportedAllFiles = glob($downloadFolder.'/contacts_export_*');
|
||||
|
||||
foreach ($contactExportedAllFiles as $file) {
|
||||
if (filectime($file) <= $cleanUpTimestamp) {
|
||||
@unlink($file);
|
||||
}
|
||||
}
|
||||
|
||||
return ExitCode::SUCCESS;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Mautic\LeadBundle\Command;
|
||||
|
||||
use Mautic\CoreBundle\Helper\ExitCode;
|
||||
use Mautic\CoreBundle\ProcessSignal\Exception\SignalCaughtException;
|
||||
use Mautic\CoreBundle\ProcessSignal\ProcessSignalService;
|
||||
use Mautic\CoreBundle\Twig\Helper\FormatterHelper;
|
||||
use Mautic\LeadBundle\Event\ContactExportSchedulerEvent;
|
||||
use Mautic\LeadBundle\LeadEvents;
|
||||
use Mautic\LeadBundle\Model\ContactExportSchedulerModel;
|
||||
use Symfony\Component\Console\Attribute\AsCommand;
|
||||
use Symfony\Component\Console\Command\Command;
|
||||
use Symfony\Component\Console\Input\InputInterface;
|
||||
use Symfony\Component\Console\Input\InputOption;
|
||||
use Symfony\Component\Console\Output\OutputInterface;
|
||||
use Symfony\Component\EventDispatcher\EventDispatcherInterface;
|
||||
|
||||
#[AsCommand(
|
||||
name: ContactScheduledExportCommand::COMMAND_NAME,
|
||||
description: 'Export contacts which are scheduled in `contact_export_scheduler` table.'
|
||||
)]
|
||||
class ContactScheduledExportCommand extends Command
|
||||
{
|
||||
private const PICK_SCHEDULED_EXPORTS_LIMIT = 10;
|
||||
|
||||
public const COMMAND_NAME = 'mautic:contacts:scheduled_export';
|
||||
|
||||
public function __construct(
|
||||
private ContactExportSchedulerModel $contactExportSchedulerModel,
|
||||
private EventDispatcherInterface $eventDispatcher,
|
||||
private FormatterHelper $formatterHelper,
|
||||
private ProcessSignalService $processSignalService,
|
||||
) {
|
||||
parent::__construct();
|
||||
}
|
||||
|
||||
protected function configure(): void
|
||||
{
|
||||
$this
|
||||
->addOption(
|
||||
'--ids',
|
||||
null,
|
||||
InputOption::VALUE_REQUIRED,
|
||||
'Comma separated contact_export_scheduler ids.'
|
||||
);
|
||||
|
||||
parent::configure();
|
||||
}
|
||||
|
||||
protected function execute(InputInterface $input, OutputInterface $output): int
|
||||
{
|
||||
$this->processSignalService->registerSignalHandler(
|
||||
fn (int $signal) => $output->writeln(sprintf('Signal %d caught.', $signal))
|
||||
);
|
||||
|
||||
$ids = $this->formatterHelper->simpleCsvToArray($input->getOption('ids'), 'int');
|
||||
|
||||
if ($ids) {
|
||||
$contactExportSchedulers = $this->contactExportSchedulerModel->getRepository()->findBy(['id' => $ids]);
|
||||
} else {
|
||||
$contactExportSchedulers = $this->contactExportSchedulerModel->getRepository()
|
||||
->findBy([], [], self::PICK_SCHEDULED_EXPORTS_LIMIT);
|
||||
}
|
||||
|
||||
$count = 0;
|
||||
|
||||
try {
|
||||
foreach ($contactExportSchedulers as $contactExportScheduler) {
|
||||
$contactExportSchedulerEvent = new ContactExportSchedulerEvent($contactExportScheduler);
|
||||
$this->eventDispatcher->dispatch($contactExportSchedulerEvent, LeadEvents::CONTACT_EXPORT_PREPARE_FILE);
|
||||
$this->eventDispatcher->dispatch($contactExportSchedulerEvent, LeadEvents::CONTACT_EXPORT_SEND_EMAIL);
|
||||
$this->eventDispatcher->dispatch($contactExportSchedulerEvent, LeadEvents::POST_CONTACT_EXPORT_SEND_EMAIL);
|
||||
++$count;
|
||||
}
|
||||
|
||||
$output->writeln('Contact export email(s) sent: '.$count);
|
||||
|
||||
return ExitCode::SUCCESS;
|
||||
} catch (SignalCaughtException) {
|
||||
return ExitCode::TERMINATED;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Mautic\LeadBundle\Command;
|
||||
|
||||
use Mautic\CoreBundle\Service\ProcessQueue;
|
||||
use Mautic\LeadBundle\Deduplicate\ContactDeduper;
|
||||
use Symfony\Component\Console\Attribute\AsCommand;
|
||||
use Symfony\Component\Console\Command\Command;
|
||||
use Symfony\Component\Console\Helper\ProgressBar;
|
||||
use Symfony\Component\Console\Input\InputInterface;
|
||||
use Symfony\Component\Console\Input\InputOption;
|
||||
use Symfony\Component\Console\Output\OutputInterface;
|
||||
use Symfony\Component\DependencyInjection\ParameterBag\ParameterBagInterface;
|
||||
use Symfony\Component\Process\Process;
|
||||
use Symfony\Component\Stopwatch\Stopwatch;
|
||||
|
||||
#[AsCommand(
|
||||
name: DeduplicateCommand::NAME,
|
||||
description: 'Merge contacts based on same unique identifiers'
|
||||
)]
|
||||
class DeduplicateCommand extends Command
|
||||
{
|
||||
public const NAME = 'mautic:contacts:deduplicate';
|
||||
|
||||
public function __construct(
|
||||
private ContactDeduper $contactDeduper,
|
||||
private ParameterBagInterface $params,
|
||||
) {
|
||||
parent::__construct();
|
||||
}
|
||||
|
||||
public function configure(): void
|
||||
{
|
||||
parent::configure();
|
||||
|
||||
$this
|
||||
->addOption(
|
||||
'--newer-into-older',
|
||||
null,
|
||||
InputOption::VALUE_NONE,
|
||||
'By default, this command will merge older contacts and activity into the newer. Use this flag to reverse that behavior.'
|
||||
)
|
||||
->addOption(
|
||||
'--batch',
|
||||
null,
|
||||
InputOption::VALUE_REQUIRED,
|
||||
'How many contact duplicates to process at once. Defaults to 100.',
|
||||
100
|
||||
)
|
||||
->addOption(
|
||||
'--processes',
|
||||
null,
|
||||
InputOption::VALUE_REQUIRED,
|
||||
'The commands can run in multiple PHP processes. This option defines how many processes to run. Defaults to 1.',
|
||||
1
|
||||
)
|
||||
->setHelp(
|
||||
<<<'EOT'
|
||||
The <info>%command.name%</info> command will dedpulicate contacts based on unique identifier values.
|
||||
|
||||
<info>php %command.full_name%</info>
|
||||
EOT
|
||||
);
|
||||
}
|
||||
|
||||
protected function execute(InputInterface $input, OutputInterface $output): int
|
||||
{
|
||||
$newerIntoOlder = (bool) $input->getOption('newer-into-older');
|
||||
$batch = (int) $input->getOption('batch');
|
||||
$processes = (int) $input->getOption('processes');
|
||||
$uniqueFields = $this->contactDeduper->getUniqueFields('lead');
|
||||
$duplicateCount = $this->contactDeduper->countDuplicatedContacts(array_keys($uniqueFields));
|
||||
$stopwatch = new Stopwatch();
|
||||
|
||||
if (!$duplicateCount) {
|
||||
$output->writeln('<error>No contacts to deduplicate.</error>');
|
||||
|
||||
return Command::FAILURE;
|
||||
}
|
||||
|
||||
$stopwatch->start('deduplicate');
|
||||
|
||||
$output->writeln('Deduplicating contacts based on unique identifiers: '.implode(', ', $uniqueFields));
|
||||
$output->writeln("{$duplicateCount} contacts found to deduplicate");
|
||||
|
||||
$processQueue = new ProcessQueue($processes);
|
||||
$processCount = (int) ceil($duplicateCount / $batch);
|
||||
|
||||
$output->writeln('');
|
||||
$output->writeln("Finding duplicates and creating processes for deduplication. {$processCount} processes will be queued.");
|
||||
|
||||
$contactIds = $this->contactDeduper->getDuplicateContactIds(array_keys($uniqueFields));
|
||||
$contactIdChunks = array_chunk($contactIds, $batch);
|
||||
foreach ($contactIdChunks as $contactIdBatch) {
|
||||
$command = [
|
||||
$this->params->get('kernel.project_dir').'/bin/console',
|
||||
DeduplicateIdsCommand::NAME,
|
||||
'--contact-ids',
|
||||
implode(',', $contactIdBatch),
|
||||
'-e',
|
||||
MAUTIC_ENV,
|
||||
];
|
||||
|
||||
if ($newerIntoOlder) {
|
||||
$command[] = '--newer-into-older';
|
||||
}
|
||||
|
||||
$envParams = [
|
||||
'db_table_prefix' => MAUTIC_TABLE_PREFIX,
|
||||
'contact_unique_identifiers_operator' => $this->params->get('mautic.contact_unique_identifiers_operator'),
|
||||
];
|
||||
|
||||
$processQueue->enqueue(new Process($command, null, ['MAUTIC_CONFIG_PARAMETERS' => json_encode($envParams)]));
|
||||
}
|
||||
|
||||
$output->writeln('');
|
||||
$output->writeln("Starting to execute the {$processCount} processes for deduplication. {$processes} processes will be executed in parallel.");
|
||||
|
||||
$progressBar = new ProgressBar($output, $processCount);
|
||||
$progressBar->setFormat('debug');
|
||||
$progressBar->start();
|
||||
|
||||
$processQueue->refresh();
|
||||
|
||||
while ($processQueue->isProcessing()) {
|
||||
usleep(100);
|
||||
$processQueue->refresh();
|
||||
$progressBar->setProgress($processQueue->getProcessedCount());
|
||||
}
|
||||
|
||||
$output->writeln('');
|
||||
$output->writeln('');
|
||||
$output->writeln('All processes have finished. The output of each process is below.');
|
||||
|
||||
foreach ($processQueue->getProcessed() as $process) {
|
||||
$output->writeln("<comment>{$process->getCommandLine()}</comment>");
|
||||
if (0 === $process->getExitCode()) {
|
||||
$output->writeln("<info>{$process->getOutput()}</info>");
|
||||
} else {
|
||||
$output->writeln("<error>{$process->getErrorOutput()}</error>");
|
||||
}
|
||||
}
|
||||
|
||||
$progressBar->finish();
|
||||
|
||||
$event = $stopwatch->stop('deduplicate');
|
||||
$output->writeln('');
|
||||
$output->writeln("Duration: {$event->getDuration()} ms, Memory: {$event->getMemory()} bytes");
|
||||
|
||||
return Command::SUCCESS;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Mautic\LeadBundle\Command;
|
||||
|
||||
use Mautic\LeadBundle\Deduplicate\ContactDeduper;
|
||||
use Symfony\Component\Console\Attribute\AsCommand;
|
||||
use Symfony\Component\Console\Command\Command;
|
||||
use Symfony\Component\Console\Helper\ProgressBar;
|
||||
use Symfony\Component\Console\Input\InputInterface;
|
||||
use Symfony\Component\Console\Input\InputOption;
|
||||
use Symfony\Component\Console\Output\OutputInterface;
|
||||
use Symfony\Component\Stopwatch\Stopwatch;
|
||||
|
||||
#[AsCommand(
|
||||
name: DeduplicateIdsCommand::NAME,
|
||||
description: 'Merge contacts based on same unique identifiers'
|
||||
)]
|
||||
class DeduplicateIdsCommand extends Command
|
||||
{
|
||||
public const NAME = 'mautic:contacts:deduplicate:ids';
|
||||
|
||||
public function __construct(
|
||||
private ContactDeduper $contactDeduper,
|
||||
) {
|
||||
parent::__construct();
|
||||
}
|
||||
|
||||
public function configure(): void
|
||||
{
|
||||
parent::configure();
|
||||
|
||||
$this
|
||||
->addOption(
|
||||
'--newer-into-older',
|
||||
null,
|
||||
InputOption::VALUE_NONE,
|
||||
'By default, this command will merge older contacts and activity into the newer. Use this flag to reverse that behavior.'
|
||||
)
|
||||
->addOption(
|
||||
'--contact-ids',
|
||||
null,
|
||||
InputOption::VALUE_REQUIRED,
|
||||
'Comma separated list of contact IDs to deduplicate. If not provided, all contacts will be deduplicated. Example: --contact-ids=23,3,11'
|
||||
)
|
||||
->setHelp(
|
||||
<<<'EOT'
|
||||
The <info>%command.name%</info> command will dedpulicate contacts based on unique identifier values.
|
||||
|
||||
<info>php %command.full_name%</info>
|
||||
EOT
|
||||
);
|
||||
}
|
||||
|
||||
protected function execute(InputInterface $input, OutputInterface $output): int
|
||||
{
|
||||
$newerIntoOlder = (bool) $input->getOption('newer-into-older');
|
||||
$contactIds = array_filter(explode(',', $input->getOption('contact-ids')));
|
||||
$duplicateCount = count($contactIds);
|
||||
$progressBar = new ProgressBar($output, $duplicateCount);
|
||||
$stopwatch = new Stopwatch();
|
||||
|
||||
if (!$contactIds) {
|
||||
$output->writeln('<error>No contacts to deduplicate.</error>');
|
||||
|
||||
return Command::FAILURE;
|
||||
}
|
||||
|
||||
$output->writeln("{$duplicateCount} contacts passed to deduplicate");
|
||||
|
||||
$progressBar->setFormat('debug');
|
||||
$progressBar->start();
|
||||
$stopwatch->start('deduplicate');
|
||||
|
||||
$contacts = $this->contactDeduper->getContactsByIds($contactIds);
|
||||
$this->contactDeduper->deduplicateContactBatch($contacts, $newerIntoOlder, fn () => $progressBar->advance());
|
||||
|
||||
$progressBar->finish();
|
||||
|
||||
$event = $stopwatch->stop('deduplicate');
|
||||
$output->writeln("Duration: {$event->getDuration()} ms, Memory: {$event->getMemory()} bytes");
|
||||
|
||||
return Command::SUCCESS;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Mautic\LeadBundle\Command;
|
||||
|
||||
use Doctrine\ORM\Exception\ORMException;
|
||||
use Mautic\CoreBundle\Helper\CoreParametersHelper;
|
||||
use Mautic\LeadBundle\Entity\CompanyLeadRepository;
|
||||
use Psr\Log\LoggerInterface;
|
||||
use Symfony\Component\Console\Attribute\AsCommand;
|
||||
use Symfony\Component\Console\Command\Command;
|
||||
use Symfony\Component\Console\Input\InputInterface;
|
||||
use Symfony\Component\Console\Output\OutputInterface;
|
||||
use Symfony\Contracts\Translation\TranslatorInterface;
|
||||
|
||||
#[AsCommand(
|
||||
name: DeleteContactSecondaryCompaniesCommand::NAME,
|
||||
description: "Deletes all contact\'s secondary companies."
|
||||
)]
|
||||
class DeleteContactSecondaryCompaniesCommand extends Command
|
||||
{
|
||||
public const NAME = 'mautic:contact:delete:secondary-companies';
|
||||
|
||||
public function __construct(private LoggerInterface $logger, private TranslatorInterface $translator, private CoreParametersHelper $coreParametersHelper, private CompanyLeadRepository $companyLeadsRepository)
|
||||
{
|
||||
parent::__construct();
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritdoc}
|
||||
*/
|
||||
protected function configure(): void
|
||||
{
|
||||
$this
|
||||
->setHelp(
|
||||
<<<'EOT'
|
||||
The <info>%command.name%</info> command deletes non-primary companies of every contact.
|
||||
|
||||
<info>php %command.full_name%</info>
|
||||
EOT
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritdoc}
|
||||
*/
|
||||
protected function execute(InputInterface $input, OutputInterface $output): int
|
||||
{
|
||||
$allowMultiple = $this->coreParametersHelper->get('contact_allow_multiple_companies');
|
||||
|
||||
// We process only if the config is set to false
|
||||
if ($allowMultiple) {
|
||||
$output->writeln($this->translator->trans('mautic.lead.command.delete_contact_secondary_company.allow_multiple_enabled'));
|
||||
|
||||
return Command::SUCCESS;
|
||||
}
|
||||
|
||||
try {
|
||||
$this->companyLeadsRepository->removeAllSecondaryCompanies();
|
||||
} catch (ORMException $e) {
|
||||
$errorMessage = $this->translator->trans('mautic.lead.command.error', ['%name%' => self::NAME, '%error%' => $e->getMessage()]);
|
||||
$output->writeln($errorMessage);
|
||||
$this->logger->error($errorMessage);
|
||||
}
|
||||
|
||||
$output->writeln($this->translator->trans('mautic.lead.command.delete_contact_secondary_company.success'));
|
||||
|
||||
return Command::SUCCESS;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,187 @@
|
||||
<?php
|
||||
|
||||
namespace Mautic\LeadBundle\Command;
|
||||
|
||||
use Mautic\CoreBundle\Model\NotificationModel;
|
||||
use Mautic\CoreBundle\ProcessSignal\ProcessSignalService;
|
||||
use Mautic\LeadBundle\Entity\Import;
|
||||
use Mautic\LeadBundle\Exception\ImportDelayedException;
|
||||
use Mautic\LeadBundle\Exception\ImportFailedException;
|
||||
use Mautic\LeadBundle\Helper\Progress;
|
||||
use Mautic\LeadBundle\Model\ImportModel;
|
||||
use Mautic\UserBundle\Security\UserTokenSetter;
|
||||
use Psr\Log\LoggerInterface;
|
||||
use Symfony\Component\Console\Attribute\AsCommand;
|
||||
use Symfony\Component\Console\Command\Command;
|
||||
use Symfony\Component\Console\Input\InputInterface;
|
||||
use Symfony\Component\Console\Input\InputOption;
|
||||
use Symfony\Component\Console\Output\OutputInterface;
|
||||
use Symfony\Contracts\Translation\TranslatorInterface;
|
||||
|
||||
/**
|
||||
* CLI Command to import data.
|
||||
*/
|
||||
#[AsCommand(
|
||||
name: ImportCommand::COMMAND_NAME,
|
||||
description: 'Imports data to Mautic'
|
||||
)]
|
||||
class ImportCommand extends Command
|
||||
{
|
||||
public const COMMAND_NAME = 'mautic:import';
|
||||
|
||||
public function __construct(
|
||||
private TranslatorInterface $translator,
|
||||
private ImportModel $importModel,
|
||||
private ProcessSignalService $processSignalService,
|
||||
private UserTokenSetter $userTokenSetter,
|
||||
private LoggerInterface $logger,
|
||||
private NotificationModel $notificationModel,
|
||||
) {
|
||||
parent::__construct();
|
||||
}
|
||||
|
||||
protected function configure(): void
|
||||
{
|
||||
$this
|
||||
->addOption('--id', '-i', InputOption::VALUE_OPTIONAL, 'Specific ID to import. Defaults to next in the queue.', false)
|
||||
->addOption('--limit', '-l', InputOption::VALUE_OPTIONAL, 'Maximum number of records to import for this script execution.', 0)
|
||||
->setHelp(
|
||||
<<<'EOT'
|
||||
The <info>%command.name%</info> command starts to import CSV files when some are created.
|
||||
|
||||
<info>php %command.full_name%</info>
|
||||
EOT
|
||||
);
|
||||
}
|
||||
|
||||
protected function execute(InputInterface $input, OutputInterface $output): int
|
||||
{
|
||||
$start = microtime(true);
|
||||
$progress = new Progress($output);
|
||||
$id = (int) $input->getOption('id');
|
||||
$limit = (int) $input->getOption('limit');
|
||||
|
||||
$this->processSignalService->registerSignalHandler(fn (int $signal) => $output->writeln(sprintf('Signal %d caught.', $signal)));
|
||||
|
||||
if ($id) {
|
||||
$import = $this->importModel->getEntity($id);
|
||||
|
||||
// This specific import was not found
|
||||
if (!$import) {
|
||||
$output->writeln('<error>'.$this->translator->trans('mautic.core.error.notfound', [], 'flashes').'</error>');
|
||||
|
||||
return Command::FAILURE;
|
||||
}
|
||||
} else {
|
||||
$import = $this->importModel->getImportToProcess();
|
||||
|
||||
// No import waiting in the queue. Finish silently.
|
||||
if (null === $import) {
|
||||
return Command::SUCCESS;
|
||||
}
|
||||
}
|
||||
|
||||
$user = $import->getModifiedBy();
|
||||
|
||||
if (!$user) {
|
||||
throw new \RuntimeException('Import does not have "modifiedBy" property set.');
|
||||
}
|
||||
|
||||
$this->userTokenSetter->setUser($user);
|
||||
|
||||
$output->writeln('<info>'.$this->translator->trans(
|
||||
'mautic.lead.import.is.starting',
|
||||
[
|
||||
'%id%' => $import->getId(),
|
||||
'%lines%' => $import->getLineCount(),
|
||||
]
|
||||
).'</info>');
|
||||
|
||||
try {
|
||||
$this->importModel->beginImport($import, $progress, $limit, $start);
|
||||
} catch (ImportFailedException $e) {
|
||||
$output->writeln('<error>'.$this->translator->trans(
|
||||
'mautic.lead.import.failed',
|
||||
[
|
||||
'%reason%' => $import->getStatusInfo(),
|
||||
]
|
||||
).'</error>');
|
||||
|
||||
$this->logError($import, $e);
|
||||
|
||||
$this->notify(
|
||||
$import,
|
||||
$start,
|
||||
$this->translator->trans('mautic.lead.import.failed', ['%reason%' => $import->getStatusInfo()]),
|
||||
'error'
|
||||
);
|
||||
|
||||
return Command::FAILURE;
|
||||
} catch (ImportDelayedException $e) {
|
||||
$output->writeln('<info>'.$this->translator->trans(
|
||||
'mautic.lead.import.delayed',
|
||||
[
|
||||
'%reason%' => $import->getStatusInfo(),
|
||||
]
|
||||
).'</info>');
|
||||
|
||||
$this->logError($import, $e);
|
||||
|
||||
$this->notify(
|
||||
$import,
|
||||
$start,
|
||||
$this->translator->trans('mautic.lead.import.delayed', ['%reason%' => $import->getStatusInfo()]),
|
||||
'warning'
|
||||
);
|
||||
|
||||
return Command::FAILURE;
|
||||
}
|
||||
|
||||
// Success
|
||||
$output->writeln('<info>'.$this->translator->trans(
|
||||
'mautic.lead.import.result',
|
||||
[
|
||||
'%lines%' => $import->getProcessedRows(),
|
||||
'%created%' => $import->getInsertedCount(),
|
||||
'%updated%' => $import->getUpdatedCount(),
|
||||
'%ignored%' => $import->getIgnoredCount(),
|
||||
'%time%' => round(microtime(true) - $start, 2),
|
||||
]
|
||||
).'</info>');
|
||||
|
||||
// Notification is now handled in ImportModel::beginImport to avoid duplicates
|
||||
// and to include the link to the imported file
|
||||
|
||||
return Command::SUCCESS;
|
||||
}
|
||||
|
||||
private function logError(Import $import, \Exception $exception): void
|
||||
{
|
||||
$message = ' Import id: '.$import->getId();
|
||||
$message .= ' Import Status: '.$import->getStatus();
|
||||
$message .= ' Reason: '.$import->getStatusInfo();
|
||||
$message .= ' Exception: '.$exception;
|
||||
|
||||
$this->logger->warning($message);
|
||||
}
|
||||
|
||||
private function notify(Import $import, float $start, string $header, string $type = 'info'): void
|
||||
{
|
||||
$this->notificationModel->addNotification(
|
||||
$this->translator->trans(
|
||||
'mautic.lead.import.result',
|
||||
[
|
||||
'%lines%' => $import->getProcessedRows(),
|
||||
'%created%' => $import->getInsertedCount(),
|
||||
'%updated%' => $import->getUpdatedCount(),
|
||||
'%ignored%' => $import->getIgnoredCount(),
|
||||
'%time%' => round(microtime(true) - $start, 2),
|
||||
]
|
||||
),
|
||||
$type,
|
||||
false,
|
||||
$header,
|
||||
'ri-download-line'
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Mautic\LeadBundle\Command;
|
||||
|
||||
use Mautic\CoreBundle\Helper\ExitCode;
|
||||
use Mautic\LeadBundle\Entity\LeadListRepository;
|
||||
use Mautic\LeadBundle\Helper\SegmentCountCacheHelper;
|
||||
use Psr\Cache\InvalidArgumentException;
|
||||
use Symfony\Component\Console\Command\Command;
|
||||
use Symfony\Component\Console\Input\InputInterface;
|
||||
use Symfony\Component\Console\Output\OutputInterface;
|
||||
|
||||
class SegmentCountCacheCommand extends Command
|
||||
{
|
||||
public const COMMAND_NAME = 'lead:list:count-cache-update';
|
||||
|
||||
public function __construct(
|
||||
private LeadListRepository $leadListRepository,
|
||||
private SegmentCountCacheHelper $segmentCountCacheHelper,
|
||||
) {
|
||||
parent::__construct();
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritdoc}
|
||||
*/
|
||||
protected function configure(): void
|
||||
{
|
||||
$this->setName(self::COMMAND_NAME)
|
||||
->setDescription('Update segment count cache for changed segments.');
|
||||
}
|
||||
|
||||
/**
|
||||
* @throws InvalidArgumentException
|
||||
*/
|
||||
protected function execute(InputInterface $input, OutputInterface $output): int
|
||||
{
|
||||
$segmentsForRecount = $this->getAllSegmentsForRecount();
|
||||
if (count($segmentsForRecount) > 0) {
|
||||
$totalLeadCount = $this->leadListRepository->getLeadCount($segmentsForRecount);
|
||||
if (!is_array($totalLeadCount)) {
|
||||
$totalLeadCount = [$segmentsForRecount[0] => $totalLeadCount];
|
||||
}
|
||||
foreach ($totalLeadCount as $segmentId => $leadCount) {
|
||||
$this->segmentCountCacheHelper->setSegmentContactCount((int) $segmentId, (int) $leadCount);
|
||||
}
|
||||
}
|
||||
$output->writeln(sprintf('<info>%s segment\'s contact count have been updated.</info>', count($segmentsForRecount)));
|
||||
|
||||
return ExitCode::SUCCESS;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return int[]
|
||||
*/
|
||||
private function getAllSegmentsForRecount(): array
|
||||
{
|
||||
$segmentsForRecount = [];
|
||||
$segmentIds = $this->leadListRepository->getLists();
|
||||
foreach ($segmentIds as $segment) {
|
||||
$segmentId = $segment['id'];
|
||||
if ($this->segmentCountCacheHelper->hasSegmentIdForReCount($segmentId)) {
|
||||
$segmentsForRecount[] = $segmentId;
|
||||
}
|
||||
}
|
||||
|
||||
return $segmentsForRecount;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Mautic\LeadBundle\Command;
|
||||
|
||||
use Mautic\CoreBundle\Command\ModeratedCommand;
|
||||
use Mautic\CoreBundle\Helper\CoreParametersHelper;
|
||||
use Mautic\CoreBundle\Helper\PathsHelper;
|
||||
use Mautic\LeadBundle\Event\GetStatDataEvent;
|
||||
use Symfony\Component\Console\Command\Command;
|
||||
use Symfony\Component\Console\Input\InputInterface;
|
||||
use Symfony\Component\Console\Output\OutputInterface;
|
||||
use Symfony\Component\Console\Style\SymfonyStyle;
|
||||
use Symfony\Component\EventDispatcher\EventDispatcherInterface;
|
||||
|
||||
final class SegmentStatCommand extends ModeratedCommand
|
||||
{
|
||||
public function __construct(
|
||||
private EventDispatcherInterface $dispatcher,
|
||||
PathsHelper $pathsHelper,
|
||||
CoreParametersHelper $coreParametersHelper,
|
||||
) {
|
||||
parent::__construct($pathsHelper, $coreParametersHelper);
|
||||
}
|
||||
|
||||
protected function configure(): void
|
||||
{
|
||||
$this
|
||||
->setName('mautic:segments:stat')
|
||||
->setDescription('Gather Segment Statistics');
|
||||
|
||||
parent::configure();
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritdoc}
|
||||
*/
|
||||
protected function execute(InputInterface $input, OutputInterface $output): int
|
||||
{
|
||||
$io = new SymfonyStyle($input, $output);
|
||||
$event = new GetStatDataEvent();
|
||||
$this->dispatcher->dispatch($event);
|
||||
|
||||
if (empty($event->getResults())) {
|
||||
$io->write('There is no segment to show!!');
|
||||
} else {
|
||||
$io->table([
|
||||
'Title',
|
||||
'Id',
|
||||
'IsPublished',
|
||||
'IsUsed',
|
||||
],
|
||||
$event->getResults()
|
||||
);
|
||||
}
|
||||
|
||||
return Command::SUCCESS;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,266 @@
|
||||
<?php
|
||||
|
||||
namespace Mautic\LeadBundle\Command;
|
||||
|
||||
use Mautic\CoreBundle\Command\ModeratedCommand;
|
||||
use Mautic\CoreBundle\Helper\CoreParametersHelper;
|
||||
use Mautic\CoreBundle\Helper\PathsHelper;
|
||||
use Mautic\LeadBundle\Entity\LeadList;
|
||||
use Mautic\LeadBundle\Model\ListModel;
|
||||
use Symfony\Component\Console\Attribute\AsCommand;
|
||||
use Symfony\Component\Console\Input\InputInterface;
|
||||
use Symfony\Component\Console\Input\InputOption;
|
||||
use Symfony\Component\Console\Output\NullOutput;
|
||||
use Symfony\Component\Console\Output\OutputInterface;
|
||||
use Symfony\Contracts\Translation\TranslatorInterface;
|
||||
|
||||
#[AsCommand(
|
||||
name: UpdateLeadListsCommand::NAME,
|
||||
description: 'Update contacts in smart segments based on new contact data.',
|
||||
aliases: ['mautic:segments:rebuild']
|
||||
)]
|
||||
class UpdateLeadListsCommand extends ModeratedCommand
|
||||
{
|
||||
public const NAME = 'mautic:segments:update';
|
||||
|
||||
public function __construct(
|
||||
private ListModel $listModel,
|
||||
private TranslatorInterface $translator,
|
||||
PathsHelper $pathsHelper,
|
||||
CoreParametersHelper $coreParametersHelper,
|
||||
) {
|
||||
parent::__construct($pathsHelper, $coreParametersHelper);
|
||||
}
|
||||
|
||||
protected function configure()
|
||||
{
|
||||
$this
|
||||
->addOption(
|
||||
'--batch-limit',
|
||||
'-b',
|
||||
InputOption::VALUE_OPTIONAL,
|
||||
'Set batch size of contacts to process per round. Defaults to 300.',
|
||||
300
|
||||
)
|
||||
->addOption(
|
||||
'--max-contacts',
|
||||
'-m',
|
||||
InputOption::VALUE_OPTIONAL,
|
||||
'Set max number of contacts to process per segment for this script execution. Defaults to all.',
|
||||
false
|
||||
)
|
||||
->addOption(
|
||||
'--list-id',
|
||||
'-i',
|
||||
InputOption::VALUE_OPTIONAL,
|
||||
'Specific ID to rebuild. Defaults to all.',
|
||||
false
|
||||
)
|
||||
->addOption(
|
||||
'--timing',
|
||||
'-tm',
|
||||
InputOption::VALUE_OPTIONAL,
|
||||
'Measure timing of build with output to CLI .',
|
||||
false
|
||||
)
|
||||
->addOption(
|
||||
'exclude',
|
||||
'd',
|
||||
InputOption::VALUE_IS_ARRAY | InputOption::VALUE_OPTIONAL,
|
||||
'Exclude a specific segment from being rebuilt. Otherwise, all segments will be rebuilt.',
|
||||
[]
|
||||
);
|
||||
|
||||
parent::configure();
|
||||
}
|
||||
|
||||
protected function execute(InputInterface $input, OutputInterface $output): int
|
||||
{
|
||||
$id = $input->getOption('list-id');
|
||||
$batch = $input->getOption('batch-limit');
|
||||
$max = $input->getOption('max-contacts') ? (int) $input->getOption('max-contacts') : null;
|
||||
$enableTimeMeasurement = (bool) $input->getOption('timing');
|
||||
$output = ($input->getOption('quiet')) ? new NullOutput() : $output;
|
||||
$excludeSegments = $input->getOption('exclude');
|
||||
|
||||
if (!$this->checkRunStatus($input, $output, $id)) {
|
||||
return \Symfony\Component\Console\Command\Command::SUCCESS;
|
||||
}
|
||||
|
||||
if ($enableTimeMeasurement) {
|
||||
$startTime = microtime(true);
|
||||
}
|
||||
|
||||
if ($id) {
|
||||
$list = $this->listModel->getEntity($id);
|
||||
|
||||
if (!$list) {
|
||||
$output->writeln('<error>'.$this->translator->trans('mautic.lead.list.rebuild.not_found', ['%id%' => $id]).'</error>');
|
||||
|
||||
return \Symfony\Component\Console\Command\Command::FAILURE;
|
||||
}
|
||||
|
||||
// Track already rebuilt lists to avoid rebuilding them multiple times
|
||||
$rebuiltLists = [];
|
||||
|
||||
// First check if this segment has dependencies and rebuild them
|
||||
if ($list->hasFilterTypeOf('leadlist')) {
|
||||
$this->rebuildDependentSegments($list, $rebuiltLists, $batch, $max, $output, $enableTimeMeasurement, [], $excludeSegments);
|
||||
}
|
||||
|
||||
// Add the current list ID to the rebuilt lists to avoid rebuilding it again
|
||||
$rebuiltLists[] = (int) $list->getId();
|
||||
|
||||
$this->rebuildSegment($list, $batch, $max, $output, $enableTimeMeasurement);
|
||||
} else {
|
||||
$filter = [
|
||||
'iterable_mode' => true,
|
||||
];
|
||||
|
||||
if (is_array($excludeSegments) && count($excludeSegments) > 0) {
|
||||
$filter['filter'] = [
|
||||
'force' => [
|
||||
[
|
||||
'expr' => 'notIn',
|
||||
'column' => $this->listModel->getRepository()->getTableAlias().'.id',
|
||||
'value' => $excludeSegments,
|
||||
],
|
||||
],
|
||||
];
|
||||
}
|
||||
|
||||
$rebuiltLists = [];
|
||||
$leadLists = $this->listModel->getEntities($filter);
|
||||
|
||||
/** @var LeadList $leadList */
|
||||
foreach ($leadLists as $leadList) {
|
||||
$listId = $leadList->getId();
|
||||
|
||||
// Skip if already rebuilt
|
||||
if (in_array($listId, $rebuiltLists)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Process any dependent segments first (segments that are used as filters in this segment)
|
||||
if ($leadList->hasFilterTypeOf('leadlist')) {
|
||||
$this->rebuildDependentSegments($leadList, $rebuiltLists, $batch, $max, $output, $enableTimeMeasurement, [], $excludeSegments);
|
||||
}
|
||||
|
||||
// Add the current list ID to the rebuilt lists to avoid rebuilding it again
|
||||
$rebuiltLists[] = $listId;
|
||||
|
||||
// Rebuild the current segment
|
||||
$this->rebuildSegment($leadList, $batch, $max, $output, $enableTimeMeasurement);
|
||||
}
|
||||
}
|
||||
|
||||
$this->completeRun();
|
||||
|
||||
if ($enableTimeMeasurement) {
|
||||
$totalTime = round(microtime(true) - $startTime, 2);
|
||||
$output->writeln('<fg=magenta>'.$this->translator->trans('mautic.lead.list.rebuild.total.time', ['%time%' => $totalTime]).'</>'."\n");
|
||||
}
|
||||
|
||||
return \Symfony\Component\Console\Command\Command::SUCCESS;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<int> $rebuiltLists List of segment IDs that have already been rebuilt
|
||||
* @param array<int> $dependencyChain Chain of segment IDs to detect circular dependencies
|
||||
* @param array<int|string> $excludeSegments List of segment IDs to exclude from rebuilding
|
||||
*
|
||||
* @param-out array<int> $rebuiltLists Updated list of segment IDs that have been rebuilt
|
||||
*/
|
||||
private function rebuildDependentSegments(
|
||||
LeadList $leadList,
|
||||
array &$rebuiltLists,
|
||||
int $batch,
|
||||
?int $max,
|
||||
OutputInterface $output,
|
||||
bool $enableTimeMeasurement,
|
||||
array $dependencyChain = [],
|
||||
array $excludeSegments = [],
|
||||
): void {
|
||||
// Track the current segment in our dependency chain
|
||||
$currentId = $leadList->getId();
|
||||
$dependencyChain[] = $currentId;
|
||||
|
||||
foreach ($leadList->getFilters() as $filter) {
|
||||
if ('leadlist' === $filter['type']) {
|
||||
foreach ($filter['filter'] ?? [] as $dependentListId) {
|
||||
$dependentListId = (int) $dependentListId;
|
||||
|
||||
// Skip if already rebuilt or in exclude list
|
||||
if (in_array($dependentListId, $rebuiltLists) || in_array($dependentListId, $excludeSegments)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check for circular dependency
|
||||
if (in_array($dependentListId, $dependencyChain)) {
|
||||
$output->writeln(
|
||||
'<error>'.$this->translator->trans(
|
||||
'Circular dependency detected in segment chain: %chain%',
|
||||
['%chain%' => implode(' → ', array_merge($dependencyChain, [$dependentListId]))]
|
||||
).'</error>'
|
||||
);
|
||||
continue; // Skip this dependency to prevent infinite recursion
|
||||
}
|
||||
|
||||
$dependentLeadList = $this->listModel->getEntity($dependentListId);
|
||||
if (!$dependentLeadList) {
|
||||
continue; // Skip if the dependent segment doesn't exist - it may have been deleted
|
||||
}
|
||||
|
||||
// Check if this dependent segment has its own dependencies
|
||||
if ($dependentLeadList->hasFilterTypeOf('leadlist')) {
|
||||
// Recursively process this segment's dependencies first, passing the current chain
|
||||
$this->rebuildDependentSegments(
|
||||
$dependentLeadList,
|
||||
$rebuiltLists,
|
||||
$batch,
|
||||
$max,
|
||||
$output,
|
||||
$enableTimeMeasurement,
|
||||
$dependencyChain,
|
||||
$excludeSegments
|
||||
);
|
||||
}
|
||||
|
||||
// Now rebuild this dependent segment
|
||||
$this->rebuildSegment($dependentLeadList, $batch, $max, $output, $enableTimeMeasurement);
|
||||
$rebuiltLists[] = $dependentListId;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private function rebuildSegment(LeadList $segment, int $batch, ?int $max, OutputInterface $output, bool $enableTimeMeasurement = false): void
|
||||
{
|
||||
if (!$segment->isPublished()) {
|
||||
return;
|
||||
}
|
||||
|
||||
$output->writeln('<info>'.$this->translator->trans('mautic.lead.list.rebuild.rebuilding', ['%id%' => $segment->getId()]).'</info>');
|
||||
$startTime = microtime(true);
|
||||
$processed = $this->listModel->rebuildListLeads($segment, $batch, $max, $output);
|
||||
$rebuildTime = round(microtime(true) - $startTime, 2);
|
||||
|
||||
if (0 >= (int) $max) {
|
||||
// Only full segment rebuilds count
|
||||
$segment->setLastBuiltDateToCurrentDatetime();
|
||||
$segment->setLastBuiltTime($rebuildTime);
|
||||
$this->listModel->saveEntity($segment);
|
||||
}
|
||||
|
||||
$output->writeln(
|
||||
'<comment>'.$this->translator->trans('mautic.lead.list.rebuild.leads_affected', ['%leads%' => $processed]).'</comment>'
|
||||
);
|
||||
|
||||
if ($enableTimeMeasurement) {
|
||||
$output->writeln('<fg=cyan>'.$this->translator->trans(
|
||||
'mautic.lead.list.rebuild.contacts.time',
|
||||
['%time%' => $rebuildTime]
|
||||
).'</>'."\n");
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user