public async Task <bool> ExecuteAsync()
{
ResetCounters();
await EnsureTemplatesAreLoaded();
List <ApiListAdvertisement> advertisements;
if (string.IsNullOrEmpty(_options.AdvertisementIdsFilename))
{
advertisements = new List <ApiListAdvertisement>(_destTemplates.Count * _options.TruncatedCloneSize);
foreach (var templateId in _destTemplates.Keys)
{
if (_options.AdvertisementsTemplateId.HasValue && templateId != _options.AdvertisementsTemplateId)
{
_logger.LogInformation("Skip fetching ads for template {templateId}", templateId);
continue;
}
if (_sourceTemplates.ContainsKey(templateId))
{
var templateAds = await SourceRestClient.GetAdvertisementsByTemplateAsync(templateId, _isTruncatedCloning?_options.TruncatedCloneSize : (int?)null);
_logger.LogInformation("Found {count} ads for template {templateId}", templateAds.Count, templateId);
advertisements.AddRange(_options.AdvertisementsCreatedAtBeginDate.HasValue
? templateAds.Where(a => a.CreatedAt >= _options.AdvertisementsCreatedAtBeginDate.Value)
: templateAds);
}
else
{
_logger.LogWarning("Template {template} does not exist in source", _destTemplates[templateId]);
}
}
}
else
{
var ids = LoadAdvertisementIdsFromFile(_options.AdvertisementIdsFilename);
advertisements = new List <ApiListAdvertisement>(ids.Count);
for (var i = 0; i <= ids.Count / MaxIdsCountToFetch; ++i)
{
var portionIds = ids.Skip(MaxIdsCountToFetch * i).Take(MaxIdsCountToFetch);
var portionAds = await SourceRestClient.GetAdvertisementsByIdsAsync(portionIds);
advertisements.AddRange(portionAds);
_logger.LogInformation("Found {count} advertisements for {num} batch", portionAds.Count, i + 1);
}
}
var clonedCount = 0L;
var failedAds = new ConcurrentBag <ApiListAdvertisement>();
_logger.LogInformation("Total advertisements to clone: {total}", advertisements.Count);
await CloneHelpers.ParallelRunAsync(advertisements,
_options.MaxDegreeOfParallelism,
async advertisement =>
{
try
{
_logger.LogInformation("Start to clone advertisement {id} with created date {createdAt:o}",
advertisement.Id,
advertisement.CreatedAt);
await CloneAdvertisementAsync(advertisement, _options.FetchAdvertisementBeforeClone);
Interlocked.Increment(ref clonedCount);
}
catch (Exception ex)
{
failedAds.Add(advertisement);
_logger.LogError(new EventId(), ex, "Advertisement {id} cloning error", advertisement.Id);
}
});
_logger.LogInformation("Total cloned advertisements: {cloned} of {total}", clonedCount, advertisements.Count);
_logger.LogInformation("Total uploaded binaries: {totalBinaries}", _uploadedBinariesCount);
_logger.LogInformation("Total advertisements selected to whitelist: {selectedToWhitelistCount}", _selectedToWhitelistCount);
_logger.LogInformation("Total moderated advertisements: {totalModerated} (approved: {approvedCount}; rejected: {rejectedCount}). Total drafted: {draftedCount}; nominally approved: {nominallyCount}",
_approvedCount + _rejectedCount,
_approvedCount,
_rejectedCount,
_draftedCount,
_nominallyApprovedCount);
// All advertisements have been cloned, check the failed ones:
if (failedAds.Count > 0)
{
return(!_isTruncatedCloning && await CloneFailedAdvertisements(failedAds));
}
return(true);
}
public async Task <bool> ExecuteAsync()
{
ResetCounters();
if (!await LoadAndCheckTemplatesWithVersions())
{
_logger.LogWarning(
"Try to synchronize templates in source and destination by setting {param} option to {mode}",
nameof(CloningToolOptions.Mode),
nameof(CloneMode.CloneTemplates));
return(false);
}
if (_options.AdvertisementsProjectId.HasValue)
{
_logger.LogInformation("Fetching all ads for project {projectId}", _options.AdvertisementsProjectId.Value);
}
List <ApiListAdvertisement> advertisements;
if (string.IsNullOrEmpty(_options.AdvertisementIdsFilename))
{
advertisements = new List <ApiListAdvertisement>(_destTemplates.Count * _options.TruncatedCloneSize);
foreach (var templateId in _destTemplates.Keys)
{
if (_options.AdvertisementsTemplateId.HasValue && templateId != _options.AdvertisementsTemplateId)
{
_logger.LogInformation("Skip fetching ads for template {templateId}", templateId);
continue;
}
if (_sourceTemplates.ContainsKey(templateId))
{
var templateAds = await SourceRestClient.GetAdvertisementsByTemplateAsync(
templateId,
_isTruncatedCloning?_options.TruncatedCloneSize : (int?)null,
_options.AdvertisementsProjectId);
_logger.LogInformation("Found {count} ads for template {templateId}", templateAds.Count, templateId);
advertisements.AddRange(_options.AdvertisementsCreatedAtBeginDate.HasValue
? templateAds.Where(a => a.CreatedAt >= _options.AdvertisementsCreatedAtBeginDate.Value)
: templateAds);
}
else
{
_logger.LogWarning("Template {template} does not exist in source", _destTemplates[templateId]);
}
}
}
else
{
var ids = LoadAdvertisementIdsFromFile(_options.AdvertisementIdsFilename);
advertisements = new List <ApiListAdvertisement>(ids.Count);
for (var i = 0; i <= ids.Count / MaxIdsCountToFetch; ++i)
{
var portionIds = ids.Skip(MaxIdsCountToFetch * i).Take(MaxIdsCountToFetch);
var portionAds = await SourceRestClient.GetAdvertisementsByIdsAsync(portionIds);
advertisements.AddRange(portionAds);
_logger.LogInformation("Found {count} advertisements for {num} batch", portionAds.Count, i + 1);
}
}
var clonedCount = 0L;
var failedAds = new ConcurrentBag <ApiListAdvertisement>();
_logger.LogInformation("Total advertisements to clone: {total}", advertisements.Count);
await CloneHelpers.ParallelRunAsync(
advertisements,
_options.MaxDegreeOfParallelism,
async advertisement =>
{
try
{
_logger.LogInformation(
"Start to clone advertisement {id} with created date {createdAt:o}",
advertisement.Id,
advertisement.CreatedAt);
await CloneAdvertisementAsync(advertisement);
Interlocked.Increment(ref clonedCount);
}
catch (Exception ex)
{
failedAds.Add(advertisement);
_logger.LogError(default, ex, "Advertisement {id} cloning error", advertisement.Id);
