public async Task <bool> ExecuteAsync() { ResetCounters(); await EnsureTemplatesAreLoaded(); List <ApiListAdvertisement> advertisements; if (string.IsNullOrEmpty(_options.AdvertisementIdsFilename)) { advertisements = new List <ApiListAdvertisement>(_destTemplates.Count * _options.TruncatedCloneSize); foreach (var templateId in _destTemplates.Keys) { if (_options.AdvertisementsTemplateId.HasValue && templateId != _options.AdvertisementsTemplateId) { _logger.LogInformation("Skip fetching ads for template {templateId}", templateId); continue; } if (_sourceTemplates.ContainsKey(templateId)) { var templateAds = await SourceRestClient.GetAdvertisementsByTemplateAsync(templateId, _isTruncatedCloning?_options.TruncatedCloneSize : (int?)null); _logger.LogInformation("Found {count} ads for template {templateId}", templateAds.Count, templateId); advertisements.AddRange(_options.AdvertisementsCreatedAtBeginDate.HasValue ? templateAds.Where(a => a.CreatedAt >= _options.AdvertisementsCreatedAtBeginDate.Value) : templateAds); } else { _logger.LogWarning("Template {template} does not exist in source", _destTemplates[templateId]); } } } else { var ids = LoadAdvertisementIdsFromFile(_options.AdvertisementIdsFilename); advertisements = new List <ApiListAdvertisement>(ids.Count); for (var i = 0; i <= ids.Count / MaxIdsCountToFetch; ++i) { var portionIds = ids.Skip(MaxIdsCountToFetch * i).Take(MaxIdsCountToFetch); var portionAds = await SourceRestClient.GetAdvertisementsByIdsAsync(portionIds); advertisements.AddRange(portionAds); _logger.LogInformation("Found {count} advertisements for {num} batch", portionAds.Count, i + 1); } } var clonedCount = 0L; var failedAds = new ConcurrentBag <ApiListAdvertisement>(); _logger.LogInformation("Total advertisements to clone: {total}", advertisements.Count); await CloneHelpers.ParallelRunAsync(advertisements, _options.MaxDegreeOfParallelism, async advertisement => { try { _logger.LogInformation("Start to clone advertisement {id} with created date {createdAt:o}", advertisement.Id, advertisement.CreatedAt); await CloneAdvertisementAsync(advertisement, _options.FetchAdvertisementBeforeClone); Interlocked.Increment(ref clonedCount); } catch (Exception ex) { failedAds.Add(advertisement); _logger.LogError(new EventId(), ex, "Advertisement {id} cloning error", advertisement.Id); } }); _logger.LogInformation("Total cloned advertisements: {cloned} of {total}", clonedCount, advertisements.Count); _logger.LogInformation("Total uploaded binaries: {totalBinaries}", _uploadedBinariesCount); _logger.LogInformation("Total advertisements selected to whitelist: {selectedToWhitelistCount}", _selectedToWhitelistCount); _logger.LogInformation("Total moderated advertisements: {totalModerated} (approved: {approvedCount}; rejected: {rejectedCount}). Total drafted: {draftedCount}; nominally approved: {nominallyCount}", _approvedCount + _rejectedCount, _approvedCount, _rejectedCount, _draftedCount, _nominallyApprovedCount); // All advertisements have been cloned, check the failed ones: if (failedAds.Count > 0) { return(!_isTruncatedCloning && await CloneFailedAdvertisements(failedAds)); } return(true); }
public async Task <bool> ExecuteAsync() { ResetCounters(); if (!await LoadAndCheckTemplatesWithVersions()) { _logger.LogWarning( "Try to synchronize templates in source and destination by setting {param} option to {mode}", nameof(CloningToolOptions.Mode), nameof(CloneMode.CloneTemplates)); return(false); } if (_options.AdvertisementsProjectId.HasValue) { _logger.LogInformation("Fetching all ads for project {projectId}", _options.AdvertisementsProjectId.Value); } List <ApiListAdvertisement> advertisements; if (string.IsNullOrEmpty(_options.AdvertisementIdsFilename)) { advertisements = new List <ApiListAdvertisement>(_destTemplates.Count * _options.TruncatedCloneSize); foreach (var templateId in _destTemplates.Keys) { if (_options.AdvertisementsTemplateId.HasValue && templateId != _options.AdvertisementsTemplateId) { _logger.LogInformation("Skip fetching ads for template {templateId}", templateId); continue; } if (_sourceTemplates.ContainsKey(templateId)) { var templateAds = await SourceRestClient.GetAdvertisementsByTemplateAsync( templateId, _isTruncatedCloning?_options.TruncatedCloneSize : (int?)null, _options.AdvertisementsProjectId); _logger.LogInformation("Found {count} ads for template {templateId}", templateAds.Count, templateId); advertisements.AddRange(_options.AdvertisementsCreatedAtBeginDate.HasValue ? templateAds.Where(a => a.CreatedAt >= _options.AdvertisementsCreatedAtBeginDate.Value) : templateAds); } else { _logger.LogWarning("Template {template} does not exist in source", _destTemplates[templateId]); } } } else { var ids = LoadAdvertisementIdsFromFile(_options.AdvertisementIdsFilename); advertisements = new List <ApiListAdvertisement>(ids.Count); for (var i = 0; i <= ids.Count / MaxIdsCountToFetch; ++i) { var portionIds = ids.Skip(MaxIdsCountToFetch * i).Take(MaxIdsCountToFetch); var portionAds = await SourceRestClient.GetAdvertisementsByIdsAsync(portionIds); advertisements.AddRange(portionAds); _logger.LogInformation("Found {count} advertisements for {num} batch", portionAds.Count, i + 1); } } var clonedCount = 0L; var failedAds = new ConcurrentBag <ApiListAdvertisement>(); _logger.LogInformation("Total advertisements to clone: {total}", advertisements.Count); await CloneHelpers.ParallelRunAsync( advertisements, _options.MaxDegreeOfParallelism, async advertisement => { try { _logger.LogInformation( "Start to clone advertisement {id} with created date {createdAt:o}", advertisement.Id, advertisement.CreatedAt); await CloneAdvertisementAsync(advertisement); Interlocked.Increment(ref clonedCount); } catch (Exception ex) { failedAds.Add(advertisement); _logger.LogError(default, ex, "Advertisement {id} cloning error", advertisement.Id);