public async Task <bool> ExecuteAsync() { await EnsureTemplatesAreLoaded(); if (_destTemplates.Count > 0) { var diff = new HashSet <long>(_sourceTemplates.Keys); diff.SymmetricExceptWith(_destTemplates.Keys); if (diff.Count > 0) { var missedInSource = diff.Where(d => !_sourceTemplates.ContainsKey(d)).ToList(); if (missedInSource.Count > 0) { _logger.LogWarning("Next {count} templates are not present in source: {list}", missedInSource.Count, missedInSource); } var missedInDest = diff.Where(d => !_destTemplates.ContainsKey(d)).ToList(); if (missedInDest.Count > 0) { _logger.LogInformation("Next {count} templates are not present in destination: {list}", missedInDest.Count, missedInDest); } } else { _logger.LogInformation("All {count} templates are present both in source and destination", _sourceTemplates.Count); } } else { _logger.LogInformation("There are no templates in destination and total {count} templates in source", _sourceTemplates.Count); } var clonedCount = 0L; var failedIds = new ConcurrentBag <long>(); await CloneHelpers.ParallelRunAsync(_sourceTemplates.Values, _options.MaxDegreeOfParallelism, async template => { try { await CloneTemplateAsync(template); _logger.LogInformation("Template cloning succeeded: {template}", template); Interlocked.Increment(ref clonedCount); } catch (Exception ex) { failedIds.Add(template.Id); _logger.LogError(new EventId(), ex, "Template cloning error: {template}", template); } }); _logger.LogInformation("Cloned templates: {cloned} of {total}", clonedCount, _sourceTemplates.Count); if (failedIds.Count > 0) { _logger.LogWarning("Id's of failed templates: {list}", failedIds); return(false); } return(true); }
public async Task <bool> ExecuteAsync() { var sourcePositions = (await SourceRestClient.GetContentPositionsAsync()).ToDictionary(p => p.Id); var destPositions = (await DestRestClient.GetContentPositionsAsync()).ToDictionary(p => p.Id); if (destPositions.Count == 0 && sourcePositions.Count > 0) { throw new InvalidOperationException($"There are {sourcePositions.Count} positions in source, but no positions in destination"); } var diff = new HashSet <long>(sourcePositions.Keys); diff.SymmetricExceptWith(destPositions.Keys); if (diff.Count > 0) { var missedInSource = diff.Where(d => !sourcePositions.ContainsKey(d)).ToList(); if (missedInSource.Count > 0) { _logger.LogWarning("Next {count} positions are not present in source: {list}", missedInSource.Count, missedInSource.Select(p => new { Id = p, destPositions[p].Name })); } var missedInDest = diff.Where(d => !destPositions.ContainsKey(d)).ToList(); if (missedInDest.Count > 0) { _logger.LogWarning("Next {count} positions are not present in destination: {list}", missedInDest.Count, missedInDest); } } else { _logger.LogInformation("All {count} content positions are present both in source and destination", sourcePositions.Count); } var positionsLinksToClone = sourcePositions.Values .Where(p => p.Template != null && destPositions.ContainsKey(p.Id)) .ToList(); _logger.LogInformation("There are total {count} positions with links in source that are also present in destination", positionsLinksToClone.Count); var clonedCount = 0L; var failedIds = new ConcurrentBag <long>(); await CloneHelpers.ParallelRunAsync( positionsLinksToClone, _options.MaxDegreeOfParallelism, async position => { try { EnsurePositionsAreEqual(destPositions[position.Id], position); await ClonePositionLinkAsync(position, destPositions[position.Id]); Interlocked.Increment(ref clonedCount); _logger.LogInformation("Position link cloning succeeded: {position}", position); } catch (Exception ex) { failedIds.Add(position.Id); _logger.LogError(default, ex, "Position link cloning error: {position}", position);
private async Task <bool> CloneFailedAdvertisements(IReadOnlyCollection <ApiListAdvertisement> failedAds) { ResetCounters(); var clonedCount = 0; var totallyFailedAds = new ConcurrentBag <long>(); _logger.LogInformation("Start to clone failed advertisements, total {count}", failedAds.Count.ToString()); await CloneHelpers.ParallelRunAsync(failedAds, _options.MaxDegreeOfParallelism, async advertisement => { bool hasFailed; var tries = 0; do { try { ++tries; await CloneAdvertisementAsync(advertisement, true); Interlocked.Increment(ref clonedCount); hasFailed = false; } catch (Exception ex) { hasFailed = true; _logger.LogError(new EventId(), ex, "Advertisement {id} repeated cloning error", advertisement.Id.ToString()); await Task.Delay(200); } }while (hasFailed && tries < _options.MaxCloneTries); if (hasFailed) { totallyFailedAds.Add(advertisement.Id); } }); _logger.LogInformation("Failed advertisements repeated cloning done, cloned: {cloned} of {total}", clonedCount, failedAds.Count); _logger.LogInformation("Total uploaded binaries during repeated cloning: {totalBinaries}", _uploadedBinariesCount); _logger.LogInformation("Total advertisements selected to whitelist during repeated cloning: {selectedToWhitelistCount}", _selectedToWhitelistCount); _logger.LogInformation("Total moderated advertisements during repeated cloning: {totalModerated} (approved: {approvedCount}; rejected: {rejectedCount}). Total drafted: {draftedCount}; nominally approved: {nominallyCount}", _approvedCount + _rejectedCount, _approvedCount, _rejectedCount, _draftedCount, _nominallyApprovedCount); if (totallyFailedAds.Count < 1) { return(true); } _logger.LogError("Failed advertisements after repeated cloning: {ids}", totallyFailedAds); return(false); }
public async Task <bool> ExecuteAsync() { var sourceCategories = (await SourceRestClient.GetRemarkCategoriesAsync()).ToDictionary(p => p.Id); var destCategories = (await DestRestClient.GetRemarkCategoriesAsync()).ToDictionary(p => p.Id); var diff = new HashSet <long>(sourceCategories.Keys); diff.SymmetricExceptWith(destCategories.Keys); if (diff.Count > 0) { var missedInSource = diff.Where(d => !sourceCategories.ContainsKey(d)).ToList(); if (missedInSource.Count > 0) { _logger.LogWarning( "Next {count} remark categories are not present in source: {list}", missedInSource.Count, missedInSource.Select(p => new { Id = p, destCategories[p].Name })); } var missedInDest = diff.Where(d => !destCategories.ContainsKey(d)).ToList(); if (missedInDest.Count > 0) { _logger.LogWarning("Next {count} remark categories are not present in destination: {list}", missedInDest.Count, missedInDest); } } else { _logger.LogInformation("All {count} remark categories are present both in source and destination", sourceCategories.Count); } var clonedCount = 0L; var failedIds = new ConcurrentBag <long>(); await CloneHelpers.ParallelRunAsync( sourceCategories.Values, _options.MaxDegreeOfParallelism, async sourceCategory => { try { var destCategory = destCategories.ContainsKey(sourceCategory.Id) ? destCategories[sourceCategory.Id] : null; await CloneRemarkCategoryAsync(sourceCategory, destCategory); Interlocked.Increment(ref clonedCount); _logger.LogInformation("Remark category cloning succeeded: {category}", sourceCategory); } catch (Exception ex) { failedIds.Add(sourceCategory.Id); _logger.LogError(default, ex, "Remark category cloning error: {category}", sourceCategory);
public async Task <bool> ExecuteAsync() { ResetCounters(); await EnsureTemplatesAreLoaded(); var advertisements = new List <ApiListAdvertisement>(_destTemplates.Count * _options.TruncatedCloneSize); foreach (var templateId in _destTemplates.Keys) { if (_options.AdvertisementsTemplateId.HasValue && templateId != _options.AdvertisementsTemplateId) { _logger.LogInformation("Skip fetching ads for template {templateId}", templateId); continue; } if (_sourceTemplates.ContainsKey(templateId)) { var templateAds = await SourceRestClient.GetAdvertisementsByTemplateAsync(templateId, _isTruncatedCloning?_options.TruncatedCloneSize : (int?)null); _logger.LogInformation("Found {count} ads for template {templateId}", templateAds.Count, templateId); advertisements.AddRange(_options.AdvertisementsCreatedAtBeginDate.HasValue ? templateAds.Where(a => a.CreatedAt >= _options.AdvertisementsCreatedAtBeginDate.Value) : templateAds); } else { _logger.LogWarning("Template {template} does not exist in source", _destTemplates[templateId]); } } var clonedCount = 0L; var failedAds = new ConcurrentBag <ApiListAdvertisement>(); _logger.LogInformation("Total advertisements to clone: {total}", advertisements.Count); await CloneHelpers.ParallelRunAsync(advertisements, _options.MaxDegreeOfParallelism, async advertisement => { try { _logger.LogInformation("Start to clone advertisement {id} with created date {createdAt:o}", advertisement.Id, advertisement.CreatedAt); await CloneAdvertisementAsync(advertisement, _options.FetchAdvertisementBeforeClone); Interlocked.Increment(ref clonedCount); } catch (Exception ex) { failedAds.Add(advertisement); _logger.LogError(new EventId(), ex, "Advertisement {id} cloning error", advertisement.Id); } }); _logger.LogInformation("Total cloned advertisements: {cloned} of {total}", clonedCount, advertisements.Count); _logger.LogInformation("Total uploaded binaries: {totalBinaries}", _uploadedBinariesCount); _logger.LogInformation("Total advertisements selected to whitelist: {selectedToWhitelistCount}", _selectedToWhitelistCount); _logger.LogInformation("Total moderated advertisements: {totalModerated} (approved: {approvedCount}; rejected: {rejectedCount}). Total drafted: {draftedCount}; nominally approved: {nominallyCount}", _approvedCount + _rejectedCount, _approvedCount, _rejectedCount, _draftedCount, _nominallyApprovedCount); // All advertisements have been cloned, check the failed ones: if (failedAds.Count > 0) { return(!_isTruncatedCloning && await CloneFailedAdvertisements(failedAds)); } return(true); }
public async Task <bool> ExecuteAsync() { ResetCounters(); if (!await LoadAndCheckTemplatesWithVersions()) { _logger.LogWarning( "Try to synchronize templates in source and destination by setting {param} option to {mode}", nameof(CloningToolOptions.Mode), nameof(CloneMode.CloneTemplates)); return(false); } if (_options.AdvertisementsProjectId.HasValue) { _logger.LogInformation("Fetching all ads for project {projectId}", _options.AdvertisementsProjectId.Value); } List <ApiListAdvertisement> advertisements; if (string.IsNullOrEmpty(_options.AdvertisementIdsFilename)) { advertisements = new List <ApiListAdvertisement>(_destTemplates.Count * _options.TruncatedCloneSize); foreach (var templateId in _destTemplates.Keys) { if (_options.AdvertisementsTemplateId.HasValue && templateId != _options.AdvertisementsTemplateId) { _logger.LogInformation("Skip fetching ads for template {templateId}", templateId); continue; } if (_sourceTemplates.ContainsKey(templateId)) { var templateAds = await SourceRestClient.GetAdvertisementsByTemplateAsync( templateId, _isTruncatedCloning?_options.TruncatedCloneSize : (int?)null, _options.AdvertisementsProjectId); _logger.LogInformation("Found {count} ads for template {templateId}", templateAds.Count, templateId); advertisements.AddRange(_options.AdvertisementsCreatedAtBeginDate.HasValue ? templateAds.Where(a => a.CreatedAt >= _options.AdvertisementsCreatedAtBeginDate.Value) : templateAds); } else { _logger.LogWarning("Template {template} does not exist in source", _destTemplates[templateId]); } } } else { var ids = LoadAdvertisementIdsFromFile(_options.AdvertisementIdsFilename); advertisements = new List <ApiListAdvertisement>(ids.Count); for (var i = 0; i <= ids.Count / MaxIdsCountToFetch; ++i) { var portionIds = ids.Skip(MaxIdsCountToFetch * i).Take(MaxIdsCountToFetch); var portionAds = await SourceRestClient.GetAdvertisementsByIdsAsync(portionIds); advertisements.AddRange(portionAds); _logger.LogInformation("Found {count} advertisements for {num} batch", portionAds.Count, i + 1); } } var clonedCount = 0L; var failedAds = new ConcurrentBag <ApiListAdvertisement>(); _logger.LogInformation("Total advertisements to clone: {total}", advertisements.Count); await CloneHelpers.ParallelRunAsync( advertisements, _options.MaxDegreeOfParallelism, async advertisement => { try { _logger.LogInformation( "Start to clone advertisement {id} with created date {createdAt:o}", advertisement.Id, advertisement.CreatedAt); await CloneAdvertisementAsync(advertisement); Interlocked.Increment(ref clonedCount); } catch (Exception ex) { failedAds.Add(advertisement); _logger.LogError(default, ex, "Advertisement {id} cloning error", advertisement.Id);