public virtual object Clone()
{
var newObj = CloneHelpers.CloneProperties(this);
newObj.ID = Guid.NewGuid();
return(newObj);
}
public async Task <bool> ExecuteAsync()
{
await EnsureTemplatesAreLoaded();
if (_destTemplates.Count > 0)
{
var diff = new HashSet <long>(_sourceTemplates.Keys);
diff.SymmetricExceptWith(_destTemplates.Keys);
if (diff.Count > 0)
{
var missedInSource = diff.Where(d => !_sourceTemplates.ContainsKey(d)).ToList();
if (missedInSource.Count > 0)
{
_logger.LogWarning("Next {count} templates are not present in source: {list}", missedInSource.Count, missedInSource);
}
var missedInDest = diff.Where(d => !_destTemplates.ContainsKey(d)).ToList();
if (missedInDest.Count > 0)
{
_logger.LogInformation("Next {count} templates are not present in destination: {list}", missedInDest.Count, missedInDest);
}
}
else
{
_logger.LogInformation("All {count} templates are present both in source and destination", _sourceTemplates.Count);
}
}
else
{
_logger.LogInformation("There are no templates in destination and total {count} templates in source", _sourceTemplates.Count);
}
var clonedCount = 0L;
var failedIds = new ConcurrentBag <long>();
await CloneHelpers.ParallelRunAsync(_sourceTemplates.Values, _options.MaxDegreeOfParallelism,
async template =>
{
try
{
await CloneTemplateAsync(template);
_logger.LogInformation("Template cloning succeeded: {template}", template);
Interlocked.Increment(ref clonedCount);
}
catch (Exception ex)
{
failedIds.Add(template.Id);
_logger.LogError(new EventId(), ex, "Template cloning error: {template}", template);
}
});
_logger.LogInformation("Cloned templates: {cloned} of {total}", clonedCount, _sourceTemplates.Count);
if (failedIds.Count > 0)
{
_logger.LogWarning("Id's of failed templates: {list}", failedIds);
return(false);
}
return(true);
}
public async Task <bool> ExecuteAsync()
{
var sourcePositions = (await SourceRestClient.GetContentPositionsAsync()).ToDictionary(p => p.Id);
var destPositions = (await DestRestClient.GetContentPositionsAsync()).ToDictionary(p => p.Id);
if (destPositions.Count == 0 && sourcePositions.Count > 0)
{
throw new InvalidOperationException($"There are {sourcePositions.Count} positions in source, but no positions in destination");
}
var diff = new HashSet <long>(sourcePositions.Keys);
diff.SymmetricExceptWith(destPositions.Keys);
if (diff.Count > 0)
{
var missedInSource = diff.Where(d => !sourcePositions.ContainsKey(d)).ToList();
if (missedInSource.Count > 0)
{
_logger.LogWarning("Next {count} positions are not present in source: {list}", missedInSource.Count, missedInSource.Select(p => new { Id = p, destPositions[p].Name }));
}
var missedInDest = diff.Where(d => !destPositions.ContainsKey(d)).ToList();
if (missedInDest.Count > 0)
{
_logger.LogWarning("Next {count} positions are not present in destination: {list}", missedInDest.Count, missedInDest);
}
}
else
{
_logger.LogInformation("All {count} content positions are present both in source and destination", sourcePositions.Count);
}
var positionsLinksToClone = sourcePositions.Values
.Where(p => p.Template != null && destPositions.ContainsKey(p.Id))
.ToList();
_logger.LogInformation("There are total {count} positions with links in source that are also present in destination", positionsLinksToClone.Count);
var clonedCount = 0L;
var failedIds = new ConcurrentBag <long>();
await CloneHelpers.ParallelRunAsync(
positionsLinksToClone,
_options.MaxDegreeOfParallelism,
async position =>
{
try
{
EnsurePositionsAreEqual(destPositions[position.Id], position);
await ClonePositionLinkAsync(position, destPositions[position.Id]);
Interlocked.Increment(ref clonedCount);
_logger.LogInformation("Position link cloning succeeded: {position}", position);
}
catch (Exception ex)
{
failedIds.Add(position.Id);
_logger.LogError(default, ex, "Position link cloning error: {position}", position);
private async Task <bool> CloneFailedAdvertisements(IReadOnlyCollection <ApiListAdvertisement> failedAds)
{
ResetCounters();
var clonedCount = 0;
var totallyFailedAds = new ConcurrentBag <long>();
_logger.LogInformation("Start to clone failed advertisements, total {count}", failedAds.Count.ToString());
await CloneHelpers.ParallelRunAsync(failedAds,
_options.MaxDegreeOfParallelism,
async advertisement =>
{
bool hasFailed;
var tries = 0;
do
{
try
{
++tries;
await CloneAdvertisementAsync(advertisement, true);
Interlocked.Increment(ref clonedCount);
hasFailed = false;
}
catch (Exception ex)
{
hasFailed = true;
_logger.LogError(new EventId(), ex, "Advertisement {id} repeated cloning error", advertisement.Id.ToString());
await Task.Delay(200);
}
}while (hasFailed && tries < _options.MaxCloneTries);
if (hasFailed)
{
totallyFailedAds.Add(advertisement.Id);
}
});
_logger.LogInformation("Failed advertisements repeated cloning done, cloned: {cloned} of {total}", clonedCount, failedAds.Count);
_logger.LogInformation("Total uploaded binaries during repeated cloning: {totalBinaries}", _uploadedBinariesCount);
_logger.LogInformation("Total advertisements selected to whitelist during repeated cloning: {selectedToWhitelistCount}", _selectedToWhitelistCount);
_logger.LogInformation("Total moderated advertisements during repeated cloning: {totalModerated} (approved: {approvedCount}; rejected: {rejectedCount}). Total drafted: {draftedCount}; nominally approved: {nominallyCount}",
_approvedCount + _rejectedCount,
_approvedCount,
_rejectedCount,
_draftedCount,
_nominallyApprovedCount);
if (totallyFailedAds.Count < 1)
{
return(true);
}
_logger.LogError("Failed advertisements after repeated cloning: {ids}", totallyFailedAds);
return(false);
}
public async Task <bool> ExecuteAsync()
{
var sourceCategories = (await SourceRestClient.GetRemarkCategoriesAsync()).ToDictionary(p => p.Id);
var destCategories = (await DestRestClient.GetRemarkCategoriesAsync()).ToDictionary(p => p.Id);
var diff = new HashSet <long>(sourceCategories.Keys);
diff.SymmetricExceptWith(destCategories.Keys);
if (diff.Count > 0)
{
var missedInSource = diff.Where(d => !sourceCategories.ContainsKey(d)).ToList();
if (missedInSource.Count > 0)
{
_logger.LogWarning(
"Next {count} remark categories are not present in source: {list}",
missedInSource.Count,
missedInSource.Select(p => new { Id = p, destCategories[p].Name }));
}
var missedInDest = diff.Where(d => !destCategories.ContainsKey(d)).ToList();
if (missedInDest.Count > 0)
{
_logger.LogWarning("Next {count} remark categories are not present in destination: {list}", missedInDest.Count, missedInDest);
}
}
else
{
_logger.LogInformation("All {count} remark categories are present both in source and destination", sourceCategories.Count);
}
var clonedCount = 0L;
var failedIds = new ConcurrentBag <long>();
await CloneHelpers.ParallelRunAsync(
sourceCategories.Values,
_options.MaxDegreeOfParallelism,
async sourceCategory =>
{
try
{
var destCategory = destCategories.ContainsKey(sourceCategory.Id) ? destCategories[sourceCategory.Id] : null;
await CloneRemarkCategoryAsync(sourceCategory, destCategory);
Interlocked.Increment(ref clonedCount);
_logger.LogInformation("Remark category cloning succeeded: {category}", sourceCategory);
}
catch (Exception ex)
{
failedIds.Add(sourceCategory.Id);
_logger.LogError(default, ex, "Remark category cloning error: {category}", sourceCategory);
public async Task <bool> ExecuteAsync()
{
ResetCounters();
await EnsureTemplatesAreLoaded();
var advertisements = new List <ApiListAdvertisement>(_destTemplates.Count * _options.TruncatedCloneSize);
foreach (var templateId in _destTemplates.Keys)
{
if (_options.AdvertisementsTemplateId.HasValue && templateId != _options.AdvertisementsTemplateId)
{
_logger.LogInformation("Skip fetching ads for template {templateId}", templateId);
continue;
}
if (_sourceTemplates.ContainsKey(templateId))
{
var templateAds = await SourceRestClient.GetAdvertisementsByTemplateAsync(templateId, _isTruncatedCloning?_options.TruncatedCloneSize : (int?)null);
_logger.LogInformation("Found {count} ads for template {templateId}", templateAds.Count, templateId);
advertisements.AddRange(_options.AdvertisementsCreatedAtBeginDate.HasValue
? templateAds.Where(a => a.CreatedAt >= _options.AdvertisementsCreatedAtBeginDate.Value)
: templateAds);
}
else
{
_logger.LogWarning("Template {template} does not exist in source", _destTemplates[templateId]);
}
}
var clonedCount = 0L;
var failedAds = new ConcurrentBag <ApiListAdvertisement>();
_logger.LogInformation("Total advertisements to clone: {total}", advertisements.Count);
await CloneHelpers.ParallelRunAsync(advertisements,
_options.MaxDegreeOfParallelism,
async advertisement =>
{
try
{
_logger.LogInformation("Start to clone advertisement {id} with created date {createdAt:o}",
advertisement.Id,
advertisement.CreatedAt);
await CloneAdvertisementAsync(advertisement, _options.FetchAdvertisementBeforeClone);
Interlocked.Increment(ref clonedCount);
}
catch (Exception ex)
{
failedAds.Add(advertisement);
_logger.LogError(new EventId(), ex, "Advertisement {id} cloning error", advertisement.Id);
}
});
_logger.LogInformation("Total cloned advertisements: {cloned} of {total}", clonedCount, advertisements.Count);
_logger.LogInformation("Total uploaded binaries: {totalBinaries}", _uploadedBinariesCount);
_logger.LogInformation("Total advertisements selected to whitelist: {selectedToWhitelistCount}", _selectedToWhitelistCount);
_logger.LogInformation("Total moderated advertisements: {totalModerated} (approved: {approvedCount}; rejected: {rejectedCount}). Total drafted: {draftedCount}; nominally approved: {nominallyCount}",
_approvedCount + _rejectedCount,
_approvedCount,
_rejectedCount,
_draftedCount,
_nominallyApprovedCount);
// All advertisements have been cloned, check the failed ones:
if (failedAds.Count > 0)
{
return(!_isTruncatedCloning && await CloneFailedAdvertisements(failedAds));
}
return(true);
}
public async Task <bool> ExecuteAsync()
{
ResetCounters();
if (!await LoadAndCheckTemplatesWithVersions())
{
_logger.LogWarning(
"Try to synchronize templates in source and destination by setting {param} option to {mode}",
nameof(CloningToolOptions.Mode),
nameof(CloneMode.CloneTemplates));
return(false);
}
if (_options.AdvertisementsProjectId.HasValue)
{
_logger.LogInformation("Fetching all ads for project {projectId}", _options.AdvertisementsProjectId.Value);
}
List <ApiListAdvertisement> advertisements;
if (string.IsNullOrEmpty(_options.AdvertisementIdsFilename))
{
advertisements = new List <ApiListAdvertisement>(_destTemplates.Count * _options.TruncatedCloneSize);
foreach (var templateId in _destTemplates.Keys)
{
if (_options.AdvertisementsTemplateId.HasValue && templateId != _options.AdvertisementsTemplateId)
{
_logger.LogInformation("Skip fetching ads for template {templateId}", templateId);
continue;
}
if (_sourceTemplates.ContainsKey(templateId))
{
var templateAds = await SourceRestClient.GetAdvertisementsByTemplateAsync(
templateId,
_isTruncatedCloning?_options.TruncatedCloneSize : (int?)null,
_options.AdvertisementsProjectId);
_logger.LogInformation("Found {count} ads for template {templateId}", templateAds.Count, templateId);
advertisements.AddRange(_options.AdvertisementsCreatedAtBeginDate.HasValue
? templateAds.Where(a => a.CreatedAt >= _options.AdvertisementsCreatedAtBeginDate.Value)
: templateAds);
}
else
{
_logger.LogWarning("Template {template} does not exist in source", _destTemplates[templateId]);
}
}
}
else
{
var ids = LoadAdvertisementIdsFromFile(_options.AdvertisementIdsFilename);
advertisements = new List <ApiListAdvertisement>(ids.Count);
for (var i = 0; i <= ids.Count / MaxIdsCountToFetch; ++i)
{
var portionIds = ids.Skip(MaxIdsCountToFetch * i).Take(MaxIdsCountToFetch);
var portionAds = await SourceRestClient.GetAdvertisementsByIdsAsync(portionIds);
advertisements.AddRange(portionAds);
_logger.LogInformation("Found {count} advertisements for {num} batch", portionAds.Count, i + 1);
}
}
var clonedCount = 0L;
var failedAds = new ConcurrentBag <ApiListAdvertisement>();
_logger.LogInformation("Total advertisements to clone: {total}", advertisements.Count);
await CloneHelpers.ParallelRunAsync(
advertisements,
_options.MaxDegreeOfParallelism,
async advertisement =>
{
try
{
_logger.LogInformation(
"Start to clone advertisement {id} with created date {createdAt:o}",
advertisement.Id,
advertisement.CreatedAt);
await CloneAdvertisementAsync(advertisement);
Interlocked.Increment(ref clonedCount);
}
catch (Exception ex)
{
failedAds.Add(advertisement);
_logger.LogError(default, ex, "Advertisement {id} cloning error", advertisement.Id);
public object Clone() => CloneHelpers.CloneProperties(this);
