private async Task SendBinaryContent(ApiObjectDescriptor sourceDescriptor, ApiObjectDescriptor newAdvertisement)
{
foreach (var element in sourceDescriptor.Elements)
{
if (IsBinaryAdvertisementElementType(element.Type))
{
var value = element.Value as IBinaryElementValue
?? throw new InvalidOperationException($"Cannot cast advertisement {sourceDescriptor.Id} binary element {element.TemplateCode} value");
if (string.IsNullOrEmpty(value.Raw))
{
continue;
}
var fileData = await SourceRestClient.DownloadFileAsync(sourceDescriptor.Id, value.DownloadUri);
var matchedElement = newAdvertisement.Elements.First(e => e.TemplateCode == element.TemplateCode);
EnsureUploadUrlIsValid(sourceDescriptor.Id, matchedElement);
var res = await DestRestClient.UploadFileAsync(sourceDescriptor.Id,
new Uri(matchedElement.UploadUrl),
value.Filename,
fileData);
Interlocked.Increment(ref _uploadedBinariesCount);
element.Value = res;
}
}
}
private async Task CloneTemplateAsync(ApiListTemplate template)
{
var templateIdStr = template.Id.ToString();
var sourceTemplate = await SourceRestClient.GetTemplateAsync(templateIdStr);
var destTemplate = await DestRestClient.GetTemplateAsync(templateIdStr);
if (destTemplate == null)
{
await DestRestClient.CreateTemplateAsync(templateIdStr, sourceTemplate);
}
else
{
_logger.LogInformation("Template {id} already exists in destination", template.Id);
if (!CompareTemplateDescriptors(destTemplate, sourceTemplate))
{
if (_options.OverwriteUnequalTemplates)
{
_logger.LogWarning("Updating template {id} with version {versionId} in destination", template.Id, destTemplate.VersionId);
await DestRestClient.UpdateTemplateAsync(sourceTemplate, destTemplate.VersionId);
}
else
{
throw new InvalidOperationException("Templates with id = " + templateIdStr + " are not equal");
}
}
else
{
_logger.LogInformation("Templates with id {id} are equal in source and destination", template.Id);
}
}
}
public async Task <bool> ExecuteAsync()
{
var sourcePositions = (await SourceRestClient.GetContentPositionsAsync()).ToDictionary(p => p.Id);
var destPositions = (await DestRestClient.GetContentPositionsAsync()).ToDictionary(p => p.Id);
if (destPositions.Count == 0 && sourcePositions.Count > 0)
{
throw new InvalidOperationException($"There are {sourcePositions.Count} positions in source, but no positions in destination");
}
var diff = new HashSet <long>(sourcePositions.Keys);
diff.SymmetricExceptWith(destPositions.Keys);
if (diff.Count > 0)
{
var missedInSource = diff.Where(d => !sourcePositions.ContainsKey(d)).ToList();
if (missedInSource.Count > 0)
{
_logger.LogWarning("Next {count} positions are not present in source: {list}", missedInSource.Count, missedInSource.Select(p => new { Id = p, destPositions[p].Name }));
}
var missedInDest = diff.Where(d => !destPositions.ContainsKey(d)).ToList();
if (missedInDest.Count > 0)
{
_logger.LogWarning("Next {count} positions are not present in destination: {list}", missedInDest.Count, missedInDest);
}
}
else
{
_logger.LogInformation("All {count} content positions are present both in source and destination", sourcePositions.Count);
}
var positionsLinksToClone = sourcePositions.Values
.Where(p => p.Template != null && destPositions.ContainsKey(p.Id))
.ToList();
_logger.LogInformation("There are total {count} positions with links in source that are also present in destination", positionsLinksToClone.Count);
var clonedCount = 0L;
var failedIds = new ConcurrentBag <long>();
await CloneHelpers.ParallelRunAsync(
positionsLinksToClone,
_options.MaxDegreeOfParallelism,
async position =>
{
try
{
EnsurePositionsAreEqual(destPositions[position.Id], position);
await ClonePositionLinkAsync(position, destPositions[position.Id]);
Interlocked.Increment(ref clonedCount);
_logger.LogInformation("Position link cloning succeeded: {position}", position);
}
catch (Exception ex)
{
failedIds.Add(position.Id);
_logger.LogError(default, ex, "Position link cloning error: {position}", position);
public async Task <bool> ExecuteAsync()
{
ResetCounters();
await EnsureTemplatesAreLoaded();
var advertisements = new List <ApiListAdvertisement>(_destTemplates.Count * _options.TruncatedCloneSize);
foreach (var templateId in _destTemplates.Keys)
{
if (_options.AdvertisementsTemplateId.HasValue && templateId != _options.AdvertisementsTemplateId)
{
_logger.LogInformation("Skip fetching ads for template {templateId}", templateId);
continue;
}
if (_sourceTemplates.ContainsKey(templateId))
{
var templateAds = await SourceRestClient.GetAdvertisementsByTemplateAsync(templateId, _isTruncatedCloning?_options.TruncatedCloneSize : (int?)null);
_logger.LogInformation("Found {count} ads for template {templateId}", templateAds.Count, templateId);
advertisements.AddRange(_options.AdvertisementsCreatedAtBeginDate.HasValue
? templateAds.Where(a => a.CreatedAt >= _options.AdvertisementsCreatedAtBeginDate.Value)
: templateAds);
}
else
{
_logger.LogWarning("Template {template} does not exist in source", _destTemplates[templateId]);
}
}
var clonedCount = 0L;
var failedAds = new ConcurrentBag <ApiListAdvertisement>();
_logger.LogInformation("Total advertisements to clone: {total}", advertisements.Count);
await CloneHelpers.ParallelRunAsync(advertisements,
_options.MaxDegreeOfParallelism,
async advertisement =>
{
try
{
_logger.LogInformation("Start to clone advertisement {id} with created date {createdAt:o}",
advertisement.Id,
advertisement.CreatedAt);
await CloneAdvertisementAsync(advertisement, _options.FetchAdvertisementBeforeClone);
Interlocked.Increment(ref clonedCount);
}
catch (Exception ex)
{
failedAds.Add(advertisement);
_logger.LogError(new EventId(), ex, "Advertisement {id} cloning error", advertisement.Id);
}
});
_logger.LogInformation("Total cloned advertisements: {cloned} of {total}", clonedCount, advertisements.Count);
_logger.LogInformation("Total uploaded binaries: {totalBinaries}", _uploadedBinariesCount);
_logger.LogInformation("Total advertisements selected to whitelist: {selectedToWhitelistCount}", _selectedToWhitelistCount);
_logger.LogInformation("Total moderated advertisements: {totalModerated} (approved: {approvedCount}; rejected: {rejectedCount}). Total drafted: {draftedCount}; nominally approved: {nominallyCount}",
_approvedCount + _rejectedCount,
_approvedCount,
_rejectedCount,
_draftedCount,
_nominallyApprovedCount);
// All advertisements have been cloned, check the failed ones:
if (failedAds.Count > 0)
{
return(!_isTruncatedCloning && await CloneFailedAdvertisements(failedAds));
}
return(true);
}
private async Task CloneAdvertisementAsync(ApiListAdvertisement advertisement, bool fetchAdvertisementBeforeCloning)
{
var versionId = string.Empty;
var objectId = advertisement.Id.ToString();
if (fetchAdvertisementBeforeCloning)
{
versionId = await DestRestClient.GetAdvertisementVersionAsync(advertisement.Id);
_logger.LogInformation("Object {id} has been fetched from destination preliminarily with version {versionId}", objectId, versionId);
}
if (string.IsNullOrEmpty(versionId))
{
var sourceDescriptor = await SourceRestClient.GetAdvertisementAsync(advertisement.Id);
sourceDescriptor.TemplateVersionId = _destTemplates[sourceDescriptor.TemplateId].VersionId;
if (sourceDescriptor.Elements.Any(e => IsBinaryAdvertisementElementType(e.Type)))
{
var newAm = await DestRestClient.CreateAdvertisementPrototypeAsync(advertisement.Template.Id, advertisement.Language.ToString(), advertisement.Firm.Id);
await SendBinaryContent(sourceDescriptor, newAm);
}
try
{
versionId = await DestRestClient.CreateAdvertisementAsync(advertisement.Id, advertisement.Firm.Id, sourceDescriptor);
}
catch (ObjectAlreadyExistsException ex)
{
_logger.LogWarning(new EventId(), ex, "Object {id} already exists in destination, try to continue execution", objectId);
}
}
if (advertisement.IsWhiteListed)
{
await DestRestClient.SelectAdvertisementToWhitelistAsync(objectId);
Interlocked.Increment(ref _selectedToWhitelistCount);
}
if (advertisement.Moderation != null && advertisement.Moderation.Status != ModerationStatus.OnApproval &&
advertisement.Moderation.Status != ModerationStatus.NominallyApproved)
{
if (string.IsNullOrEmpty(versionId))
{
_logger.LogWarning("VersionId for object {id} is unknown, need to get latest version", objectId);
versionId = await SourceRestClient.GetAdvertisementVersionAsync(advertisement.Id);
}
await DestRestClient.UpdateAdvertisementModerationStatusAsync(objectId, versionId, advertisement.Moderation);
}
switch (advertisement.Moderation?.Status)
{
case null:
break;
case ModerationStatus.Approved:
Interlocked.Increment(ref _approvedCount);
break;
case ModerationStatus.Rejected:
Interlocked.Increment(ref _rejectedCount);
break;
case ModerationStatus.OnApproval:
Interlocked.Increment(ref _draftedCount);
break;
case ModerationStatus.NominallyApproved:
Interlocked.Increment(ref _nominallyApprovedCount);
break;
default:
throw new ArgumentOutOfRangeException(nameof(advertisement.Moderation), advertisement.Moderation.Status, "Unsupported moderation status");
}
}
private async Task EnsureTemplatesAreLoaded()
{
_sourceTemplates = _sourceTemplates ?? (await SourceRestClient.GetTemplatesAsync()).ToDictionary(p => p.Id);
_destTemplates = _destTemplates ?? (await DestRestClient.GetTemplatesAsync()).ToDictionary(p => p.Id);
}
private async Task SendBinaryContent(ApiObjectDescriptor sourceDescriptor, ApiObjectDescriptor newAdvertisement)
{
var binaryElements = sourceDescriptor.Elements.Where(e => IsBinaryAdvertisementElementType(e.Type));
foreach (var element in binaryElements)
{
var value = element.Value as IBinaryElementValue
?? throw new InvalidOperationException($"Cannot cast advertisement {sourceDescriptor.Id} binary element {element.TemplateCode} value");
if (string.IsNullOrEmpty(value.Raw))
{
continue;
}
var fileData = await SourceRestClient.DownloadFileAsync(sourceDescriptor.Id, value.DownloadUri);
var matchedElement = newAdvertisement.Elements.First(e => e.TemplateCode == element.TemplateCode);
EnsureUploadUrlIsValid(sourceDescriptor.Id, matchedElement);
var uploadResponse = await DestRestClient.UploadFileAsync(
sourceDescriptor.Id,
new Uri(matchedElement.UploadUrl),
value.Filename,
fileData);
Interlocked.Increment(ref _uploadedBinariesCount);
if (element.Type == ElementDescriptorType.CompositeBitmapImage)
{
var compositeBitmapImageElementValue = element.Value as ICompositeBitmapImageElementValue
?? throw new InvalidOperationException($"Cannot cast advertisement {sourceDescriptor.Id} composite image element {element.TemplateCode} value");
var sizeSpecificImagesUploadTasks = compositeBitmapImageElementValue
.SizeSpecificImages
.Select(async image =>
{
var imageFileData = await SourceRestClient.DownloadFileAsync(sourceDescriptor.Id, image.DownloadUri);
var headers = new[]
{
new NameValueHeaderValue(HeaderNames.AmsFileType, FileType.SizeSpecificBitmapImage.ToString()),
new NameValueHeaderValue(HeaderNames.AmsImageSize, image.Size.ToString())
};
var imageUploadResponse = await DestRestClient.UploadFileAsync(
sourceDescriptor.Id,
new Uri(matchedElement.UploadUrl),
image.Filename,
imageFileData,
headers);
Interlocked.Increment(ref _uploadedBinariesCount);
return(new SizeSpecificImage
{
Size = image.Size,
Raw = imageUploadResponse.Raw
});
})
.ToList();
element.Value = new CompositeBitmapImageElementValue
{
Raw = uploadResponse.Raw,
CropArea = compositeBitmapImageElementValue.CropArea,
SizeSpecificImages = await Task.WhenAll(sizeSpecificImagesUploadTasks)
};
}
else
{
element.Value = uploadResponse;
}
}
}
public async Task <bool> ExecuteAsync()
{
ResetCounters();
if (!await LoadAndCheckTemplatesWithVersions())
{
_logger.LogWarning(
"Try to synchronize templates in source and destination by setting {param} option to {mode}",
nameof(CloningToolOptions.Mode),
nameof(CloneMode.CloneTemplates));
return(false);
}
if (_options.AdvertisementsProjectId.HasValue)
{
_logger.LogInformation("Fetching all ads for project {projectId}", _options.AdvertisementsProjectId.Value);
}
List <ApiListAdvertisement> advertisements;
if (string.IsNullOrEmpty(_options.AdvertisementIdsFilename))
{
advertisements = new List <ApiListAdvertisement>(_destTemplates.Count * _options.TruncatedCloneSize);
foreach (var templateId in _destTemplates.Keys)
{
if (_options.AdvertisementsTemplateId.HasValue && templateId != _options.AdvertisementsTemplateId)
{
_logger.LogInformation("Skip fetching ads for template {templateId}", templateId);
continue;
}
if (_sourceTemplates.ContainsKey(templateId))
{
var templateAds = await SourceRestClient.GetAdvertisementsByTemplateAsync(
templateId,
_isTruncatedCloning?_options.TruncatedCloneSize : (int?)null,
_options.AdvertisementsProjectId);
_logger.LogInformation("Found {count} ads for template {templateId}", templateAds.Count, templateId);
advertisements.AddRange(_options.AdvertisementsCreatedAtBeginDate.HasValue
? templateAds.Where(a => a.CreatedAt >= _options.AdvertisementsCreatedAtBeginDate.Value)
: templateAds);
}
else
{
_logger.LogWarning("Template {template} does not exist in source", _destTemplates[templateId]);
}
}
}
else
{
var ids = LoadAdvertisementIdsFromFile(_options.AdvertisementIdsFilename);
advertisements = new List <ApiListAdvertisement>(ids.Count);
for (var i = 0; i <= ids.Count / MaxIdsCountToFetch; ++i)
{
var portionIds = ids.Skip(MaxIdsCountToFetch * i).Take(MaxIdsCountToFetch);
var portionAds = await SourceRestClient.GetAdvertisementsByIdsAsync(portionIds);
advertisements.AddRange(portionAds);
_logger.LogInformation("Found {count} advertisements for {num} batch", portionAds.Count, i + 1);
}
}
var clonedCount = 0L;
var failedAds = new ConcurrentBag <ApiListAdvertisement>();
_logger.LogInformation("Total advertisements to clone: {total}", advertisements.Count);
await CloneHelpers.ParallelRunAsync(
advertisements,
_options.MaxDegreeOfParallelism,
async advertisement =>
{
try
{
_logger.LogInformation(
"Start to clone advertisement {id} with created date {createdAt:o}",
advertisement.Id,
advertisement.CreatedAt);
await CloneAdvertisementAsync(advertisement);
Interlocked.Increment(ref clonedCount);
}
catch (Exception ex)
{
failedAds.Add(advertisement);
_logger.LogError(default, ex, "Advertisement {id} cloning error", advertisement.Id);
