private void ItemCreator(PullItemModel args, CancellationToken cancellationToken)
{
var bulkLoader = new BulkLoader();
try
{
var context = bulkLoader.NewBulkLoadContext("master");
bulkLoader.LoadItems(context, GetAllItemsToCreate(context, cancellationToken));
_checksumManager.RegenerateChecksum();
}
catch (OperationCanceledException e)
{
Log.Warn("Content migration operation was cancelled", e, this);
Status.Cancelled = true;
}
catch (Exception e)
{
Log.Error("Catastrophic error when creating items", e, this);
}
}
public void StartInstallingItems(PullItemModel args, BlockingCollection <IItemData> itemsToInstall, int threads, CancellationToken cancellationToken)
{
Status.StartedTime = DateTime.Now;
Status.RootNodes = args.Ids.Select(x => new ContentTreeNode(x));
Status.IsPreview = args.Preview;
Status.Server = args.Server;
Task.Run(() =>
{
try
{
List <Task> running = new List <Task>();
for (int i = 0; i < threads; i++)
{
running.Add(Task.Run(() => { ItemInstaller(args, itemsToInstall, cancellationToken); }, cancellationToken));
}
Task itemBlasterTask = null;
if (args.UseItemBlaster)
{
itemBlasterTask = Task.Run(() => { ItemCreator(args, cancellationToken); }, cancellationToken);
}
foreach (var t in running)
{
t.Wait(cancellationToken);
}
_itemsToCreate.CompleteAdding();
if (itemBlasterTask != null)
{
itemBlasterTask.Wait();
}
}
catch (OperationCanceledException)
{
Status.Cancelled = true;
}
finally
{
Finalize(ItemsInstalled, args);
}
}, cancellationToken);
}
public void StartContentMigration(PullItemModel model)
{
_model = model;
if (model.PullParent)
{
foreach (var id in model.Ids.Select(Guid.Parse).Where(x => _sitecoreAccess.GetItem(x) == null))
{
var item = _remoteContent.GetRemoteItemData(id, model.Server);
var parent = _sitecoreAccess.GetItem(item.ParentId);
while (parent == null)
{
item = _remoteContent.GetRemoteItemData(item.ParentId, model.Server);
_puller.ItemsToInstall.Add(item);
parent = _sitecoreAccess.GetItem(item.ParentId);
}
}
}
if (model.RemoveLocalNotInRemote)
{
_installer.SetupTrackerForUnwantedLocalItems(model.Ids.Select(Guid.Parse));
}
_puller.StartGatheringItems(model.Ids.Select(Guid.Parse), _registration.GetScsRegistration <ContentMigrationRegistration>().RemoteThreads, model.Children, model.Server, _cancellation);
_installer.StartInstallingItems(model, _puller.ItemsToInstall, _registration.GetScsRegistration <ContentMigrationRegistration>().WriterThreads, _cancellation);
}
internal void ProcessItem(PullItemModel args, IItemData localData, IItemData remoteData)
{
AllowedItems.Remove(remoteData.Id);
if (args.Preview)
{
if (localData != null)
{
var results = _comparer.Compare(remoteData, localData);
if (results.AreEqual)
{
_logger.BeginEvent(remoteData, LogStatus.Skipped, GetSrc(_sitecore.GetIconSrc(localData)), false);
}
else if (results.IsMoved)
{
_logger.BeginEvent(remoteData, LogStatus.Moved, GetSrc(_sitecore.GetIconSrc(localData)), false);
}
else if (results.IsRenamed)
{
_logger.BeginEvent(remoteData, LogStatus.Renamed, GetSrc(_sitecore.GetIconSrc(localData)), false);
}
else if (results.IsTemplateChanged)
{
_logger.BeginEvent(remoteData, LogStatus.TemplateChange, GetSrc(_sitecore.GetIconSrc(localData)), false);
}
else if (args.Overwrite)
{
_logger.BeginEvent(remoteData, LogStatus.Changed, GetSrc(_sitecore.GetIconSrc(localData)), false);
}
else
{
_logger.BeginEvent(remoteData, LogStatus.Skipped, GetSrc(_sitecore.GetIconSrc(localData)), false);
}
}
else
{
_logger.BeginEvent(remoteData, LogStatus.Created, "", false);
}
}
else
{
bool skip = false;
if (!args.Overwrite && localData != null)
{
_logger.BeginEvent(remoteData, LogStatus.Skipped, GetSrc(_sitecore.GetIconSrc(localData)), false);
skip = true;
}
if (!skip && localData != null)
{
var results = _comparer.Compare(remoteData, localData);
if (results.AreEqual)
{
_logger.BeginEvent(remoteData, LogStatus.Skipped, GetSrc(_sitecore.GetIconSrc(localData)), false);
skip = true;
}
}
else if (!skip && !args.UseItemBlaster)
{
while (CurrentlyProcessing.Contains(remoteData.ParentId))
{
if (Errors.Contains(remoteData.ParentId))
{
Errors.Add(remoteData.Id);
skip = true;
break;
}
Task.Delay(WaitForParentDelay).Wait();
}
}
if (!skip)
{
try
{
if (localData != null || !args.UseItemBlaster)
{
_logger.BeginEvent(remoteData, LogStatus.Changed, GetSrc(_sitecore.GetIconSrc(localData)), true);
_scDatastore.Save(remoteData);
}
else if (args.UseItemBlaster)
{
string icon = remoteData.SharedFields.FirstOrDefault(x => x.NameHint == "__Icon")?.Value;
if (string.IsNullOrWhiteSpace(icon))
{
icon = _sitecore.GetIcon(remoteData.TemplateId);
}
_logger.BeginEvent(remoteData, LogStatus.Created, $"/scs/platform/scsicon.scsvc?icon={icon}", false);
_logger.AddToLog($"{DateTime.Now:h:mm:ss tt} [Created] Staging creation of item using Data Blaster {remoteData.Name} - {remoteData.Id}");
_itemsToCreate.Add(remoteData);
}
else
{
_scDatastore.Save(remoteData);
}
}
catch (TemplateMissingFieldException tm)
{
_logger.BeginEvent(new ErrorItemData()
{
Name = remoteData.Name, Path = tm.ToString()
}, LogStatus.Warning, "", false);
}
catch (ParentItemNotFoundException)
{
_logger.BeginEvent(remoteData, LogStatus.SkippedParentError, "", false);
Errors.Add(remoteData.Id);
}
catch (Exception e)
{
Errors.Add(remoteData.Id);
_logger.BeginEvent(new ErrorItemData()
{
Name = remoteData?.Name ?? "Unknown item", Path = e.ToString()
}, LogStatus.Error, "", false);
}
if (localData != null)
{
if (_logger.HasLinesSupportEvents(localData.Id.ToString()))
{
_logger.CompleteEvent(localData.Id.ToString());
}
else
{
_logger.BeginEvent(localData, LogStatus.Skipped, _logger.GetSrc(GetSrc(_sitecore.GetIconSrc(localData))), false);
}
}
}
}
}
private void ItemInstaller(PullItemModel args, BlockingCollection <IItemData> itemsToInstall, CancellationToken cancellationToken)
{
Thread.CurrentThread.Priority = ThreadPriority.Lowest;
BulkUpdateContext bu = null;
EventDisabler ed = null;
try
{
if (args.BulkUpdate)
{
bu = new BulkUpdateContext();
}
if (args.EventDisabler)
{
ed = new EventDisabler();
}
using (new SecurityDisabler())
using (new SyncOperationContext())
{
while (!Completed)
{
if (!itemsToInstall.TryTake(out var remoteData, int.MaxValue, cancellationToken))
{
break;
}
if (!args.UseItemBlaster)
{
CurrentlyProcessing.Add(remoteData.Id);
}
IItemData localData = _sitecore.GetItemData(remoteData.Id);
ProcessItem(args, localData, remoteData);
lock (_locker)
{
ItemsInstalled++;
if (!args.UseItemBlaster)
{
CurrentlyProcessing.Remove(remoteData.Id);
}
}
}
}
}
catch (OperationCanceledException e)
{
Log.Warn("Content migration operation was cancelled", e, this);
Status.Cancelled = true;
lock (_locker)
{
if (!Completed)
{
Finalize(ItemsInstalled, args);
}
}
}
catch (Exception e)
{
Log.Error("Catastrophic error when installing items", e, this);
}
finally
{
if (args.BulkUpdate)
{
bu?.Dispose();
}
if (args.EventDisabler)
{
ed?.Dispose();
}
}
}
public ActionResult StartOperation(PullItemModel data)
{
return(Content(_migrationManager.StartContentMigration(data)));
}
private async Task ProcessItem(PullItemModel args, IItemData localData, IItemData remoteData, Item localItem)
{
_allowedItems.Remove(remoteData.Id);
if (args.Preview)
{
if (localData != null)
{
var results = _comparer.Compare(remoteData, localData);
if (results.AreEqual)
{
_logger.BeginEvent(remoteData, "Skipped", _sitecore.GetItemIconSrc(localData), false);
}
else if (results.IsMoved)
{
_logger.BeginEvent(remoteData, "Moved", _sitecore.GetItemIconSrc(localData), false);
}
else if (results.IsRenamed)
{
_logger.BeginEvent(remoteData, "Renamed", _sitecore.GetItemIconSrc(localData), false);
}
else if (results.IsTemplateChanged)
{
_logger.BeginEvent(remoteData, "Template Change", _sitecore.GetItemIconSrc(localData), false);
}
else if (args.Overwrite)
{
_logger.BeginEvent(remoteData, "Changed", _sitecore.GetItemIconSrc(localData), false);
}
else
{
_logger.BeginEvent(remoteData, "Skipped", _sitecore.GetItemIconSrc(localData), false);
}
}
else
{
_logger.BeginEvent(remoteData, "Created", "", false);
}
}
else
{
bool skip = false;
if (!args.Overwrite && localData != null)
{
_logger.BeginEvent(remoteData, "Skipped", _sitecore.GetItemIconSrc(localData), false);
skip = true;
}
if (!skip && localData != null)
{
var results = _comparer.Compare(remoteData, localData);
if (results.AreEqual)
{
_logger.BeginEvent(remoteData, "Skipped", _sitecore.GetItemIconSrc(localData), false);
skip = true;
}
}
else if (!skip)
{
while (_currentlyProcessing.Contains(remoteData.ParentId))
{
if (_errors.Contains(remoteData.ParentId))
{
_errors.Add(remoteData.Id);
skip = true;
break;
}
await Task.Delay(50);
}
}
if (!skip)
{
try
{
if (localData != null)
{
_logger.BeginEvent(remoteData, "Changed", _logger.GetSrc(ThemeManager.GetIconImage(localItem, 32, 32, "", "")), true);
}
_scDatastore.Save(remoteData);
}
catch (TemplateMissingFieldException tm)
{
_logger.BeginEvent(new ErrorItemData()
{
Name = remoteData.Name, Path = tm.ToString()
}, "Warning", "", false);
}
catch (ParentItemNotFoundException)
{
_logger.BeginEvent(remoteData, "Skipped parent error", "", false);
_errors.Add(remoteData.Id);
}
catch (Exception e)
{
_errors.Add(remoteData.Id);
_logger.BeginEvent(new ErrorItemData()
{
Name = remoteData?.Name ?? "Unknown item", Path = e.ToString()
}, "Error", "", false);
}
if (localData != null)
{
if (_logger.LinesSupport[localData.Id.ToString()].Events.Count != 0)
{
_logger.CompleteEvent(localData.Id.ToString());
}
else
{
_logger.BeginEvent(localData, "Skipped", _logger.GetSrc(ThemeManager.GetIconImage(localItem, 32, 32, "", "")), false);
}
}
}
}
}
public void StartInstallingItems(PullItemModel args, BlockingCollection <IItemData> itemsToInstall, int threads, CancellationTokenSource cancellation)
{
Status.StartedTime = DateTime.Now;
Status.RootNodes = args.Ids.Select(x => new ContentTreeNode(x));
Status.IsPreview = args.Preview;
Status.Server = args.Server;
int items = 0;
for (int i = 0; i < threads; i++)
{
Task.Run(async() =>
{
Thread.CurrentThread.Priority = ThreadPriority.Lowest;
BulkUpdateContext bu = null;
EventDisabler ed = null;
try
{
if (args.BulkUpdate)
{
bu = new BulkUpdateContext();
}
if (args.EventDisabler)
{
ed = new EventDisabler();
}
using (new SecurityDisabler())
{
while (!Completed)
{
IItemData remoteData;
if (!itemsToInstall.TryTake(out remoteData, int.MaxValue, cancellation.Token))
{
lock (_locker)
{
if (!Completed && !_currentlyProcessing.Any())
{
Finalize(items, args);
}
}
break;
}
_currentlyProcessing.Add(remoteData.Id);
Item localItem = _sitecore.GetItem(remoteData.Id);
IItemData localData = localItem == null ? null : new Rainbow.Storage.Sc.ItemData(localItem);
await ProcessItem(args, localData, remoteData, localItem);
lock (_locker)
{
items++;
_currentlyProcessing.Remove(remoteData.Id);
if (_currentlyProcessing.Any() || !itemsToInstall.IsAddingCompleted || itemsToInstall.Count != 0)
{
continue;
}
if (!Completed)
{
Finalize(items, args);
}
}
}
}
}
catch (OperationCanceledException e)
{
Log.Warn("Content migration operation was cancelled", e, this);
Status.Cancelled = true;
lock (_locker)
{
if (!Completed)
{
Finalize(items, args);
}
}
}
catch (Exception e)
{
Log.Error("Catastrophic error when installing items", e, this);
}
finally
{
if (args.BulkUpdate)
{
bu?.Dispose();
}
if (args.EventDisabler)
{
ed?.Dispose();
}
}
});
}
}
internal async Task ProcessItem(PullItemModel args, IItemData localData, IItemData remoteData)
{
AllowedItems.Remove(remoteData.Id);
if (args.Preview)
{
if (localData != null)
{
var results = _comparer.Compare(remoteData, localData);
if (results.AreEqual)
{
_logger.BeginEvent(remoteData, LogStatus.Skipped, GetSrc(_sitecore.GetIconSrc(localData)), false);
}
else if (results.IsMoved)
{
_logger.BeginEvent(remoteData, LogStatus.Moved, GetSrc(_sitecore.GetIconSrc(localData)), false);
}
else if (results.IsRenamed)
{
_logger.BeginEvent(remoteData, LogStatus.Renamed, GetSrc(_sitecore.GetIconSrc(localData)), false);
}
else if (results.IsTemplateChanged)
{
_logger.BeginEvent(remoteData, LogStatus.TemplateChange, GetSrc(_sitecore.GetIconSrc(localData)), false);
}
else if (args.Overwrite)
{
_logger.BeginEvent(remoteData, LogStatus.Changed, GetSrc(_sitecore.GetIconSrc(localData)), false);
}
else
{
_logger.BeginEvent(remoteData, LogStatus.Skipped, GetSrc(_sitecore.GetIconSrc(localData)), false);
}
}
else
{
_logger.BeginEvent(remoteData, LogStatus.Created, "", false);
}
}
else
{
bool skip = false;
if (!args.Overwrite && localData != null)
{
_logger.BeginEvent(remoteData, LogStatus.Skipped, GetSrc(_sitecore.GetIconSrc(localData)), false);
skip = true;
}
if (!skip && localData != null)
{
var results = _comparer.Compare(remoteData, localData);
if (results.AreEqual)
{
_logger.BeginEvent(remoteData, LogStatus.Skipped, GetSrc(_sitecore.GetIconSrc(localData)), false);
skip = true;
}
}
else if (!skip)
{
while (CurrentlyProcessing.Contains(remoteData.ParentId))
{
if (Errors.Contains(remoteData.ParentId))
{
Errors.Add(remoteData.Id);
skip = true;
break;
}
await Task.Delay(WaitForParentDelay).ConfigureAwait(false);
}
}
if (!skip)
{
try
{
if (localData != null)
{
_logger.BeginEvent(remoteData, LogStatus.Changed, _logger.GetSrc(GetSrc(_sitecore.GetIconSrc(localData))), true);
}
_scDatastore.Save(remoteData);
}
catch (TemplateMissingFieldException tm)
{
_logger.BeginEvent(new ErrorItemData()
{
Name = remoteData.Name, Path = tm.ToString()
}, LogStatus.Warning, "", false);
}
catch (ParentItemNotFoundException)
{
_logger.BeginEvent(remoteData, LogStatus.SkippedParentError, "", false);
Errors.Add(remoteData.Id);
}
catch (Exception e)
{
Errors.Add(remoteData.Id);
_logger.BeginEvent(new ErrorItemData()
{
Name = remoteData?.Name ?? "Unknown item", Path = e.ToString()
}, LogStatus.Error, "", false);
}
if (localData != null)
{
if (!_logger.HasLinesSupportEvents(localData.Id.ToString()))
{
_logger.CompleteEvent(localData.Id.ToString());
}
else
{
_logger.BeginEvent(localData, LogStatus.Skipped, _logger.GetSrc(GetSrc(_sitecore.GetIconSrc(localData))), false);
}
}
}
}
}
private async Task ItemInstaller(PullItemModel args, BlockingCollection <IItemData> itemsToInstall, CancellationToken cancellationToken)
{
Thread.CurrentThread.Priority = ThreadPriority.Lowest;
BulkUpdateContext bu = null;
EventDisabler ed = null;
try
{
if (args.BulkUpdate)
{
bu = new BulkUpdateContext();
}
if (args.EventDisabler)
{
ed = new EventDisabler();
}
using (new SecurityDisabler())
{
while (!Completed)
{
IItemData remoteData;
if (!itemsToInstall.TryTake(out remoteData, int.MaxValue, cancellationToken))
{
lock (_locker)
{
if (!Completed && !CurrentlyProcessing.Any())
{
Finalize(ItemsInstalled, args);
}
}
break;
}
CurrentlyProcessing.Add(remoteData.Id);
IItemData localData = _sitecore.GetItemData(remoteData.Id);
await ProcessItem(args, localData, remoteData).ConfigureAwait(false);
lock (_locker)
{
ItemsInstalled++;
CurrentlyProcessing.Remove(remoteData.Id);
if (CurrentlyProcessing.Any() || !itemsToInstall.IsAddingCompleted || itemsToInstall.Count != 0)
{
continue;
}
if (!Completed)
{
Finalize(ItemsInstalled, args);
}
}
}
}
}
catch (OperationCanceledException e)
{
Log.Warn("Content migration operation was cancelled", e, this);
Status.Cancelled = true;
lock (_locker)
{
if (!Completed)
{
Finalize(ItemsInstalled, args);
}
}
}
catch (Exception e)
{
Log.Error("Catastrophic error when installing items", e, this);
}
finally
{
if (args.BulkUpdate)
{
bu?.Dispose();
}
if (args.EventDisabler)
{
ed?.Dispose();
}
}
}