protected virtual void LoadItems(IConfiguration[] configurations, DataBlasterParameters parameters, ILogger logger)
{
var databaseNames = configurations
.SelectMany(c => c.Resolve <PredicateRootPathResolver>().GetRootPaths().Select(rp => rp.DatabaseName))
.Distinct();
foreach (var databaseName in databaseNames)
{
logger.Info($"Syncing database '{databaseName}'...");
var context = CreateBulkLoadContext(BulkLoader, databaseName, configurations, parameters, logger);
var bulkItems = ItemExtractor.ExtractBulkItems(context, configurations, databaseName);
BulkLoader.LoadItems(context, bulkItems);
if (context.AnyStageFailed)
{
throw new Exception($"Stage failed during bulkload of database '{databaseName}': {context.FailureMessage}");
}
// Support publishing after sync.
if (!IsUnicornPublishEnabled && !databaseName.Equals("core", StringComparison.OrdinalIgnoreCase))
{
foreach (var itemChange in context.ItemChanges)
{
ManualPublishQueueHandler.AddItemToPublish(itemChange.ItemId);
}
}
}
}
protected virtual BulkLoadAction GetBulkLoadAction(DataBlasterParameters parameters, IConfiguration configuration, PresetTreeRoot treeRoot)
{
if (parameters.ForceBulkLoadAction.HasValue)
{
return(parameters.ForceBulkLoadAction.Value);
}
var evaluator = configuration.Resolve <IEvaluator>();
if (evaluator is SerializedAsMasterEvaluator)
{
// Only revert the tree when there are no exclusions for this tree root.
return(treeRoot.Exclusions == null || treeRoot.Exclusions.Count == 0
? BulkLoadAction.RevertTree
: BulkLoadAction.Revert);
}
if (evaluator is NewItemOnlyEvaluator)
{
return(BulkLoadAction.AddItemOnly);
}
//if (evaluator is AddOnlyEvaluator)
//{
// return BulkLoadAction.AddOnly;
//}
throw new ArgumentException($"Unknown evaluator type: '{evaluator.GetType().Name}'");
}
public virtual void Process(UnicornSyncStartPipelineArgs args)
{
// Find optional data blaster parameters in custom data of arguments.
object parms;
args.CustomData.TryGetValue(PipelineArgsParametersKey, out parms);
var parameters = parms as DataBlasterParameters;
if (parameters == null)
{
// Is DataBlaster disabled through config?
if (Settings.GetBoolSetting(DisableDataBlasterSettingName, false))
{
return;
}
// Use default parameters.
parameters = new DataBlasterParameters();
}
// Is DataBlaster disabled through parameters?
if (parameters.DisableDataBlaster)
{
return;
}
try
{
args.Logger.Info($"Start Bulk Unicorn Sync for configurations: '{string.Join("', '", args.Configurations.Select(x => x.Name))}'.");
var watch = Stopwatch.StartNew();
var startTimestamp = DateTime.Now;
LoadItems(args.Configurations, parameters, args.Logger);
args.Logger.Info($"Extracted and loaded items ({(int)watch.Elapsed.TotalMilliseconds}ms)");
watch.Restart();
ClearCaches();
args.Logger.Info($"Caches cleared ({(int)watch.Elapsed.TotalMilliseconds}ms)");
ExecuteUnicornSyncComplete(args, parameters, startTimestamp);
ExecuteUnicornSyncEnd(args, parameters);
}
catch (Exception ex)
{
args.Logger.Error(ex);
throw;
}
finally
{
// This will signal that we handled the sync for all configurations,
// and no further handling should be done.
args.SyncIsHandled = true;
}
}
protected virtual void ExecuteUnicornSyncEnd(UnicornSyncStartPipelineArgs args,
DataBlasterParameters parameters)
{
if (parameters.SkipUnicornSyncEnd)
{
return;
}
// When we tell Unicorn that sync is handled, end pipeline is not called anymore.
var watch = Stopwatch.StartNew();
CorePipeline.Run("unicornSyncEnd", new UnicornSyncEndPipelineArgs(args.Logger, true, args.Configurations));
args.Logger.Info($"Ran sync end pipeline ({(int)watch.Elapsed.TotalMilliseconds}ms)");
}
protected virtual void ExecuteUnicornSyncComplete(UnicornSyncStartPipelineArgs args,
DataBlasterParameters parameters,
DateTime syncStartTimestamp)
{
if (parameters.SkipUnicornSyncComplete)
{
return;
}
// Run complete pipelines to support post-processing, e.g. users and roles.
var watch = Stopwatch.StartNew();
foreach (var config in args.Configurations)
{
CorePipeline.Run("unicornSyncComplete",
new UnicornSyncCompletePipelineArgs(config, syncStartTimestamp));
}
args.Logger.Info($"Ran sync complete pipelines ({(int)watch.Elapsed.TotalMilliseconds}ms)");
}
protected virtual BulkLoadContext CreateBulkLoadContext(BulkLoader bulkLoader, string databaseName,
IConfiguration[] configurations, DataBlasterParameters parameters, ILogger logger)
{
var context = bulkLoader.NewBulkLoadContext(databaseName);
context.Log = new SitecoreAndUnicornLog(LoggerFactory.GetLogger(GetType()), logger);
context.AllowTemplateChanges = true;
context.StageDataWithoutProcessing = parameters.StageDataWithoutProcessing;
// Use the shotgun, removing items one by one is too slow for full deserialize.
context.RemoveItemsFromCaches = false;
context.UpdateHistory = !SkipHistoryEngine;
context.UpdatePublishQueue = !SkipPublishQueue;
context.UpdateLinkDatabase = !SkipLinkDatabase &&
configurations.Any(x => x.Resolve <ISyncConfiguration>().UpdateLinkDatabase);
context.UpdateIndexes = !SkipIndexes &&
configurations.Any(x => x.Resolve <ISyncConfiguration>().UpdateSearchIndex);
return(context);
}
protected virtual BulkLoadContext CreateBulkLoadContext(BulkLoader bulkLoader, string databaseName,
IConfiguration[] configurations, DataBlasterParameters parameters, ILogger logger)
{
var context = bulkLoader.NewBulkLoadContext(databaseName);
context.Log = new SitecoreAndUnicornLog(LoggerFactory.GetLogger(GetType()), logger);
context.AllowTemplateChanges = true;
// In Sitecore 9.3 initial release they did change properties on fields (versioned to unversioned) in the core database but it wasn't done in a clean way.
// There are now more than 3000 field records stored in the versioned table which should have been moved to the unverioned table.
// As we don't do bulk imports, or many template changes during development on the core database, we will disable this by default, and avoid deleting relevant content in the core database.
// Sitecore its default deserialize operation is also not doing cleanup work.
context.AllowCleanupOfFields = !databaseName.Equals("core", StringComparison.InvariantCultureIgnoreCase);
context.StageDataWithoutProcessing = parameters.StageDataWithoutProcessing;
// Use the shotgun, removing items one by one is too slow for full deserialize.
context.RemoveItemsFromCaches = false;
context.UpdateHistory = !SkipHistoryEngine;
context.UpdatePublishQueue = !SkipPublishQueue;
context.UpdateLinkDatabase = !SkipLinkDatabase &&
configurations.Any(x => x.Resolve <ISyncConfiguration>().UpdateLinkDatabase);
context.UpdateIndexes = !SkipIndexes &&
configurations.Any(x => x.Resolve <ISyncConfiguration>().UpdateSearchIndex);
return(context);
}
public virtual IEnumerable <BulkLoadItem> ExtractBulkItems(BulkLoadContext context, DataBlasterParameters parameters,
IConfiguration[] configurations, string database)
{
var uniqueItems = new HashSet <Guid>();
return(GetTreeRoots(configurations, database)
.SelectMany(tr =>
{
var action = GetBulkLoadAction(parameters, tr.Item1, tr.Item2);
var dataStore = tr.Item1.Resolve <ITargetDataStore>();
return dataStore.GetByPath(tr.Item2.Path, database)
.SelectMany(i => GetSelfAndAllDescendants(dataStore, i))
// For example '/sitecore/layout/Layouts/User Defined' can occur more than once
// because it has children from different configurations.
// Make sure we add the item itself only once.
.Where(item => uniqueItems.Add(item.Id))
.Where(item => string.IsNullOrEmpty(parameters.AncestorFilter) || item.Path.StartsWith(parameters.AncestorFilter, StringComparison.OrdinalIgnoreCase))
.Select(y => ItemMapper.ToBulkLoadItem(y, context, action));
}));
}