protected virtual void LoadItems(IConfiguration[] configurations, DataBlasterParameters parameters, ILogger logger) { var databaseNames = configurations .SelectMany(c => c.Resolve <PredicateRootPathResolver>().GetRootPaths().Select(rp => rp.DatabaseName)) .Distinct(); foreach (var databaseName in databaseNames) { logger.Info($"Syncing database '{databaseName}'..."); var context = CreateBulkLoadContext(BulkLoader, databaseName, configurations, parameters, logger); var bulkItems = ItemExtractor.ExtractBulkItems(context, configurations, databaseName); BulkLoader.LoadItems(context, bulkItems); if (context.AnyStageFailed) { throw new Exception($"Stage failed during bulkload of database '{databaseName}': {context.FailureMessage}"); } // Support publishing after sync. if (!IsUnicornPublishEnabled && !databaseName.Equals("core", StringComparison.OrdinalIgnoreCase)) { foreach (var itemChange in context.ItemChanges) { ManualPublishQueueHandler.AddItemToPublish(itemChange.ItemId); } } } }
protected virtual BulkLoadAction GetBulkLoadAction(DataBlasterParameters parameters, IConfiguration configuration, PresetTreeRoot treeRoot) { if (parameters.ForceBulkLoadAction.HasValue) { return(parameters.ForceBulkLoadAction.Value); } var evaluator = configuration.Resolve <IEvaluator>(); if (evaluator is SerializedAsMasterEvaluator) { // Only revert the tree when there are no exclusions for this tree root. return(treeRoot.Exclusions == null || treeRoot.Exclusions.Count == 0 ? BulkLoadAction.RevertTree : BulkLoadAction.Revert); } if (evaluator is NewItemOnlyEvaluator) { return(BulkLoadAction.AddItemOnly); } //if (evaluator is AddOnlyEvaluator) //{ // return BulkLoadAction.AddOnly; //} throw new ArgumentException($"Unknown evaluator type: '{evaluator.GetType().Name}'"); }
public virtual void Process(UnicornSyncStartPipelineArgs args) { // Find optional data blaster parameters in custom data of arguments. object parms; args.CustomData.TryGetValue(PipelineArgsParametersKey, out parms); var parameters = parms as DataBlasterParameters; if (parameters == null) { // Is DataBlaster disabled through config? if (Settings.GetBoolSetting(DisableDataBlasterSettingName, false)) { return; } // Use default parameters. parameters = new DataBlasterParameters(); } // Is DataBlaster disabled through parameters? if (parameters.DisableDataBlaster) { return; } try { args.Logger.Info($"Start Bulk Unicorn Sync for configurations: '{string.Join("', '", args.Configurations.Select(x => x.Name))}'."); var watch = Stopwatch.StartNew(); var startTimestamp = DateTime.Now; LoadItems(args.Configurations, parameters, args.Logger); args.Logger.Info($"Extracted and loaded items ({(int)watch.Elapsed.TotalMilliseconds}ms)"); watch.Restart(); ClearCaches(); args.Logger.Info($"Caches cleared ({(int)watch.Elapsed.TotalMilliseconds}ms)"); ExecuteUnicornSyncComplete(args, parameters, startTimestamp); ExecuteUnicornSyncEnd(args, parameters); } catch (Exception ex) { args.Logger.Error(ex); throw; } finally { // This will signal that we handled the sync for all configurations, // and no further handling should be done. args.SyncIsHandled = true; } }
protected virtual void ExecuteUnicornSyncEnd(UnicornSyncStartPipelineArgs args, DataBlasterParameters parameters) { if (parameters.SkipUnicornSyncEnd) { return; } // When we tell Unicorn that sync is handled, end pipeline is not called anymore. var watch = Stopwatch.StartNew(); CorePipeline.Run("unicornSyncEnd", new UnicornSyncEndPipelineArgs(args.Logger, true, args.Configurations)); args.Logger.Info($"Ran sync end pipeline ({(int)watch.Elapsed.TotalMilliseconds}ms)"); }
protected virtual void ExecuteUnicornSyncComplete(UnicornSyncStartPipelineArgs args, DataBlasterParameters parameters, DateTime syncStartTimestamp) { if (parameters.SkipUnicornSyncComplete) { return; } // Run complete pipelines to support post-processing, e.g. users and roles. var watch = Stopwatch.StartNew(); foreach (var config in args.Configurations) { CorePipeline.Run("unicornSyncComplete", new UnicornSyncCompletePipelineArgs(config, syncStartTimestamp)); } args.Logger.Info($"Ran sync complete pipelines ({(int)watch.Elapsed.TotalMilliseconds}ms)"); }
protected virtual BulkLoadContext CreateBulkLoadContext(BulkLoader bulkLoader, string databaseName, IConfiguration[] configurations, DataBlasterParameters parameters, ILogger logger) { var context = bulkLoader.NewBulkLoadContext(databaseName); context.Log = new SitecoreAndUnicornLog(LoggerFactory.GetLogger(GetType()), logger); context.AllowTemplateChanges = true; context.StageDataWithoutProcessing = parameters.StageDataWithoutProcessing; // Use the shotgun, removing items one by one is too slow for full deserialize. context.RemoveItemsFromCaches = false; context.UpdateHistory = !SkipHistoryEngine; context.UpdatePublishQueue = !SkipPublishQueue; context.UpdateLinkDatabase = !SkipLinkDatabase && configurations.Any(x => x.Resolve <ISyncConfiguration>().UpdateLinkDatabase); context.UpdateIndexes = !SkipIndexes && configurations.Any(x => x.Resolve <ISyncConfiguration>().UpdateSearchIndex); return(context); }
protected virtual BulkLoadContext CreateBulkLoadContext(BulkLoader bulkLoader, string databaseName, IConfiguration[] configurations, DataBlasterParameters parameters, ILogger logger) { var context = bulkLoader.NewBulkLoadContext(databaseName); context.Log = new SitecoreAndUnicornLog(LoggerFactory.GetLogger(GetType()), logger); context.AllowTemplateChanges = true; // In Sitecore 9.3 initial release they did change properties on fields (versioned to unversioned) in the core database but it wasn't done in a clean way. // There are now more than 3000 field records stored in the versioned table which should have been moved to the unverioned table. // As we don't do bulk imports, or many template changes during development on the core database, we will disable this by default, and avoid deleting relevant content in the core database. // Sitecore its default deserialize operation is also not doing cleanup work. context.AllowCleanupOfFields = !databaseName.Equals("core", StringComparison.InvariantCultureIgnoreCase); context.StageDataWithoutProcessing = parameters.StageDataWithoutProcessing; // Use the shotgun, removing items one by one is too slow for full deserialize. context.RemoveItemsFromCaches = false; context.UpdateHistory = !SkipHistoryEngine; context.UpdatePublishQueue = !SkipPublishQueue; context.UpdateLinkDatabase = !SkipLinkDatabase && configurations.Any(x => x.Resolve <ISyncConfiguration>().UpdateLinkDatabase); context.UpdateIndexes = !SkipIndexes && configurations.Any(x => x.Resolve <ISyncConfiguration>().UpdateSearchIndex); return(context); }
public virtual IEnumerable <BulkLoadItem> ExtractBulkItems(BulkLoadContext context, DataBlasterParameters parameters, IConfiguration[] configurations, string database) { var uniqueItems = new HashSet <Guid>(); return(GetTreeRoots(configurations, database) .SelectMany(tr => { var action = GetBulkLoadAction(parameters, tr.Item1, tr.Item2); var dataStore = tr.Item1.Resolve <ITargetDataStore>(); return dataStore.GetByPath(tr.Item2.Path, database) .SelectMany(i => GetSelfAndAllDescendants(dataStore, i)) // For example '/sitecore/layout/Layouts/User Defined' can occur more than once // because it has children from different configurations. // Make sure we add the item itself only once. .Where(item => uniqueItems.Add(item.Id)) .Where(item => string.IsNullOrEmpty(parameters.AncestorFilter) || item.Path.StartsWith(parameters.AncestorFilter, StringComparison.OrdinalIgnoreCase)) .Select(y => ItemMapper.ToBulkLoadItem(y, context, action)); })); }