protected virtual void LoadItems(IConfiguration[] configurations, DataBlasterParameters parameters, ILogger logger)
        {
            var databaseNames = configurations
                                .SelectMany(c => c.Resolve <PredicateRootPathResolver>().GetRootPaths().Select(rp => rp.DatabaseName))
                                .Distinct();

            foreach (var databaseName in databaseNames)
            {
                logger.Info($"Syncing database '{databaseName}'...");

                var context   = CreateBulkLoadContext(BulkLoader, databaseName, configurations, parameters, logger);
                var bulkItems = ItemExtractor.ExtractBulkItems(context, configurations, databaseName);
                BulkLoader.LoadItems(context, bulkItems);

                if (context.AnyStageFailed)
                {
                    throw new Exception($"Stage failed during bulkload of database '{databaseName}': {context.FailureMessage}");
                }

                // Support publishing after sync.
                if (!IsUnicornPublishEnabled && !databaseName.Equals("core", StringComparison.OrdinalIgnoreCase))
                {
                    foreach (var itemChange in context.ItemChanges)
                    {
                        ManualPublishQueueHandler.AddItemToPublish(itemChange.ItemId);
                    }
                }
            }
        }
        protected virtual BulkLoadAction GetBulkLoadAction(DataBlasterParameters parameters, IConfiguration configuration, PresetTreeRoot treeRoot)
        {
            if (parameters.ForceBulkLoadAction.HasValue)
            {
                return(parameters.ForceBulkLoadAction.Value);
            }

            var evaluator = configuration.Resolve <IEvaluator>();

            if (evaluator is SerializedAsMasterEvaluator)
            {
                // Only revert the tree when there are no exclusions for this tree root.
                return(treeRoot.Exclusions == null || treeRoot.Exclusions.Count == 0
                    ? BulkLoadAction.RevertTree
                    : BulkLoadAction.Revert);
            }

            if (evaluator is NewItemOnlyEvaluator)
            {
                return(BulkLoadAction.AddItemOnly);
            }

            //if (evaluator is AddOnlyEvaluator)
            //{
            //	return BulkLoadAction.AddOnly;
            //}

            throw new ArgumentException($"Unknown evaluator type: '{evaluator.GetType().Name}'");
        }
        public virtual void Process(UnicornSyncStartPipelineArgs args)
        {
            // Find optional data blaster parameters in custom data of arguments.
            object parms;

            args.CustomData.TryGetValue(PipelineArgsParametersKey, out parms);
            var parameters = parms as DataBlasterParameters;

            if (parameters == null)
            {
                // Is DataBlaster disabled through config?
                if (Settings.GetBoolSetting(DisableDataBlasterSettingName, false))
                {
                    return;
                }

                // Use default parameters.
                parameters = new DataBlasterParameters();
            }

            // Is DataBlaster disabled through parameters?
            if (parameters.DisableDataBlaster)
            {
                return;
            }

            try
            {
                args.Logger.Info($"Start Bulk Unicorn Sync for configurations: '{string.Join("', '", args.Configurations.Select(x => x.Name))}'.");

                var watch          = Stopwatch.StartNew();
                var startTimestamp = DateTime.Now;

                LoadItems(args.Configurations, parameters, args.Logger);
                args.Logger.Info($"Extracted and loaded items ({(int)watch.Elapsed.TotalMilliseconds}ms)");

                watch.Restart();
                ClearCaches();
                args.Logger.Info($"Caches cleared ({(int)watch.Elapsed.TotalMilliseconds}ms)");

                ExecuteUnicornSyncComplete(args, parameters, startTimestamp);
                ExecuteUnicornSyncEnd(args, parameters);
            }
            catch (Exception ex)
            {
                args.Logger.Error(ex);
                throw;
            }
            finally
            {
                // This will signal that we handled the sync for all configurations,
                // and no further handling should be done.
                args.SyncIsHandled = true;
            }
        }
Пример #4
0
        protected virtual void ExecuteUnicornSyncEnd(UnicornSyncStartPipelineArgs args,
                                                     DataBlasterParameters parameters)
        {
            if (parameters.SkipUnicornSyncEnd)
            {
                return;
            }

            // When we tell Unicorn that sync is handled, end pipeline is not called anymore.
            var watch = Stopwatch.StartNew();

            CorePipeline.Run("unicornSyncEnd", new UnicornSyncEndPipelineArgs(args.Logger, true, args.Configurations));
            args.Logger.Info($"Ran sync end pipeline ({(int)watch.Elapsed.TotalMilliseconds}ms)");
        }
Пример #5
0
        protected virtual void ExecuteUnicornSyncComplete(UnicornSyncStartPipelineArgs args,
                                                          DataBlasterParameters parameters,
                                                          DateTime syncStartTimestamp)
        {
            if (parameters.SkipUnicornSyncComplete)
            {
                return;
            }

            // Run complete pipelines to support post-processing, e.g. users and roles.
            var watch = Stopwatch.StartNew();

            foreach (var config in args.Configurations)
            {
                CorePipeline.Run("unicornSyncComplete",
                                 new UnicornSyncCompletePipelineArgs(config, syncStartTimestamp));
            }
            args.Logger.Info($"Ran sync complete pipelines ({(int)watch.Elapsed.TotalMilliseconds}ms)");
        }
Пример #6
0
        protected virtual BulkLoadContext CreateBulkLoadContext(BulkLoader bulkLoader, string databaseName,
                                                                IConfiguration[] configurations, DataBlasterParameters parameters, ILogger logger)
        {
            var context = bulkLoader.NewBulkLoadContext(databaseName);

            context.Log = new SitecoreAndUnicornLog(LoggerFactory.GetLogger(GetType()), logger);

            context.AllowTemplateChanges       = true;
            context.StageDataWithoutProcessing = parameters.StageDataWithoutProcessing;

            // Use the shotgun, removing items one by one is too slow for full deserialize.
            context.RemoveItemsFromCaches = false;

            context.UpdateHistory      = !SkipHistoryEngine;
            context.UpdatePublishQueue = !SkipPublishQueue;
            context.UpdateLinkDatabase = !SkipLinkDatabase &&
                                         configurations.Any(x => x.Resolve <ISyncConfiguration>().UpdateLinkDatabase);
            context.UpdateIndexes = !SkipIndexes &&
                                    configurations.Any(x => x.Resolve <ISyncConfiguration>().UpdateSearchIndex);

            return(context);
        }
Пример #7
0
        protected virtual BulkLoadContext CreateBulkLoadContext(BulkLoader bulkLoader, string databaseName,
                                                                IConfiguration[] configurations, DataBlasterParameters parameters, ILogger logger)
        {
            var context = bulkLoader.NewBulkLoadContext(databaseName);

            context.Log = new SitecoreAndUnicornLog(LoggerFactory.GetLogger(GetType()), logger);

            context.AllowTemplateChanges = true;

            // In Sitecore 9.3 initial release they did change properties on fields (versioned to unversioned) in the core database but it wasn't done in a clean way.
            // There are now more than 3000 field records stored in the versioned table which should have been moved to the unverioned table.
            // As we don't do bulk imports, or many template changes during development on the core database, we will disable this by default, and avoid deleting relevant content in the core database.
            // Sitecore its default deserialize operation is also not doing cleanup work.
            context.AllowCleanupOfFields       = !databaseName.Equals("core", StringComparison.InvariantCultureIgnoreCase);
            context.StageDataWithoutProcessing = parameters.StageDataWithoutProcessing;

            // Use the shotgun, removing items one by one is too slow for full deserialize.
            context.RemoveItemsFromCaches = false;

            context.UpdateHistory      = !SkipHistoryEngine;
            context.UpdatePublishQueue = !SkipPublishQueue;
            context.UpdateLinkDatabase = !SkipLinkDatabase &&
                                         configurations.Any(x => x.Resolve <ISyncConfiguration>().UpdateLinkDatabase);
            context.UpdateIndexes = !SkipIndexes &&
                                    configurations.Any(x => x.Resolve <ISyncConfiguration>().UpdateSearchIndex);

            return(context);
        }
        public virtual IEnumerable <BulkLoadItem> ExtractBulkItems(BulkLoadContext context, DataBlasterParameters parameters,
                                                                   IConfiguration[] configurations, string database)
        {
            var uniqueItems = new HashSet <Guid>();

            return(GetTreeRoots(configurations, database)
                   .SelectMany(tr =>
            {
                var action = GetBulkLoadAction(parameters, tr.Item1, tr.Item2);
                var dataStore = tr.Item1.Resolve <ITargetDataStore>();

                return dataStore.GetByPath(tr.Item2.Path, database)
                .SelectMany(i => GetSelfAndAllDescendants(dataStore, i))
                // For example '/sitecore/layout/Layouts/User Defined' can occur more than once
                // because it has children from different configurations.
                // Make sure we add the item itself only once.
                .Where(item => uniqueItems.Add(item.Id))
                .Where(item => string.IsNullOrEmpty(parameters.AncestorFilter) || item.Path.StartsWith(parameters.AncestorFilter, StringComparison.OrdinalIgnoreCase))
                .Select(y => ItemMapper.ToBulkLoadItem(y, context, action));
            }));
        }