public void Process(BulkLoadContext loadContext, BulkLoadSqlContext sqlContext) { if (!loadContext.UpdateHistory.GetValueOrDefault()) { return; } if (loadContext.ItemChanges.Count == 0) { return; } // In Sitecore 9, history engine is disabled by default if (!HistoryEngineEnabled(loadContext)) { loadContext.Log.Info($"Skipped updating history because history engine is not enabled."); return; } var stopwatch = Stopwatch.StartNew(); var sql = sqlContext.GetEmbeddedSql(loadContext, "Sql.09.UpdateHistory.sql"); sqlContext.ExecuteSql(sql, commandProcessor: cmd => cmd.Parameters.AddWithValue("@UserName", Sitecore.Context.User.Name)); loadContext.Log.Info($"Updated history: {(int)stopwatch.Elapsed.TotalSeconds}s"); }
public IEnumerable<BulkLoadItem> Process(BulkLoadContext context, IEnumerable<BulkLoadItem> items) { if (!context.UpdateLinkDatabase.GetValueOrDefault()) return items; var links = GetItemLinksFromContext(context); return _itemLinkParser.ExtractLinks(items, context, links); }
public IEnumerable <BulkLoadItem> Process(BulkLoadContext context, IEnumerable <BulkLoadItem> items) { // We don't support items without fields. items = items.Where(x => { if (x.FieldCount != 0) { return(true); } context.SkipItemWarning( $"Item with id '{x.Id}', item path '{x.ItemPath}' and source info '{x.SourceInfo}' has no fields."); HasItemsWithoutFields = true; return(false); }); // Item path must be available when item ids need to be looked up. items = items.Where(x => { if (!context.LookupItemIds) { return(true); } if (!string.IsNullOrWhiteSpace(x.ItemPath)) { return(true); } context.SkipItemWarning($"Item with id '{x.Id}', and source info '{x.SourceInfo}' has no item path."); HasItemsWithoutPaths = true; return(false); }); return(items); }
protected virtual void ExtractLinks(BulkLoadContext context, BulkLoadItem item, LinkedList <BulkItemLink> links) { foreach (var field in item.Fields) { ExtractLinks(context, item, field, links); } }
protected virtual void ExtractLinks(BulkLoadContext context, BulkLoadItem item, BulkField field, LinkedList <BulkItemLink> links) { if (string.IsNullOrWhiteSpace(field.Value)) { return; } if (IgnoredFields.Value.Contains(field.Name)) { return; } var ids = IdRegex.Value.Matches(field.Value).Cast <Match>() .Select(x => { ID id; return(ID.TryParse(x.Value, out id) ? id : (ID)null); }) .Concat(LinkRegex.Value.Matches(field.Value).Cast <Match>().Select(x => { Guid guid; return(Guid.TryParse(x.Value, out guid) ? new ID(guid) : (ID)null); })) .Where(x => x != (ID)null); foreach (var link in ids .Select(x => new BulkItemLink( context.Database, new ID(item.Id), new ID(field.Id), context.Database, x, x.ToString()))) { link.ItemAction = item.LoadAction; links.AddLast(link); } }
protected virtual void UpdateIndex(BulkLoadContext context, ICollection <ItemChange> itemChanges, Database database, ISearchIndex index) { Job job = null; if (!context.ShouldUpdateIndex(index)) { context.Log.Warn($"Skipping updating index '{index.Name}' because its empty."); return; } var touchedPercentage = (uint)Math.Ceiling((double)itemChanges.Count / Math.Max(1, index.Summary.NumberOfDocuments) * 100); if (context.IndexRebuildThresholdPercentage.HasValue && touchedPercentage > context.IndexRebuildThresholdPercentage.Value) { context.Log.Info($"Rebuilding index '{index.Name}' because {touchedPercentage}% is changed."); job = IndexCustodian.FullRebuild(index); } else if (context.Destination != null && !itemChanges.Any(ic => ic.Deleted) && // Refresh doesn't do deletes. context.IndexRefreshThresholdPercentage.HasValue && touchedPercentage > context.IndexRefreshThresholdPercentage.Value) { context.Log.Info($"Refreshing index '{index.Name}' from '{context.Destination.ItemPath}' because {touchedPercentage}% is changed."); job = IndexCustodian.Refresh(index, new SitecoreIndexableItem(database.GetItem(new ID(context.Destination.ItemId)))); } else { var sitecoreIds = GetItemsToIndex(itemChanges, database); context.Log.Info($"Updating index '{index.Name}' with {sitecoreIds.Count} items."); job = IndexCustodian.IncrementalUpdate(index, sitecoreIds); } job.Wait(); }
public virtual void Process(BulkLoadContext loadContext, BulkLoadSqlContext sqlContext, ICollection <ItemChange> changes) { if (!loadContext.UpdateIndexes) { return; } if (loadContext.IndexesToUpdate == null || loadContext.IndexesToUpdate.Count == 0) { return; } // We don't index bucket folders. changes = changes?.Where(x => x.TemplateId != BucketFolderTemplate)?.ToList(); if (changes == null || changes.Count == 0) { return; } var stopwatch = Stopwatch.StartNew(); UpdateIndexes(loadContext, changes); loadContext.Log.Info($"Updated content search indexes: {(int)stopwatch.Elapsed.TotalSeconds}s"); loadContext.OnDataIndexed?.Invoke(loadContext, changes); Event.RaiseEvent("bulkloader:dataindexed", loadContext); }
public bool ValidateLoadStage(BulkLoadContext context, BulkLoadSqlContext sqlContext) { if (!CheckTempData(context, sqlContext)) { context.StageFailed(Stage.Load, "Found missing templates, parents, items or fields."); return(false); } return(true); }
public IEnumerable <BulkLoadItem> Process(BulkLoadContext context, IEnumerable <BulkLoadItem> items) { // Statistic fields need to be present to represent the item versions. foreach (var item in items) { //item.EnsureLanguageVersions(forceUpdate: context.ForceUpdates); yield return(item); } }
public bool ValidateLoadStage(BulkLoadContext loadContext, BulkLoadSqlContext sqlContext) { if (!CheckDuplicates(loadContext, sqlContext)) { loadContext.StageFailed(Stage.Load, "Duplicate items were found in set to load."); return(false); } return(true); }
protected virtual void UpdateIndexes(BulkLoadContext context, ICollection <ItemChange> itemChanges) { var db = Factory.GetDatabase(context.Database, true); foreach (var index in context.IndexesToUpdate) { UpdateIndex(context, itemChanges, db, index); } }
public virtual string PostProcessSql(BulkLoadContext context, string sql) { if (context.StageDataWithoutProcessing) { return(sql.Replace("#", "tmp_")); } return(sql); }
public virtual IEnumerable <BulkLoadItem> ExtractLinks(IEnumerable <BulkLoadItem> bulkItems, BulkLoadContext context, LinkedList <BulkItemLink> links) { foreach (var item in bulkItems) { ExtractLinks(context, item, links); yield return(item); } }
public bool ValidateLoadStage(BulkLoadContext loadContext, BulkLoadSqlContext sqlContext) { return(true); //if (!CheckDuplicates(loadContext, sqlContext)) //{ // loadContext.StageFailed(Stage.Load, "Duplicate items were found in set to load."); // return false; //} //return true; }
public void Process(BulkLoadContext loadContext, BulkLoadSqlContext sqlContext, ICollection<ItemChange> changes) { if (!loadContext.RemoveItemsFromCaches) return; var stopwatch = Stopwatch.StartNew(); // Remove items from database cache. // We don't do this within the transaction so that items will be re-read from the committed data. var db = Factory.GetDatabase(loadContext.Database, true); _cachUtil.RemoveItemsFromCachesInBulk(db, GetCacheClearEntries(loadContext.ItemChanges)); loadContext.Log.Info($"Caches cleared: {(int)stopwatch.Elapsed.TotalSeconds}s"); }
protected virtual void UpdateLinkDatabase(BulkLoadContext loadContext, BulkLoadSqlContext sqlContext, LinkedList <BulkItemLink> links, ICollection <ItemChange> changes) { // Index all items that were actually changed in db. var touchedItemsMap = changes .GroupBy(x => x.ItemId) .ToDictionary(x => x.First().OriginalItemId, x => x.First().ItemId); // Get links and filter and map them by touched items. if (links.Count == 0) { return; } var linksForTouchedItems = links .Where(x => touchedItemsMap.ContainsKey(x.SourceItemID.Guid)) .Select(x => x.Map(x.SourceItemID.Guid, touchedItemsMap[x.SourceItemID.Guid])); // Create temp table, bulk load temp table, merge records in link database. // The link database is probably not the same physical db as the one were loading data to. var createLinkTempTable = sqlContext.GetEmbeddedSql(loadContext, "Sql.20.CreateLinkTempTable.sql"); var mergeLinkData = sqlContext.GetEmbeddedSql(loadContext, "Sql.21.MergeLinkTempData.sql"); var connectionString = ((SqlServerLinkDatabase)Factory.GetLinkDatabase()).ConnectionString; using (var conn = new SqlConnection(connectionString)) { conn.Open(); var linkSqlContext = new SqlContext(conn); linkSqlContext.ExecuteSql(createLinkTempTable); using (var bulkCopy = new SqlBulkCopy(conn)) { bulkCopy.BulkCopyTimeout = int.MaxValue; bulkCopy.EnableStreaming = true; bulkCopy.DestinationTableName = sqlContext.PostProcessSql(loadContext, "#ItemLinks"); try { bulkCopy.WriteToServer(new ItemLinksReader(() => linksForTouchedItems.GetEnumerator())); } catch (Exception exception) { loadContext.StageFailed(Stage.Load, exception, $"Write to #ItemLinks failed with message: {exception.Message}"); return; } } linkSqlContext.ExecuteSql(mergeLinkData); } }
public void Process(BulkLoadContext loadContext, BulkLoadSqlContext sqlContext, ICollection<ItemChange> changes) { if (!loadContext.UpdateLinkDatabase.GetValueOrDefault()) return; if (changes.Count == 0) return; var stopwatch = Stopwatch.StartNew(); // Update link database, is in core database, so we can't do this within the transaction. // Links are detected when reading the bulk item stream, // so we assume that the same set will be presented again after a crash. UpdateLinkDatabase(loadContext, sqlContext, GetItemLinksFromContext(loadContext), changes); loadContext.Log.Info($"Updated link database: {(int)stopwatch.Elapsed.TotalSeconds}s"); }
public virtual BulkLoadItem ToBulkLoadItem(IItemData itemData, BulkLoadContext context, BulkLoadAction loadAction) { if (itemData == null) { throw new ArgumentNullException(nameof(itemData)); } if (context == null) { throw new ArgumentNullException(nameof(context)); } var bulkItem = new BulkLoadItem( loadAction, itemData.Id, itemData.TemplateId, itemData.BranchId, itemData.ParentId, itemData.Path, sourceInfo: itemData.SerializedItemId); foreach (var sharedField in itemData.SharedFields) { AddSyncField(context, bulkItem, sharedField); } foreach (var languagedFields in itemData.UnversionedFields) { foreach (var field in languagedFields.Fields) { AddSyncField(context, bulkItem, field, languagedFields.Language.Name); } } foreach (var versionFields in itemData.Versions) { foreach (var field in versionFields.Fields) { AddSyncField(context, bulkItem, field, versionFields.Language.Name, versionFields.VersionNumber); } AddStatisticsFieldsWhenMissing(bulkItem, versionFields.Language.Name, versionFields.VersionNumber); } // Serialized items don't contain the original blob id. context.LookupBlobIds = true; return(bulkItem); }
public bool ValidateLoadStage(BulkLoadContext loadContext, BulkLoadSqlContext sqlContext) { if (HasItemsWithoutFields) { loadContext.StageFailed(Stage.Load, "Items without fields were found ."); return(false); } if (HasItemsWithoutPaths) { loadContext.StageFailed(Stage.Load, "Items were found without paths."); return(false); } return(true); }
private IEnumerable <BulkLoadItem> GetAllItemsToCreate(BulkLoadContext context, CancellationToken cancellationToken) { ItemMapper mapper = new ItemMapper(); while (!Completed) { if (_itemsToCreate.TryTake(out var remoteData, int.MaxValue, cancellationToken)) { yield return(mapper.ToBulkLoadItem(remoteData, context, BulkLoadAction.Update)); } else { break; } } }
private bool CheckTempData(BulkLoadContext loadContext, BulkLoadSqlContext sqlContext) { var check = sqlContext.GetEmbeddedSql(loadContext, "Sql.07.CheckTempData.sql"); var hasErrors = false; using (var cmd = sqlContext.NewSqlCommand(check)) using (var reader = cmd.ExecuteReader()) { while (reader.Read()) { if (!reader.GetBoolean(reader.GetOrdinal("HasParent"))) { loadContext.Log.Error( $"Unable to find parent '{reader["ParentId"]}' for item with id '{reader["Id"]}', " + $"item path '{reader["ItemPath"]}' and source info '{reader["SourceInfo"]}'."); } if (!reader.GetBoolean(reader.GetOrdinal("HasTemplate"))) { loadContext.Log.Error( $"Unable to find template '{reader["TemplateName"]}' with id '{reader["TemplateId"]}' " + $"for item with id '{reader["Id"]}', item path '{reader["ItemPath"]}' and source info '{reader["SourceInfo"]}'."); } hasErrors = true; } reader.NextResult(); while (reader.Read()) { if (!reader.GetBoolean(reader.GetOrdinal("HasItem"))) { loadContext.Log.Error( $"Unable to find item with id '{reader["ItemId"]}', item path '{reader["ItemPath"]}' " + $"and source info '{reader["SourceInfo"]}' for field '{reader["FieldName"]}' with id '{reader["FieldId"]}'."); } if (!reader.GetBoolean(reader.GetOrdinal("HasField"))) { loadContext.Log.Error( $"Unable to find field '{reader["FieldName"]}' with id '{reader["FieldId"]}' " + $"for item with id '{reader["ItemId"]}', item path '{reader["ItemPath"]}' and source info '{reader["SourceInfo"]}'."); } hasErrors = true; } } return(!hasErrors); }
private bool CheckDuplicates(BulkLoadContext context, BulkLoadSqlContext sqlContext) { var check = sqlContext.GetEmbeddedSql(context, "Sql.05.CheckDuplicates.sql"); var hasErrors = false; using (var cmd = sqlContext.NewSqlCommand(check)) using (var reader = cmd.ExecuteReader()) { while (reader.Read()) { context.Log.Error( $"Duplicate item found with id '{reader["Id"]}', item path '{reader["ItemPath"]}' " + $"and source info '{reader["SourceInfo"]}'."); hasErrors = true; } } return(!hasErrors); }
public virtual IEnumerable <BulkLoadItem> ExtractBulkItems(BulkLoadContext context, IConfiguration[] configurations, string database) { var uniqueItems = new HashSet <Guid>(); return(GetTreeRoots(configurations, database) .SelectMany(tr => { var action = GetBulkLoadAction(tr.Item1, tr.Item2); var dataStore = tr.Item1.Resolve <ITargetDataStore>(); return dataStore.GetByPath(tr.Item2.Path, database) .SelectMany(i => GetSelfAndAllDescendants(dataStore, i)) // For example '/sitecore/layout/Layouts/User Defined' can occur more than once // because it has children from different configurations. // Make sure we add the item itself only once. .Where(item => uniqueItems.Add(item.Id)) .Select(y => ItemMapper.ToBulkLoadItem(y, context, action)); })); }
public void Process(BulkLoadContext loadContext, BulkLoadSqlContext sqlContext) { if (!loadContext.UpdatePublishQueue.GetValueOrDefault()) { return; } if (loadContext.ItemChanges.Count == 0) { return; } var stopwatch = Stopwatch.StartNew(); var sql = sqlContext.GetEmbeddedSql(loadContext, "Sql.10.UpdatePublishQueue.sql"); sqlContext.ExecuteSql(sql, commandProcessor: cmd => cmd.Parameters.AddWithValue("@UserName", global::Sitecore.Context.User.Name)); loadContext.Log.Info($"Updated publish queue: {(int)stopwatch.Elapsed.TotalSeconds}s"); }
protected virtual void AddSyncField(BulkLoadContext context, BulkLoadItem bulkItem, IItemFieldValue itemField, string language = null, int versionNumber = 1) { var fieldId = itemField.FieldId; var fieldValue = itemField.Value; var fieldName = itemField.NameHint; var isBlob = itemField.BlobId.HasValue; Func <Stream> blob = null; if (isBlob) { byte[] blobBytes; try { blobBytes = Convert.FromBase64String(fieldValue); } catch (Exception ex) { blobBytes = new byte[] { }; context.Log.Error( $"Unable to read blob from field '{fieldId}' in item with id '{bulkItem.Id}', " + $"item path '{bulkItem.ParentId}' and source info '{bulkItem.SourceInfo}', defaulting to empty value.", ex); } blob = () => new MemoryStream(blobBytes); // Field value needs to be set to the blob id. fieldValue = itemField.BlobId.Value.ToString("B").ToUpper(); } if (language == null) { bulkItem.AddSharedField(fieldId, fieldValue, blob, isBlob, fieldName); } else { bulkItem.AddVersionedField(fieldId, language, versionNumber, fieldValue, blob, isBlob, fieldName); } }
public void Process(BulkLoadContext loadContext, BulkLoadSqlContext sqlContext, ICollection <ItemChange> changes) { loadContext.Log.Info( $"Item changes in database: created: {loadContext.ItemChanges.Count(ic => ic.Created)}, " + $"saved: {loadContext.ItemChanges.Count(ic => ic.Saved)}, moved: {loadContext.ItemChanges.Count(ic => ic.Moved)}, " + $"deleted: {loadContext.ItemChanges.Count(ic => ic.Deleted)}"); if (!loadContext.Log.Logger.IsEnabledFor(Level.TRACE)) { return; } foreach (var change in loadContext.ItemChanges) { if (change.Created) { loadContext.Log.Trace( $"Created item with path '{change.ItemPath ?? "UNKNOWN"}', id '{change.ItemId}', " + $"language '{change.Language ?? "NULL"}' and source info '{change.SourceInfo}' in database."); } if (change.Saved & !change.Created) { loadContext.Log.Trace( $"Saved item with path '{change.ItemPath ?? "UNKNOWN"}', id '{change.ItemId}', " + $"language '{change.Language ?? "NULL"}' and source info '{change.SourceInfo}' in database."); } if (change.Moved) { loadContext.Log.Trace( $"Moved item with path '{change.ItemPath ?? "UNKNOWN"}', id '{change.ItemId}', " + $"language '{change.Language ?? "NULL"}' and source info '{change.SourceInfo}' in database."); } if (change.Deleted) { loadContext.Log.Trace( $"Deleted item with path '{change.ItemPath ?? "UNKNOWN"}', id '{change.ItemId}', " + $"language '{change.Language ?? "NULL"}' and source info '{change.SourceInfo}' in database."); } } }
protected virtual LinkedList<BulkItemLink> GetItemLinksFromContext(BulkLoadContext context) { return context.GetOrAddState("Load.ExtractedLinks", () => new LinkedList<BulkItemLink>()); }
private bool HistoryEngineEnabled(BulkLoadContext context) { var db = Factory.GetDatabase(context.Database, true); return(db.Engines.HistoryEngine.Storage != null); }
/// <summary> /// Returns ancestor items for every needed level between the item and the root. /// </summary> /// <param name="item">Item to generate ancestors for.</param> /// <param name="root">Root that marks the ancestor that should already exist or be present in the stream.</param> /// <param name="ancestorTemplate">Template for the ancestors to generate.</param> /// <param name="context">Context to generate the ancestors in.</param> /// <returns>Stream of generated ancestors, migh be empty if ancestors were already created, /// or it might be partial if shared ancestors were already created for another item in the same context.</returns> public virtual IEnumerable <BulkLoadItem> EnsureAncestorBulkItems(BulkLoadItem item, ItemReference root, Template ancestorTemplate, BulkLoadContext context) { return(EnsureAncestorBulkItems(item, root, ancestorTemplate, item.Id, context)); }
protected virtual IEnumerable <BulkLoadItem> EnsureAncestorBulkItems(BulkLoadItem item, ItemReference root, Template ancestorTemplate, Guid dependsOnItemCreation, BulkLoadContext context) { if (item == null) { throw new ArgumentNullException(nameof(item)); } if (root == null) { throw new ArgumentNullException(nameof(root)); } if (ancestorTemplate == null) { throw new ArgumentNullException(nameof(ancestorTemplate)); } if (context == null) { throw new ArgumentNullException(nameof(context)); } if (!item.ItemPath.StartsWith(root.ItemPath)) { throw new ArgumentException("Bulk item should be a descendant of the root."); } // Detect all the ancestors to generate. var ancestorNames = item.ItemPath.Substring(root.ItemPath.Length) .Split(new[] { '/' }, StringSplitOptions.RemoveEmptyEntries); ancestorNames = ancestorNames.Take(ancestorNames.Length - 1).ToArray(); // Don't include item. // Generate all ancestors. var parent = root; foreach (var name in ancestorNames) { var itemPath = $"{parent.ItemPath}/{name}"; // Maybe we have already generated this path in a previous call to EnsureAncestors within the same context. var itemId = context.GetProcessedPath(itemPath); if (itemId.HasValue) { // Continue with next. parent = new ItemReference(itemId.Value, itemPath); continue; } // Generate stable guid for this ancestor item within its parent. itemId = _guidUtility.Create(parent.ItemId, name); // In case of forced updates, also update the child item. This will result in an ItemChange which makes sure the item gets re-published. var childLoadAction = context.ForceUpdates ? BulkLoadAction.Update : BulkLoadAction.AddOnly; // Create new bulk item. var child = new BulkLoadItem(childLoadAction, itemId.Value, ancestorTemplate.ID.Guid, Guid.Empty, parent.ItemId, itemPath, templateName: ancestorTemplate.Name, sourceInfo: item.SourceInfo) { // Only create ancestor when child is created, skip ancestor creation when child is updated. DependsOnItemCreation = dependsOnItemCreation }; // Attach asap to context, because import profiles might eagerly bucket. context.TrackPathAndTemplateInfo(child); yield return(child); parent = new ItemReference(child.Id, child.ItemPath); } // Reset parent reference for initial item. item.ParentId = parent.ItemId; }