public void Process(BulkLoadContext loadContext, BulkLoadSqlContext sqlContext)
{
if (!loadContext.UpdateHistory.GetValueOrDefault())
{
return;
}
if (loadContext.ItemChanges.Count == 0)
{
return;
}
// In Sitecore 9, history engine is disabled by default
if (!HistoryEngineEnabled(loadContext))
{
loadContext.Log.Info($"Skipped updating history because history engine is not enabled.");
return;
}
var stopwatch = Stopwatch.StartNew();
var sql = sqlContext.GetEmbeddedSql(loadContext, "Sql.09.UpdateHistory.sql");
sqlContext.ExecuteSql(sql,
commandProcessor: cmd => cmd.Parameters.AddWithValue("@UserName", Sitecore.Context.User.Name));
loadContext.Log.Info($"Updated history: {(int)stopwatch.Elapsed.TotalSeconds}s");
}
public IEnumerable<BulkLoadItem> Process(BulkLoadContext context, IEnumerable<BulkLoadItem> items)
{
if (!context.UpdateLinkDatabase.GetValueOrDefault()) return items;
var links = GetItemLinksFromContext(context);
return _itemLinkParser.ExtractLinks(items, context, links);
}
public IEnumerable <BulkLoadItem> Process(BulkLoadContext context, IEnumerable <BulkLoadItem> items)
{
// We don't support items without fields.
items = items.Where(x =>
{
if (x.FieldCount != 0)
{
return(true);
}
context.SkipItemWarning(
$"Item with id '{x.Id}', item path '{x.ItemPath}' and source info '{x.SourceInfo}' has no fields.");
HasItemsWithoutFields = true;
return(false);
});
// Item path must be available when item ids need to be looked up.
items = items.Where(x =>
{
if (!context.LookupItemIds)
{
return(true);
}
if (!string.IsNullOrWhiteSpace(x.ItemPath))
{
return(true);
}
context.SkipItemWarning($"Item with id '{x.Id}', and source info '{x.SourceInfo}' has no item path.");
HasItemsWithoutPaths = true;
return(false);
});
return(items);
}
protected virtual void ExtractLinks(BulkLoadContext context, BulkLoadItem item, LinkedList <BulkItemLink> links)
{
foreach (var field in item.Fields)
{
ExtractLinks(context, item, field, links);
}
}
protected virtual void ExtractLinks(BulkLoadContext context, BulkLoadItem item, BulkField field, LinkedList <BulkItemLink> links)
{
if (string.IsNullOrWhiteSpace(field.Value))
{
return;
}
if (IgnoredFields.Value.Contains(field.Name))
{
return;
}
var ids = IdRegex.Value.Matches(field.Value).Cast <Match>()
.Select(x =>
{
ID id;
return(ID.TryParse(x.Value, out id) ? id : (ID)null);
})
.Concat(LinkRegex.Value.Matches(field.Value).Cast <Match>().Select(x =>
{
Guid guid;
return(Guid.TryParse(x.Value, out guid) ? new ID(guid) : (ID)null);
}))
.Where(x => x != (ID)null);
foreach (var link in ids
.Select(x => new BulkItemLink(
context.Database, new ID(item.Id), new ID(field.Id),
context.Database, x, x.ToString())))
{
link.ItemAction = item.LoadAction;
links.AddLast(link);
}
}
protected virtual void UpdateIndex(BulkLoadContext context, ICollection <ItemChange> itemChanges,
Database database, ISearchIndex index)
{
Job job = null;
if (!context.ShouldUpdateIndex(index))
{
context.Log.Warn($"Skipping updating index '{index.Name}' because its empty.");
return;
}
var touchedPercentage = (uint)Math.Ceiling((double)itemChanges.Count / Math.Max(1, index.Summary.NumberOfDocuments) * 100);
if (context.IndexRebuildThresholdPercentage.HasValue &&
touchedPercentage > context.IndexRebuildThresholdPercentage.Value)
{
context.Log.Info($"Rebuilding index '{index.Name}' because {touchedPercentage}% is changed.");
job = IndexCustodian.FullRebuild(index);
}
else if (context.Destination != null &&
!itemChanges.Any(ic => ic.Deleted) && // Refresh doesn't do deletes.
context.IndexRefreshThresholdPercentage.HasValue &&
touchedPercentage > context.IndexRefreshThresholdPercentage.Value)
{
context.Log.Info($"Refreshing index '{index.Name}' from '{context.Destination.ItemPath}' because {touchedPercentage}% is changed.");
job = IndexCustodian.Refresh(index, new SitecoreIndexableItem(database.GetItem(new ID(context.Destination.ItemId))));
}
else
{
var sitecoreIds = GetItemsToIndex(itemChanges, database);
context.Log.Info($"Updating index '{index.Name}' with {sitecoreIds.Count} items.");
job = IndexCustodian.IncrementalUpdate(index, sitecoreIds);
}
job.Wait();
}
public virtual void Process(BulkLoadContext loadContext, BulkLoadSqlContext sqlContext, ICollection <ItemChange> changes)
{
if (!loadContext.UpdateIndexes)
{
return;
}
if (loadContext.IndexesToUpdate == null || loadContext.IndexesToUpdate.Count == 0)
{
return;
}
// We don't index bucket folders.
changes = changes?.Where(x => x.TemplateId != BucketFolderTemplate)?.ToList();
if (changes == null || changes.Count == 0)
{
return;
}
var stopwatch = Stopwatch.StartNew();
UpdateIndexes(loadContext, changes);
loadContext.Log.Info($"Updated content search indexes: {(int)stopwatch.Elapsed.TotalSeconds}s");
loadContext.OnDataIndexed?.Invoke(loadContext, changes);
Event.RaiseEvent("bulkloader:dataindexed", loadContext);
}
public bool ValidateLoadStage(BulkLoadContext context, BulkLoadSqlContext sqlContext)
{
if (!CheckTempData(context, sqlContext))
{
context.StageFailed(Stage.Load, "Found missing templates, parents, items or fields.");
return(false);
}
return(true);
}
public IEnumerable <BulkLoadItem> Process(BulkLoadContext context, IEnumerable <BulkLoadItem> items)
{
// Statistic fields need to be present to represent the item versions.
foreach (var item in items)
{
//item.EnsureLanguageVersions(forceUpdate: context.ForceUpdates);
yield return(item);
}
}
public bool ValidateLoadStage(BulkLoadContext loadContext, BulkLoadSqlContext sqlContext)
{
if (!CheckDuplicates(loadContext, sqlContext))
{
loadContext.StageFailed(Stage.Load, "Duplicate items were found in set to load.");
return(false);
}
return(true);
}
protected virtual void UpdateIndexes(BulkLoadContext context, ICollection <ItemChange> itemChanges)
{
var db = Factory.GetDatabase(context.Database, true);
foreach (var index in context.IndexesToUpdate)
{
UpdateIndex(context, itemChanges, db, index);
}
}
public virtual string PostProcessSql(BulkLoadContext context, string sql)
{
if (context.StageDataWithoutProcessing)
{
return(sql.Replace("#", "tmp_"));
}
return(sql);
}
public virtual IEnumerable <BulkLoadItem> ExtractLinks(IEnumerable <BulkLoadItem> bulkItems,
BulkLoadContext context,
LinkedList <BulkItemLink> links)
{
foreach (var item in bulkItems)
{
ExtractLinks(context, item, links);
yield return(item);
}
}
public bool ValidateLoadStage(BulkLoadContext loadContext, BulkLoadSqlContext sqlContext)
{
return(true);
//if (!CheckDuplicates(loadContext, sqlContext))
//{
// loadContext.StageFailed(Stage.Load, "Duplicate items were found in set to load.");
// return false;
//}
//return true;
}
public void Process(BulkLoadContext loadContext, BulkLoadSqlContext sqlContext, ICollection<ItemChange> changes)
{
if (!loadContext.RemoveItemsFromCaches) return;
var stopwatch = Stopwatch.StartNew();
// Remove items from database cache.
// We don't do this within the transaction so that items will be re-read from the committed data.
var db = Factory.GetDatabase(loadContext.Database, true);
_cachUtil.RemoveItemsFromCachesInBulk(db, GetCacheClearEntries(loadContext.ItemChanges));
loadContext.Log.Info($"Caches cleared: {(int)stopwatch.Elapsed.TotalSeconds}s");
}
protected virtual void UpdateLinkDatabase(BulkLoadContext loadContext, BulkLoadSqlContext sqlContext,
LinkedList <BulkItemLink> links, ICollection <ItemChange> changes)
{
// Index all items that were actually changed in db.
var touchedItemsMap = changes
.GroupBy(x => x.ItemId)
.ToDictionary(x => x.First().OriginalItemId, x => x.First().ItemId);
// Get links and filter and map them by touched items.
if (links.Count == 0)
{
return;
}
var linksForTouchedItems = links
.Where(x => touchedItemsMap.ContainsKey(x.SourceItemID.Guid))
.Select(x => x.Map(x.SourceItemID.Guid, touchedItemsMap[x.SourceItemID.Guid]));
// Create temp table, bulk load temp table, merge records in link database.
// The link database is probably not the same physical db as the one were loading data to.
var createLinkTempTable = sqlContext.GetEmbeddedSql(loadContext, "Sql.20.CreateLinkTempTable.sql");
var mergeLinkData = sqlContext.GetEmbeddedSql(loadContext, "Sql.21.MergeLinkTempData.sql");
var connectionString = ((SqlServerLinkDatabase)Factory.GetLinkDatabase()).ConnectionString;
using (var conn = new SqlConnection(connectionString))
{
conn.Open();
var linkSqlContext = new SqlContext(conn);
linkSqlContext.ExecuteSql(createLinkTempTable);
using (var bulkCopy = new SqlBulkCopy(conn))
{
bulkCopy.BulkCopyTimeout = int.MaxValue;
bulkCopy.EnableStreaming = true;
bulkCopy.DestinationTableName = sqlContext.PostProcessSql(loadContext, "#ItemLinks");
try
{
bulkCopy.WriteToServer(new ItemLinksReader(() => linksForTouchedItems.GetEnumerator()));
}
catch (Exception exception)
{
loadContext.StageFailed(Stage.Load, exception,
$"Write to #ItemLinks failed with message: {exception.Message}");
return;
}
}
linkSqlContext.ExecuteSql(mergeLinkData);
}
}
public void Process(BulkLoadContext loadContext, BulkLoadSqlContext sqlContext, ICollection<ItemChange> changes)
{
if (!loadContext.UpdateLinkDatabase.GetValueOrDefault()) return;
if (changes.Count == 0) return;
var stopwatch = Stopwatch.StartNew();
// Update link database, is in core database, so we can't do this within the transaction.
// Links are detected when reading the bulk item stream,
// so we assume that the same set will be presented again after a crash.
UpdateLinkDatabase(loadContext, sqlContext, GetItemLinksFromContext(loadContext), changes);
loadContext.Log.Info($"Updated link database: {(int)stopwatch.Elapsed.TotalSeconds}s");
}
public virtual BulkLoadItem ToBulkLoadItem(IItemData itemData, BulkLoadContext context,
BulkLoadAction loadAction)
{
if (itemData == null)
{
throw new ArgumentNullException(nameof(itemData));
}
if (context == null)
{
throw new ArgumentNullException(nameof(context));
}
var bulkItem = new BulkLoadItem(
loadAction,
itemData.Id,
itemData.TemplateId,
itemData.BranchId,
itemData.ParentId,
itemData.Path,
sourceInfo: itemData.SerializedItemId);
foreach (var sharedField in itemData.SharedFields)
{
AddSyncField(context, bulkItem, sharedField);
}
foreach (var languagedFields in itemData.UnversionedFields)
{
foreach (var field in languagedFields.Fields)
{
AddSyncField(context, bulkItem, field, languagedFields.Language.Name);
}
}
foreach (var versionFields in itemData.Versions)
{
foreach (var field in versionFields.Fields)
{
AddSyncField(context, bulkItem, field, versionFields.Language.Name, versionFields.VersionNumber);
}
AddStatisticsFieldsWhenMissing(bulkItem, versionFields.Language.Name, versionFields.VersionNumber);
}
// Serialized items don't contain the original blob id.
context.LookupBlobIds = true;
return(bulkItem);
}
public bool ValidateLoadStage(BulkLoadContext loadContext, BulkLoadSqlContext sqlContext)
{
if (HasItemsWithoutFields)
{
loadContext.StageFailed(Stage.Load, "Items without fields were found .");
return(false);
}
if (HasItemsWithoutPaths)
{
loadContext.StageFailed(Stage.Load, "Items were found without paths.");
return(false);
}
return(true);
}
private IEnumerable <BulkLoadItem> GetAllItemsToCreate(BulkLoadContext context, CancellationToken cancellationToken)
{
ItemMapper mapper = new ItemMapper();
while (!Completed)
{
if (_itemsToCreate.TryTake(out var remoteData, int.MaxValue, cancellationToken))
{
yield return(mapper.ToBulkLoadItem(remoteData, context, BulkLoadAction.Update));
}
else
{
break;
}
}
}
private bool CheckTempData(BulkLoadContext loadContext, BulkLoadSqlContext sqlContext)
{
var check = sqlContext.GetEmbeddedSql(loadContext, "Sql.07.CheckTempData.sql");
var hasErrors = false;
using (var cmd = sqlContext.NewSqlCommand(check))
using (var reader = cmd.ExecuteReader())
{
while (reader.Read())
{
if (!reader.GetBoolean(reader.GetOrdinal("HasParent")))
{
loadContext.Log.Error(
$"Unable to find parent '{reader["ParentId"]}' for item with id '{reader["Id"]}', " +
$"item path '{reader["ItemPath"]}' and source info '{reader["SourceInfo"]}'.");
}
if (!reader.GetBoolean(reader.GetOrdinal("HasTemplate")))
{
loadContext.Log.Error(
$"Unable to find template '{reader["TemplateName"]}' with id '{reader["TemplateId"]}' " +
$"for item with id '{reader["Id"]}', item path '{reader["ItemPath"]}' and source info '{reader["SourceInfo"]}'.");
}
hasErrors = true;
}
reader.NextResult();
while (reader.Read())
{
if (!reader.GetBoolean(reader.GetOrdinal("HasItem")))
{
loadContext.Log.Error(
$"Unable to find item with id '{reader["ItemId"]}', item path '{reader["ItemPath"]}' " +
$"and source info '{reader["SourceInfo"]}' for field '{reader["FieldName"]}' with id '{reader["FieldId"]}'.");
}
if (!reader.GetBoolean(reader.GetOrdinal("HasField")))
{
loadContext.Log.Error(
$"Unable to find field '{reader["FieldName"]}' with id '{reader["FieldId"]}' " +
$"for item with id '{reader["ItemId"]}', item path '{reader["ItemPath"]}' and source info '{reader["SourceInfo"]}'.");
}
hasErrors = true;
}
}
return(!hasErrors);
}
private bool CheckDuplicates(BulkLoadContext context, BulkLoadSqlContext sqlContext)
{
var check = sqlContext.GetEmbeddedSql(context, "Sql.05.CheckDuplicates.sql");
var hasErrors = false;
using (var cmd = sqlContext.NewSqlCommand(check))
using (var reader = cmd.ExecuteReader())
{
while (reader.Read())
{
context.Log.Error(
$"Duplicate item found with id '{reader["Id"]}', item path '{reader["ItemPath"]}' " +
$"and source info '{reader["SourceInfo"]}'.");
hasErrors = true;
}
}
return(!hasErrors);
}
public virtual IEnumerable <BulkLoadItem> ExtractBulkItems(BulkLoadContext context, IConfiguration[] configurations, string database)
{
var uniqueItems = new HashSet <Guid>();
return(GetTreeRoots(configurations, database)
.SelectMany(tr =>
{
var action = GetBulkLoadAction(tr.Item1, tr.Item2);
var dataStore = tr.Item1.Resolve <ITargetDataStore>();
return dataStore.GetByPath(tr.Item2.Path, database)
.SelectMany(i => GetSelfAndAllDescendants(dataStore, i))
// For example '/sitecore/layout/Layouts/User Defined' can occur more than once
// because it has children from different configurations.
// Make sure we add the item itself only once.
.Where(item => uniqueItems.Add(item.Id))
.Select(y => ItemMapper.ToBulkLoadItem(y, context, action));
}));
}
public void Process(BulkLoadContext loadContext, BulkLoadSqlContext sqlContext)
{
if (!loadContext.UpdatePublishQueue.GetValueOrDefault())
{
return;
}
if (loadContext.ItemChanges.Count == 0)
{
return;
}
var stopwatch = Stopwatch.StartNew();
var sql = sqlContext.GetEmbeddedSql(loadContext, "Sql.10.UpdatePublishQueue.sql");
sqlContext.ExecuteSql(sql,
commandProcessor: cmd => cmd.Parameters.AddWithValue("@UserName", global::Sitecore.Context.User.Name));
loadContext.Log.Info($"Updated publish queue: {(int)stopwatch.Elapsed.TotalSeconds}s");
}
protected virtual void AddSyncField(BulkLoadContext context, BulkLoadItem bulkItem, IItemFieldValue itemField,
string language = null, int versionNumber = 1)
{
var fieldId = itemField.FieldId;
var fieldValue = itemField.Value;
var fieldName = itemField.NameHint;
var isBlob = itemField.BlobId.HasValue;
Func <Stream> blob = null;
if (isBlob)
{
byte[] blobBytes;
try
{
blobBytes = Convert.FromBase64String(fieldValue);
}
catch (Exception ex)
{
blobBytes = new byte[] { };
context.Log.Error(
$"Unable to read blob from field '{fieldId}' in item with id '{bulkItem.Id}', " +
$"item path '{bulkItem.ParentId}' and source info '{bulkItem.SourceInfo}', defaulting to empty value.",
ex);
}
blob = () => new MemoryStream(blobBytes);
// Field value needs to be set to the blob id.
fieldValue = itemField.BlobId.Value.ToString("B").ToUpper();
}
if (language == null)
{
bulkItem.AddSharedField(fieldId, fieldValue, blob, isBlob, fieldName);
}
else
{
bulkItem.AddVersionedField(fieldId, language, versionNumber, fieldValue, blob, isBlob, fieldName);
}
}
public void Process(BulkLoadContext loadContext, BulkLoadSqlContext sqlContext, ICollection <ItemChange> changes)
{
loadContext.Log.Info(
$"Item changes in database: created: {loadContext.ItemChanges.Count(ic => ic.Created)}, " +
$"saved: {loadContext.ItemChanges.Count(ic => ic.Saved)}, moved: {loadContext.ItemChanges.Count(ic => ic.Moved)}, " +
$"deleted: {loadContext.ItemChanges.Count(ic => ic.Deleted)}");
if (!loadContext.Log.Logger.IsEnabledFor(Level.TRACE))
{
return;
}
foreach (var change in loadContext.ItemChanges)
{
if (change.Created)
{
loadContext.Log.Trace(
$"Created item with path '{change.ItemPath ?? "UNKNOWN"}', id '{change.ItemId}', " +
$"language '{change.Language ?? "NULL"}' and source info '{change.SourceInfo}' in database.");
}
if (change.Saved & !change.Created)
{
loadContext.Log.Trace(
$"Saved item with path '{change.ItemPath ?? "UNKNOWN"}', id '{change.ItemId}', " +
$"language '{change.Language ?? "NULL"}' and source info '{change.SourceInfo}' in database.");
}
if (change.Moved)
{
loadContext.Log.Trace(
$"Moved item with path '{change.ItemPath ?? "UNKNOWN"}', id '{change.ItemId}', " +
$"language '{change.Language ?? "NULL"}' and source info '{change.SourceInfo}' in database.");
}
if (change.Deleted)
{
loadContext.Log.Trace(
$"Deleted item with path '{change.ItemPath ?? "UNKNOWN"}', id '{change.ItemId}', " +
$"language '{change.Language ?? "NULL"}' and source info '{change.SourceInfo}' in database.");
}
}
}
protected virtual LinkedList<BulkItemLink> GetItemLinksFromContext(BulkLoadContext context)
{
return context.GetOrAddState("Load.ExtractedLinks", () => new LinkedList<BulkItemLink>());
}
private bool HistoryEngineEnabled(BulkLoadContext context)
{
var db = Factory.GetDatabase(context.Database, true);
return(db.Engines.HistoryEngine.Storage != null);
}
/// <summary>
/// Returns ancestor items for every needed level between the item and the root.
/// </summary>
/// <param name="item">Item to generate ancestors for.</param>
/// <param name="root">Root that marks the ancestor that should already exist or be present in the stream.</param>
/// <param name="ancestorTemplate">Template for the ancestors to generate.</param>
/// <param name="context">Context to generate the ancestors in.</param>
/// <returns>Stream of generated ancestors, migh be empty if ancestors were already created,
/// or it might be partial if shared ancestors were already created for another item in the same context.</returns>
public virtual IEnumerable <BulkLoadItem> EnsureAncestorBulkItems(BulkLoadItem item, ItemReference root,
Template ancestorTemplate, BulkLoadContext context)
{
return(EnsureAncestorBulkItems(item, root, ancestorTemplate, item.Id, context));
}
protected virtual IEnumerable <BulkLoadItem> EnsureAncestorBulkItems(BulkLoadItem item,
ItemReference root, Template ancestorTemplate, Guid dependsOnItemCreation,
BulkLoadContext context)
{
if (item == null)
{
throw new ArgumentNullException(nameof(item));
}
if (root == null)
{
throw new ArgumentNullException(nameof(root));
}
if (ancestorTemplate == null)
{
throw new ArgumentNullException(nameof(ancestorTemplate));
}
if (context == null)
{
throw new ArgumentNullException(nameof(context));
}
if (!item.ItemPath.StartsWith(root.ItemPath))
{
throw new ArgumentException("Bulk item should be a descendant of the root.");
}
// Detect all the ancestors to generate.
var ancestorNames = item.ItemPath.Substring(root.ItemPath.Length)
.Split(new[] { '/' }, StringSplitOptions.RemoveEmptyEntries);
ancestorNames = ancestorNames.Take(ancestorNames.Length - 1).ToArray(); // Don't include item.
// Generate all ancestors.
var parent = root;
foreach (var name in ancestorNames)
{
var itemPath = $"{parent.ItemPath}/{name}";
// Maybe we have already generated this path in a previous call to EnsureAncestors within the same context.
var itemId = context.GetProcessedPath(itemPath);
if (itemId.HasValue)
{
// Continue with next.
parent = new ItemReference(itemId.Value, itemPath);
continue;
}
// Generate stable guid for this ancestor item within its parent.
itemId = _guidUtility.Create(parent.ItemId, name);
// In case of forced updates, also update the child item. This will result in an ItemChange which makes sure the item gets re-published.
var childLoadAction = context.ForceUpdates ? BulkLoadAction.Update : BulkLoadAction.AddOnly;
// Create new bulk item.
var child = new BulkLoadItem(childLoadAction, itemId.Value, ancestorTemplate.ID.Guid,
Guid.Empty, parent.ItemId, itemPath,
templateName: ancestorTemplate.Name, sourceInfo: item.SourceInfo)
{
// Only create ancestor when child is created, skip ancestor creation when child is updated.
DependsOnItemCreation = dependsOnItemCreation
};
// Attach asap to context, because import profiles might eagerly bucket.
context.TrackPathAndTemplateInfo(child);
yield return(child);
parent = new ItemReference(child.Id, child.ItemPath);
}
// Reset parent reference for initial item.
item.ParentId = parent.ItemId;
}
