protected virtual bool ValidateAndPrepareData(BulkLoadContext loadContext, BulkLoadSqlContext sqlContext) { var splitTempTablesSql = sqlContext.GetEmbeddedSql(loadContext, "Sql.04.SplitTempTable.sql"); var indexesSql = sqlContext.GetEmbeddedSql(loadContext, "Sql.06.CreateIndexes.sql"); var stopwatch = Stopwatch.StartNew(); sqlContext.ExecuteSql(splitTempTablesSql); if (!OnStagedDataValidating.Execute(p => p.ValidateLoadStage(loadContext, sqlContext), breakOnDefault: true)) { return(false); } sqlContext.ExecuteSql(indexesSql); if (!OnTempDataValidating.Execute(p => p.ValidateLoadStage(loadContext, sqlContext), breakOnDefault: true)) { return(false); } loadContext.Log.Info($"Validated and prepared loaded data: {(int)stopwatch.Elapsed.TotalSeconds}s"); return(true); }
protected virtual void LookupIds(BulkLoadContext loadContext, BulkLoadSqlContext sqlContext, BulkItemsAndFieldsReader itemAndFieldReader) { var lookupBlobsSql = sqlContext.GetEmbeddedSql(loadContext, "Sql.02.LookupBlobs.sql"); var lookupItemsSql = sqlContext.GetEmbeddedSql(loadContext, "Sql.03.LookupItems.sql"); var stopwatch = Stopwatch.StartNew(); if (loadContext.LookupBlobIds) { sqlContext.ExecuteSql(lookupBlobsSql); } if (loadContext.LookupItemIds) { // Using sql parameters resets temp tables. if (loadContext.Destination != null) { lookupItemsSql = sqlContext.ReplaceOneLineSqlStringParameter(lookupItemsSql, "@destinationPath", loadContext.Destination.ItemPath); lookupItemsSql = sqlContext.ReplaceOneLineSqlStringParameter(lookupItemsSql, "@destinationId", loadContext.Destination.ItemId.ToString("D")); } sqlContext.ExecuteSql(lookupItemsSql); } loadContext.Log.Info($"Looked up ids: {(int)stopwatch.Elapsed.TotalSeconds}s"); }
protected virtual void UpdateLinkDatabase(BulkLoadContext loadContext, BulkLoadSqlContext sqlContext, LinkedList <BulkItemLink> links, ICollection <ItemChange> changes) { // Index all items that were actually changed in db. var touchedItemsMap = changes .GroupBy(x => x.ItemId) .ToDictionary(x => x.First().OriginalItemId, x => x.First().ItemId); // Get links and filter and map them by touched items. if (links.Count == 0) { return; } var linksForTouchedItems = links .Where(x => touchedItemsMap.ContainsKey(x.SourceItemID.Guid)) .Select(x => x.Map(x.SourceItemID.Guid, touchedItemsMap[x.SourceItemID.Guid])); // Create temp table, bulk load temp table, merge records in link database. // The link database is probably not the same physical db as the one were loading data to. var createLinkTempTable = sqlContext.GetEmbeddedSql(loadContext, "Sql.20.CreateLinkTempTable.sql"); var mergeLinkData = sqlContext.GetEmbeddedSql(loadContext, "Sql.21.MergeLinkTempData.sql"); var connectionString = ((SqlServerLinkDatabase)Factory.GetLinkDatabase()).ConnectionString; using (var conn = new SqlConnection(connectionString)) { conn.Open(); var linkSqlContext = new SqlContext(conn); linkSqlContext.ExecuteSql(createLinkTempTable); using (var bulkCopy = new SqlBulkCopy(conn)) { bulkCopy.BulkCopyTimeout = int.MaxValue; bulkCopy.EnableStreaming = true; bulkCopy.DestinationTableName = sqlContext.PostProcessSql(loadContext, "#ItemLinks"); try { bulkCopy.WriteToServer(new ItemLinksReader(() => linksForTouchedItems.GetEnumerator())); } catch (Exception exception) { loadContext.StageFailed(Stage.Load, exception, $"Write to #ItemLinks failed with message: {exception.Message}"); return; } } linkSqlContext.ExecuteSql(mergeLinkData); } }
public void Process(BulkLoadContext loadContext, BulkLoadSqlContext sqlContext) { if (!loadContext.UpdateHistory.GetValueOrDefault()) { return; } if (loadContext.ItemChanges.Count == 0) { return; } // In Sitecore 9, history engine is disabled by default if (!HistoryEngineEnabled(loadContext)) { loadContext.Log.Info($"Skipped updating history because history engine is not enabled."); return; } var stopwatch = Stopwatch.StartNew(); var sql = sqlContext.GetEmbeddedSql(loadContext, "Sql.09.UpdateHistory.sql"); sqlContext.ExecuteSql(sql, commandProcessor: cmd => cmd.Parameters.AddWithValue("@UserName", Sitecore.Context.User.Name)); loadContext.Log.Info($"Updated history: {(int)stopwatch.Elapsed.TotalSeconds}s"); }
protected virtual void MergeData(BulkLoadContext loadContext, BulkLoadSqlContext sqlContext, BulkItemsAndFieldsReader itemAndFieldReader) { var sql = sqlContext.GetEmbeddedSql(loadContext, "Sql.08.MergeTempData.sql"); var stopwatch = Stopwatch.StartNew(); // Execute merge and collect imported items. // Using sql parameters resets temp tables. sql = sqlContext.ReplaceOneLineSqlBitParameter(sql, "@ProcessDependingFields", itemAndFieldReader.HasFieldDependencies); sql = sqlContext.ReplaceOneLineSqlBitParameter(sql, "@CleanupBlobs", itemAndFieldReader.HasBlobFields); sql = sqlContext.ReplaceOneLineSqlBitParameter(sql, "@AllowTemplateChanges", loadContext.AllowTemplateChanges); sql = sqlContext.ReplaceOneLineSqlStringParameter(sql, "@DefaultLanguage", LanguageManager.DefaultLanguage.Name); using (var cmd = sqlContext.NewSqlCommand(sql)) { cmd.CommandTimeout = int.MaxValue; using (var reader = cmd.ExecuteReader()) { while (reader.Read()) { loadContext.ItemChanges.AddLast(new ItemChange(reader)); } } } loadContext.Log.Info($"Merged loaded data: {(int)stopwatch.Elapsed.TotalSeconds}s"); }
protected virtual bool StageData(BulkLoadContext loadContext, BulkLoadSqlContext sqlContext, IEnumerable <BulkItem> items, out BulkItemsAndFieldsReader itemAndFieldReader) { var sql = sqlContext.GetEmbeddedSql(loadContext, "Sql.01.CreateTempTable.sql"); // Cleanup left-over staging tables. if (loadContext.StageDataWithoutProcessing) { sqlContext.DropStageTables(); } // Create temp table. // We don't use table valued parameters because we don't want to change the database schema. sqlContext.ExecuteSql(sql); // Load data into temp table. itemAndFieldReader = NewReader(items); if (!BulkCopyToTempTable(loadContext, sqlContext, itemAndFieldReader, NewFieldRulesReader(loadContext))) { return(false); } loadContext.OnDataStaged?.Invoke(loadContext); Event.RaiseEvent("bulkloader:datastaged", loadContext); return(true); }
private bool CheckTempData(BulkLoadContext loadContext, BulkLoadSqlContext sqlContext) { var check = sqlContext.GetEmbeddedSql(loadContext, "Sql.07.CheckTempData.sql"); var hasErrors = false; using (var cmd = sqlContext.NewSqlCommand(check)) using (var reader = cmd.ExecuteReader()) { while (reader.Read()) { if (!reader.GetBoolean(reader.GetOrdinal("HasParent"))) { loadContext.Log.Error( $"Unable to find parent '{reader["ParentId"]}' for item with id '{reader["Id"]}', " + $"item path '{reader["ItemPath"]}' and source info '{reader["SourceInfo"]}'."); } if (!reader.GetBoolean(reader.GetOrdinal("HasTemplate"))) { loadContext.Log.Error( $"Unable to find template '{reader["TemplateName"]}' with id '{reader["TemplateId"]}' " + $"for item with id '{reader["Id"]}', item path '{reader["ItemPath"]}' and source info '{reader["SourceInfo"]}'."); } hasErrors = true; } reader.NextResult(); while (reader.Read()) { if (!reader.GetBoolean(reader.GetOrdinal("HasItem"))) { loadContext.Log.Error( $"Unable to find item with id '{reader["ItemId"]}', item path '{reader["ItemPath"]}' " + $"and source info '{reader["SourceInfo"]}' for field '{reader["FieldName"]}' with id '{reader["FieldId"]}'."); } if (!reader.GetBoolean(reader.GetOrdinal("HasField"))) { loadContext.Log.Error( $"Unable to find field '{reader["FieldName"]}' with id '{reader["FieldId"]}' " + $"for item with id '{reader["ItemId"]}', item path '{reader["ItemPath"]}' and source info '{reader["SourceInfo"]}'."); } hasErrors = true; } } return(!hasErrors); }
private bool CheckDuplicates(BulkLoadContext context, BulkLoadSqlContext sqlContext) { var check = sqlContext.GetEmbeddedSql(context, "Sql.05.CheckDuplicates.sql"); var hasErrors = false; using (var cmd = sqlContext.NewSqlCommand(check)) using (var reader = cmd.ExecuteReader()) { while (reader.Read()) { context.Log.Error( $"Duplicate item found with id '{reader["Id"]}', item path '{reader["ItemPath"]}' " + $"and source info '{reader["SourceInfo"]}'."); hasErrors = true; } } return(!hasErrors); }
public void Process(BulkLoadContext loadContext, BulkLoadSqlContext sqlContext) { if (!loadContext.UpdatePublishQueue.GetValueOrDefault()) { return; } if (loadContext.ItemChanges.Count == 0) { return; } var stopwatch = Stopwatch.StartNew(); var sql = sqlContext.GetEmbeddedSql(loadContext, "Sql.10.UpdatePublishQueue.sql"); sqlContext.ExecuteSql(sql, commandProcessor: cmd => cmd.Parameters.AddWithValue("@UserName", global::Sitecore.Context.User.Name)); loadContext.Log.Info($"Updated publish queue: {(int)stopwatch.Elapsed.TotalSeconds}s"); }