protected virtual void MergeData(BulkLoadContext loadContext, BulkLoadSqlContext sqlContext,
BulkItemsAndFieldsReader itemAndFieldReader)
{
var sql = sqlContext.GetEmbeddedSql(loadContext, "Sql.08.MergeTempData.sql");
var stopwatch = Stopwatch.StartNew();
// Execute merge and collect imported items.
// Using sql parameters resets temp tables.
sql = sqlContext.ReplaceOneLineSqlBitParameter(sql, "@ProcessDependingFields",
itemAndFieldReader.HasFieldDependencies);
sql = sqlContext.ReplaceOneLineSqlBitParameter(sql, "@CleanupBlobs",
itemAndFieldReader.HasBlobFields);
sql = sqlContext.ReplaceOneLineSqlBitParameter(sql, "@AllowTemplateChanges",
loadContext.AllowTemplateChanges);
sql = sqlContext.ReplaceOneLineSqlStringParameter(sql, "@DefaultLanguage",
LanguageManager.DefaultLanguage.Name);
using (var cmd = sqlContext.NewSqlCommand(sql))
{
cmd.CommandTimeout = int.MaxValue;
using (var reader = cmd.ExecuteReader())
{
while (reader.Read())
{
loadContext.ItemChanges.AddLast(new ItemChange(reader));
}
}
}
loadContext.Log.Info($"Merged loaded data: {(int)stopwatch.Elapsed.TotalSeconds}s");
}
protected virtual bool StageData(BulkLoadContext loadContext, BulkLoadSqlContext sqlContext,
IEnumerable <BulkItem> items, out BulkItemsAndFieldsReader itemAndFieldReader)
{
var sql = sqlContext.GetEmbeddedSql(loadContext, "Sql.01.CreateTempTable.sql");
// Cleanup left-over staging tables.
if (loadContext.StageDataWithoutProcessing)
{
sqlContext.DropStageTables();
}
// Create temp table.
// We don't use table valued parameters because we don't want to change the database schema.
sqlContext.ExecuteSql(sql);
// Load data into temp table.
itemAndFieldReader = NewReader(items);
if (!BulkCopyToTempTable(loadContext, sqlContext, itemAndFieldReader, NewFieldRulesReader(loadContext)))
{
return(false);
}
loadContext.OnDataStaged?.Invoke(loadContext);
Event.RaiseEvent("bulkloader:datastaged", loadContext);
return(true);
}
protected virtual void LookupIds(BulkLoadContext loadContext, BulkLoadSqlContext sqlContext,
BulkItemsAndFieldsReader itemAndFieldReader)
{
var lookupBlobsSql = sqlContext.GetEmbeddedSql(loadContext, "Sql.02.LookupBlobs.sql");
var lookupItemsSql = sqlContext.GetEmbeddedSql(loadContext, "Sql.03.LookupItems.sql");
var stopwatch = Stopwatch.StartNew();
if (loadContext.LookupBlobIds)
{
sqlContext.ExecuteSql(lookupBlobsSql);
}
if (loadContext.LookupItemIds)
{
// Using sql parameters resets temp tables.
if (loadContext.Destination != null)
{
lookupItemsSql = sqlContext.ReplaceOneLineSqlStringParameter(lookupItemsSql, "@destinationPath",
loadContext.Destination.ItemPath);
lookupItemsSql = sqlContext.ReplaceOneLineSqlStringParameter(lookupItemsSql, "@destinationId",
loadContext.Destination.ItemId.ToString("D"));
}
sqlContext.ExecuteSql(lookupItemsSql);
}
loadContext.Log.Info($"Looked up ids: {(int)stopwatch.Elapsed.TotalSeconds}s");
}
protected virtual bool BulkCopyToTempTable(BulkLoadContext loadContext, BulkLoadSqlContext sqlContext,
BulkItemsAndFieldsReader itemAndFieldReader, FieldRulesReader fieldRulesReader)
{
var stopwatch = Stopwatch.StartNew();
// Bulk copy into temp tables, so that we don't have to buffer stuff,
// blobs can give OutOfMemoryExceptions.
using (var bulkCopy = new SqlBulkCopy(sqlContext.Connection))
{
bulkCopy.BulkCopyTimeout = int.MaxValue;
bulkCopy.EnableStreaming = true;
bulkCopy.DestinationTableName = sqlContext.PostProcessSql(loadContext, "#BulkItemsAndFields");
try
{
bulkCopy.WriteToServer(itemAndFieldReader);
}
catch (Exception exception)
{
loadContext.StageFailed(Stage.Load, exception,
$"Write to #BulkItemsAndFields failed with message: {exception.Message}");
return(false);
}
if (fieldRulesReader != null)
{
bulkCopy.DestinationTableName = sqlContext.PostProcessSql(loadContext, "#FieldRules");
try
{
bulkCopy.WriteToServer(fieldRulesReader);
}
catch (Exception exception)
{
loadContext.StageFailed(Stage.Load, exception,
$"Write to #FieldRules failed with message: {exception}");
}
}
}
loadContext.Log.Info($"Loaded data in database: {(int)stopwatch.Elapsed.TotalSeconds}s");
stopwatch.Restart();
return(true);
}
