protected virtual void MergeData(BulkLoadContext loadContext, BulkLoadSqlContext sqlContext, BulkItemsAndFieldsReader itemAndFieldReader) { var sql = sqlContext.GetEmbeddedSql(loadContext, "Sql.08.MergeTempData.sql"); var stopwatch = Stopwatch.StartNew(); // Execute merge and collect imported items. // Using sql parameters resets temp tables. sql = sqlContext.ReplaceOneLineSqlBitParameter(sql, "@ProcessDependingFields", itemAndFieldReader.HasFieldDependencies); sql = sqlContext.ReplaceOneLineSqlBitParameter(sql, "@CleanupBlobs", itemAndFieldReader.HasBlobFields); sql = sqlContext.ReplaceOneLineSqlBitParameter(sql, "@AllowTemplateChanges", loadContext.AllowTemplateChanges); sql = sqlContext.ReplaceOneLineSqlStringParameter(sql, "@DefaultLanguage", LanguageManager.DefaultLanguage.Name); using (var cmd = sqlContext.NewSqlCommand(sql)) { cmd.CommandTimeout = int.MaxValue; using (var reader = cmd.ExecuteReader()) { while (reader.Read()) { loadContext.ItemChanges.AddLast(new ItemChange(reader)); } } } loadContext.Log.Info($"Merged loaded data: {(int)stopwatch.Elapsed.TotalSeconds}s"); }
protected virtual bool StageData(BulkLoadContext loadContext, BulkLoadSqlContext sqlContext, IEnumerable <BulkItem> items, out BulkItemsAndFieldsReader itemAndFieldReader) { var sql = sqlContext.GetEmbeddedSql(loadContext, "Sql.01.CreateTempTable.sql"); // Cleanup left-over staging tables. if (loadContext.StageDataWithoutProcessing) { sqlContext.DropStageTables(); } // Create temp table. // We don't use table valued parameters because we don't want to change the database schema. sqlContext.ExecuteSql(sql); // Load data into temp table. itemAndFieldReader = NewReader(items); if (!BulkCopyToTempTable(loadContext, sqlContext, itemAndFieldReader, NewFieldRulesReader(loadContext))) { return(false); } loadContext.OnDataStaged?.Invoke(loadContext); Event.RaiseEvent("bulkloader:datastaged", loadContext); return(true); }
protected virtual void LookupIds(BulkLoadContext loadContext, BulkLoadSqlContext sqlContext, BulkItemsAndFieldsReader itemAndFieldReader) { var lookupBlobsSql = sqlContext.GetEmbeddedSql(loadContext, "Sql.02.LookupBlobs.sql"); var lookupItemsSql = sqlContext.GetEmbeddedSql(loadContext, "Sql.03.LookupItems.sql"); var stopwatch = Stopwatch.StartNew(); if (loadContext.LookupBlobIds) { sqlContext.ExecuteSql(lookupBlobsSql); } if (loadContext.LookupItemIds) { // Using sql parameters resets temp tables. if (loadContext.Destination != null) { lookupItemsSql = sqlContext.ReplaceOneLineSqlStringParameter(lookupItemsSql, "@destinationPath", loadContext.Destination.ItemPath); lookupItemsSql = sqlContext.ReplaceOneLineSqlStringParameter(lookupItemsSql, "@destinationId", loadContext.Destination.ItemId.ToString("D")); } sqlContext.ExecuteSql(lookupItemsSql); } loadContext.Log.Info($"Looked up ids: {(int)stopwatch.Elapsed.TotalSeconds}s"); }
protected virtual bool BulkCopyToTempTable(BulkLoadContext loadContext, BulkLoadSqlContext sqlContext, BulkItemsAndFieldsReader itemAndFieldReader, FieldRulesReader fieldRulesReader) { var stopwatch = Stopwatch.StartNew(); // Bulk copy into temp tables, so that we don't have to buffer stuff, // blobs can give OutOfMemoryExceptions. using (var bulkCopy = new SqlBulkCopy(sqlContext.Connection)) { bulkCopy.BulkCopyTimeout = int.MaxValue; bulkCopy.EnableStreaming = true; bulkCopy.DestinationTableName = sqlContext.PostProcessSql(loadContext, "#BulkItemsAndFields"); try { bulkCopy.WriteToServer(itemAndFieldReader); } catch (Exception exception) { loadContext.StageFailed(Stage.Load, exception, $"Write to #BulkItemsAndFields failed with message: {exception.Message}"); return(false); } if (fieldRulesReader != null) { bulkCopy.DestinationTableName = sqlContext.PostProcessSql(loadContext, "#FieldRules"); try { bulkCopy.WriteToServer(fieldRulesReader); } catch (Exception exception) { loadContext.StageFailed(Stage.Load, exception, $"Write to #FieldRules failed with message: {exception}"); } } } loadContext.Log.Info($"Loaded data in database: {(int)stopwatch.Elapsed.TotalSeconds}s"); stopwatch.Restart(); return(true); }