protected virtual bool BulkCopyToTempTable(BulkLoadContext loadContext, BulkLoadSqlContext sqlContext, BulkItemsAndFieldsReader itemAndFieldReader, FieldRulesReader fieldRulesReader) { var stopwatch = Stopwatch.StartNew(); // Bulk copy into temp tables, so that we don't have to buffer stuff, // blobs can give OutOfMemoryExceptions. using (var bulkCopy = new SqlBulkCopy(sqlContext.Connection)) { bulkCopy.BulkCopyTimeout = int.MaxValue; bulkCopy.EnableStreaming = true; bulkCopy.DestinationTableName = sqlContext.PostProcessSql(loadContext, "#BulkItemsAndFields"); try { bulkCopy.WriteToServer(itemAndFieldReader); } catch (Exception exception) { loadContext.StageFailed(Stage.Load, exception, $"Write to #BulkItemsAndFields failed with message: {exception.Message}"); return(false); } if (fieldRulesReader != null) { bulkCopy.DestinationTableName = sqlContext.PostProcessSql(loadContext, "#FieldRules"); try { bulkCopy.WriteToServer(fieldRulesReader); } catch (Exception exception) { loadContext.StageFailed(Stage.Load, exception, $"Write to #FieldRules failed with message: {exception}"); } } } loadContext.Log.Info($"Loaded data in database: {(int)stopwatch.Elapsed.TotalSeconds}s"); stopwatch.Restart(); return(true); }
public virtual void LoadItems(BulkLoadContext context, IEnumerable <BulkLoadItem> items) { if (context == null) { throw new ArgumentNullException(nameof(context)); } if (items == null) { throw new ArgumentNullException(nameof(items)); } items = OnItemProcessing.Execute(items, (p, itms) => p.Process(context, itms)); var db = Factory.GetDatabase(context.Database, true); var connectionString = ConfigurationManager.ConnectionStrings[db.ConnectionStringName].ConnectionString; var infoMessageHandler = new SqlInfoMessageEventHandler((s, e) => OnSqlInfoMessage(context, s, e)); using (var conn = new SqlConnection(connectionString)) { var sqlContext = new BulkLoadSqlContext(conn, typeof(BulkLoader)); SqlTransaction transaction = null; try { conn.InfoMessage += infoMessageHandler; conn.Open(); BulkItemsAndFieldsReader itemAndFieldReader; if (!StageData(context, sqlContext, items, out itemAndFieldReader)) { return; } if (context.StageDataWithoutProcessing) { context.Log.Info("Data to import is staged in database, no processing done."); context.StageSucceeded(Stage.Load); return; } if (itemAndFieldReader.ReadItemCount > 0) { LookupIds(context, sqlContext, itemAndFieldReader); if (!ValidateAndPrepareData(context, sqlContext)) { return; } sqlContext.Transaction = transaction = conn.BeginTransaction(); MergeData(context, sqlContext, itemAndFieldReader); } OnTransactionCommitting.Execute(p => p.Process(context, sqlContext)); // After this point, there's no use in keeping the transaction arround, // because we cannot sync everything transactionally (e.g. indexes, publshing, ...) // Be aware that after this point the process may halt and not everything is consistent. // We mitigate this inconsistency with crash recovery, see below. transaction?.Commit(); transaction = null; // Allow clearing caches before raising event so that reading the item API in event will result in fresh reads. OnItemsLoading.Execute(p => p.Process(context, sqlContext, context.ItemChanges)); // Databases are now entirely in sync. context.OnDataLoaded?.Invoke(context); Event.RaiseEvent("bulkloader:dataloaded", context); // Execute post processors.y. var itemChanges = GetChanges(context, sqlContext); OnItemsLoaded.Execute(p => p.Process(context, sqlContext, itemChanges)); context.StageSucceeded(Stage.Load); } catch (Exception ex) { transaction?.Rollback(); context.StageFailed(Stage.Load, ex.Message); } finally { conn.InfoMessage -= infoMessageHandler; } } }