public static void Execute <T>(DbContext context, IList <T> entities, OperationType operationType, BulkConfig bulkConfig, Action <decimal> progress) where T : class { if (operationType != OperationType.Truncate && entities.Count == 0) { return; } TableInfo tableInfo = TableInfo.CreateInstance(context, entities, operationType, bulkConfig); if (operationType == OperationType.Insert && !tableInfo.BulkConfig.SetOutputIdentity) { SqlBulkOperation.Insert(context, entities, tableInfo, progress); } else if (operationType == OperationType.Read) { SqlBulkOperation.Read(context, entities, tableInfo, progress); } else if (operationType == OperationType.Truncate) { SqlBulkOperation.Truncate(context, tableInfo); } else { SqlBulkOperation.Merge(context, entities, tableInfo, operationType, progress); } }
public static Task ExecuteAsync <T>(DbContext context, IList <T> entities, OperationType operationType, BulkConfig bulkConfig, Action <decimal> progress, CancellationToken cancellationToken) where T : class { if (operationType != OperationType.Truncate && entities.Count == 0) { return(Task.CompletedTask); } TableInfo tableInfo = TableInfo.CreateInstance(context, entities, operationType, bulkConfig); if (operationType == OperationType.Insert && !tableInfo.BulkConfig.SetOutputIdentity) { return(SqlBulkOperation.InsertAsync(context, entities, tableInfo, progress, cancellationToken)); } else if (operationType == OperationType.Read) { return(SqlBulkOperation.ReadAsync(context, entities, tableInfo, progress, cancellationToken)); } else if (operationType == OperationType.Truncate) { return(SqlBulkOperation.TruncateAsync(context, tableInfo)); } else { return(SqlBulkOperation.MergeAsync(context, entities, tableInfo, operationType, progress, cancellationToken)); } }
public static void Execute(DbContext context, Type type, IList <object> entities, OperationType operationType, BulkConfig bulkConfig, Action <decimal> progress) { using (ActivitySources.StartExecuteActivity(operationType, entities.Count)) { if (operationType != OperationType.Truncate && entities.Count == 0) { return; } TableInfo tableInfo = TableInfo.CreateInstance(context, type, entities, operationType, bulkConfig); if (operationType == OperationType.Insert && !tableInfo.BulkConfig.SetOutputIdentity) { SqlBulkOperation.Insert(context, type, entities, tableInfo, progress); } else if (operationType == OperationType.Read) { SqlBulkOperation.Read(context, type, entities, tableInfo, progress); } else if (operationType == OperationType.Truncate) { SqlBulkOperation.Truncate(context, tableInfo); } else { SqlBulkOperation.Merge(context, type, entities, tableInfo, operationType, progress); } } }
public static void Merge <T>(DbContext context, IList <T> entities, TableInfo tableInfo, OperationType operationType, Action <decimal> progress) where T : class { tableInfo.InsertToTempTable = true; tableInfo.CheckHasIdentity(context); context.Database.ExecuteSqlCommand(SqlQueryBuilder.CreateTableCopy(tableInfo.FullTableName, tableInfo.FullTempTableName)); if (tableInfo.BulkConfig.SetOutputIdentity) { context.Database.ExecuteSqlCommand(SqlQueryBuilder.CreateTableCopy(tableInfo.FullTableName, tableInfo.FullTempOutputTableName)); } try { SqlBulkOperation.Insert <T>(context, entities, tableInfo, progress); context.Database.ExecuteSqlCommand(SqlQueryBuilder.MergeTable(tableInfo, operationType)); context.Database.ExecuteSqlCommand(SqlQueryBuilder.DropTable(tableInfo.FullTempTableName)); if (tableInfo.BulkConfig.SetOutputIdentity) { tableInfo.UpdateOutputIdentity(context, entities); context.Database.ExecuteSqlCommand(SqlQueryBuilder.DropTable(tableInfo.FullTempOutputTableName)); } } catch (Exception ex) { if (tableInfo.BulkConfig.SetOutputIdentity) { context.Database.ExecuteSqlCommand(SqlQueryBuilder.DropTable(tableInfo.FullTempOutputTableName)); } context.Database.ExecuteSqlCommand(SqlQueryBuilder.DropTable(tableInfo.FullTempTableName)); throw ex; } }
public static Task ExecuteAsync(DbContext context, Type type, IList <object> entities, OperationType operationType, BulkConfig bulkConfig, Action <decimal> progress, CancellationToken cancellationToken) { using (ActivitySources.StartExecuteActivity(operationType, entities.Count)) { if (operationType != OperationType.Truncate && entities.Count == 0) { return(Task.CompletedTask); } TableInfo tableInfo = TableInfo.CreateInstance(context, type, entities, operationType, bulkConfig); if (operationType == OperationType.Insert && !tableInfo.BulkConfig.SetOutputIdentity) { return(SqlBulkOperation.InsertAsync(context, type, entities, tableInfo, progress, cancellationToken)); } else if (operationType == OperationType.Read) { return(SqlBulkOperation.ReadAsync(context, type, entities, tableInfo, progress, cancellationToken)); } else if (operationType == OperationType.Truncate) { return(SqlBulkOperation.TruncateAsync(context, tableInfo, cancellationToken)); } else { return(SqlBulkOperation.MergeAsync(context, type, entities, tableInfo, operationType, progress, cancellationToken)); } } }
public static void Execute <T>(DbContext context, IList <T> entities, OperationType operationType, BulkConfig bulkConfig, Action <decimal> progress) where T : class { TableInfo tableInfo = TableInfo.CreateInstance(context, entities, operationType, bulkConfig); if (operationType == OperationType.Insert && !tableInfo.BulkConfig.SetOutputIdentity) { SqlBulkOperation.Insert <T>(context, entities, tableInfo, progress); } else { SqlBulkOperation.Merge <T>(context, entities, tableInfo, operationType, progress); } }
public static Task ExecuteAsync <T>(DbContext context, IList <T> entities, OperationType operationType, BulkConfig bulkConfig, Action <decimal> progress) where T : class { TableInfo tableInfo = TableInfo.CreateInstance(context, entities, operationType, bulkConfig); if (operationType == OperationType.Insert && !tableInfo.BulkConfig.SetOutputIdentity) { return(SqlBulkOperation.InsertAsync(context, entities, tableInfo, progress)); } else if (operationType == OperationType.Read) { return(SqlBulkOperation.ReadAsync(context, entities, tableInfo, progress)); } else { return(SqlBulkOperation.MergeAsync(context, entities, tableInfo, operationType, progress)); } }
public static void Execute <T>(DbContext context, Type type, IList <T> entities, OperationType operationType, BulkConfig bulkConfig, Action <decimal> progress) where T : class { type ??= typeof(T); using (ActivitySources.StartExecuteActivity(operationType, entities.Count)) { if (entities.Count == 0 && operationType != OperationType.InsertOrUpdateOrDelete && operationType != OperationType.Truncate && operationType != OperationType.SaveChanges && (bulkConfig == null || bulkConfig.CustomSourceTableName == null)) { return; } if (operationType == OperationType.SaveChanges) { DbContextBulkTransactionSaveChanges.SaveChanges(context, bulkConfig, progress); return; } else if (bulkConfig?.IncludeGraph == true) { DbContextBulkTransactionGraphUtil.ExecuteWithGraph(context, entities, operationType, bulkConfig, progress); } else { TableInfo tableInfo = TableInfo.CreateInstance(context, type, entities, operationType, bulkConfig); if (operationType == OperationType.Insert && !tableInfo.BulkConfig.SetOutputIdentity && tableInfo.BulkConfig.CustomSourceTableName == null) { SqlBulkOperation.Insert(context, type, entities, tableInfo, progress); } else if (operationType == OperationType.Read) { SqlBulkOperation.Read(context, type, entities, tableInfo, progress); } else if (operationType == OperationType.Truncate) { SqlBulkOperation.Truncate(context, tableInfo); } else { SqlBulkOperation.Merge(context, type, entities, tableInfo, operationType, progress); } } } }
public static void Merge <T>(DbContext context, IList <T> entities, TableInfo tableInfo, OperationType operationType) where T : class { tableInfo.InsertToTempTable = true; tableInfo.CheckHasIdentity(context); context.Database.ExecuteSqlCommand(SqlQueryBuilder.CreateTableCopy(tableInfo.FullTableName, tableInfo.FullTempTableName)); if (tableInfo.BulkConfig.SetOutputIdentity) { context.Database.ExecuteSqlCommand(SqlQueryBuilder.CreateTableCopy(tableInfo.FullTableName, tableInfo.FullTempOutputTableName)); } try { SqlBulkOperation.Insert <T>(context, entities, tableInfo); context.Database.ExecuteSqlCommand(SqlQueryBuilder.MergeTable(tableInfo, operationType)); context.Database.ExecuteSqlCommand(SqlQueryBuilder.DropTable(tableInfo.FullTempTableName)); if (tableInfo.BulkConfig.SetOutputIdentity) { var entitiesWithOutputIdentity = context.Set <T>().FromSql(SqlQueryBuilder.SelectFromTable(tableInfo.FullTempOutputTableName, tableInfo.PrimaryKeyFormated)).ToList(); if (tableInfo.BulkConfig.PreserveInsertOrder) // Updates PK in entityList { Type type = typeof(T); var accessor = TypeAccessor.Create(type); for (int i = 0; i < tableInfo.NumberOfEntities; i++) { accessor[entities[i], tableInfo.PrimaryKey] = accessor[entitiesWithOutputIdentity[i], tableInfo.PrimaryKey]; } } else // Clears entityList and then refill it with loaded entites from Db { entities.Clear(); ((List <T>)entities).AddRange(entitiesWithOutputIdentity); } context.Database.ExecuteSqlCommand(SqlQueryBuilder.DropTable(tableInfo.FullTempOutputTableName)); } } catch (Exception ex) { if (tableInfo.BulkConfig.SetOutputIdentity) { context.Database.ExecuteSqlCommand(SqlQueryBuilder.DropTable(tableInfo.FullTempOutputTableName)); } context.Database.ExecuteSqlCommand(SqlQueryBuilder.DropTable(tableInfo.FullTempTableName)); throw ex; } }
public static void Execute <T>(DbContext context, IList <T> entities, OperationType operationType, BulkConfig bulkConfig) where T : class { var tableInfo = new TableInfo(); var isDeleteOperation = operationType == OperationType.Delete; tableInfo.NumberOfEntities = entities.Count; tableInfo.LoadData <T>(context, isDeleteOperation); tableInfo.BulkConfig = bulkConfig ?? new BulkConfig(); if (operationType == OperationType.Insert && !tableInfo.BulkConfig.SetOutputIdentity) { SqlBulkOperation.Insert <T>(context, entities, tableInfo); } else { SqlBulkOperation.Merge <T>(context, entities, tableInfo, operationType); } }
public void SetSqlBulkCopyConfig <T>(SqlBulkCopy sqlBulkCopy, IList <T> entities, bool setColumnMapping, Action <decimal> progress) { sqlBulkCopy.DestinationTableName = InsertToTempTable ? FullTempTableName : FullTableName; sqlBulkCopy.BatchSize = BulkConfig.BatchSize; sqlBulkCopy.NotifyAfter = BulkConfig.NotifyAfter ?? BulkConfig.BatchSize; sqlBulkCopy.SqlRowsCopied += (sender, e) => { progress?.Invoke(SqlBulkOperation.GetProgress(entities.Count, e.RowsCopied)); // round to 4 decimal places }; sqlBulkCopy.BulkCopyTimeout = BulkConfig.BulkCopyTimeout ?? sqlBulkCopy.BulkCopyTimeout; sqlBulkCopy.EnableStreaming = BulkConfig.EnableStreaming; if (setColumnMapping) { foreach (var element in PropertyColumnNamesDict) { sqlBulkCopy.ColumnMappings.Add(element.Key, element.Value); } } }
public static async Task ExecuteAsync <T>(DbContext context, Type type, IList <T> entities, OperationType operationType, BulkConfig bulkConfig, Action <decimal> progress, CancellationToken cancellationToken = default) where T : class { type ??= typeof(T); using (ActivitySources.StartExecuteActivity(operationType, entities.Count)) { if (entities.Count == 0 && operationType != OperationType.InsertOrUpdateOrDelete && operationType != OperationType.Truncate && operationType != OperationType.SaveChanges) { return; } if (operationType == OperationType.SaveChanges) { await DbContextBulkTransactionSaveChanges.SaveChangesAsync(context, bulkConfig, progress, cancellationToken); } else if (bulkConfig?.IncludeGraph == true) { await DbContextBulkTransactionGraphUtil.ExecuteWithGraphAsync(context, entities, operationType, bulkConfig, progress, cancellationToken); } else { TableInfo tableInfo = TableInfo.CreateInstance(context, type, entities, operationType, bulkConfig); if (operationType == OperationType.Insert && !tableInfo.BulkConfig.SetOutputIdentity) { await SqlBulkOperation.InsertAsync(context, type, entities, tableInfo, progress, cancellationToken); } else if (operationType == OperationType.Read) { await SqlBulkOperation.ReadAsync(context, type, entities, tableInfo, progress, cancellationToken); } else if (operationType == OperationType.Truncate) { await SqlBulkOperation.TruncateAsync(context, tableInfo, cancellationToken); } else { await SqlBulkOperation.MergeAsync(context, type, entities, tableInfo, operationType, progress, cancellationToken); } } } }
public static async Task MergeAsync <T>(DbContext context, IList <T> entities, TableInfo tableInfo, OperationType operationType, Action <decimal> progress) where T : class { tableInfo.InsertToTempTable = true; await tableInfo.CheckHasIdentityAsync(context).ConfigureAwait(false); await context.Database.ExecuteSqlCommandAsync(SqlQueryBuilder.CreateTableCopy(tableInfo.FullTableName, tableInfo.FullTempTableName)).ConfigureAwait(false); if (tableInfo.BulkConfig.SetOutputIdentity) { await context.Database.ExecuteSqlCommandAsync(SqlQueryBuilder.CreateTableCopy(tableInfo.FullTableName, tableInfo.FullTempOutputTableName)).ConfigureAwait(false); } try { await SqlBulkOperation.InsertAsync <T>(context, entities, tableInfo, progress).ConfigureAwait(false); await context.Database.ExecuteSqlCommandAsync(SqlQueryBuilder.MergeTable(tableInfo, operationType)).ConfigureAwait(false); await context.Database.ExecuteSqlCommandAsync(SqlQueryBuilder.DropTable(tableInfo.FullTempTableName)).ConfigureAwait(false); if (tableInfo.BulkConfig.SetOutputIdentity) { tableInfo.UpdateOutputIdentity(context, entities); await context.Database.ExecuteSqlCommandAsync(SqlQueryBuilder.DropTable(tableInfo.FullTempOutputTableName)).ConfigureAwait(false); } } catch (Exception ex) { if (tableInfo.BulkConfig.SetOutputIdentity) { await context.Database.ExecuteSqlCommandAsync(SqlQueryBuilder.DropTable(tableInfo.FullTempOutputTableName)).ConfigureAwait(false); } await context.Database.ExecuteSqlCommandAsync(SqlQueryBuilder.DropTable(tableInfo.FullTempTableName)).ConfigureAwait(false); throw ex; } }
public static void Execute <T>(DbContext context, IList <T> entities, OperationType operationType, BulkConfig bulkConfig, Action <decimal> progress) where T : class { using (ActivitySources.StartExecuteActivity(operationType, entities.Count)) { if (operationType != OperationType.Truncate && entities.Count == 0) { return; } if (bulkConfig?.IncludeGraph == true) { DbContextBulkTransactionGraphUtil.ExecuteWithGraph(context, entities, operationType, bulkConfig, progress); } else { TableInfo tableInfo = TableInfo.CreateInstance(context, entities, operationType, bulkConfig); if (operationType == OperationType.Insert && !tableInfo.BulkConfig.SetOutputIdentity) { SqlBulkOperation.Insert(context, entities, tableInfo, progress); } else if (operationType == OperationType.Read) { SqlBulkOperation.Read(context, entities, tableInfo, progress); } else if (operationType == OperationType.Truncate) { SqlBulkOperation.Truncate(context, tableInfo); } else { SqlBulkOperation.Merge(context, entities, tableInfo, operationType, progress); } } } }
public static async Task ExecuteAsync(DbContext context, Type type, IList <object> entities, OperationType operationType, BulkConfig bulkConfig, Action <decimal> progress, CancellationToken cancellationToken) { using (ActivitySources.StartExecuteActivity(operationType, entities.Count)) { if (operationType != OperationType.Truncate && entities.Count == 0) { return; } if (bulkConfig?.IncludeGraph == true) { await DbContextBulkTransactionGraphUtil.ExecuteWithGraphAsync(context, entities, operationType, bulkConfig, progress, cancellationToken); } else { TableInfo tableInfo = TableInfo.CreateInstance(context, type, entities, operationType, bulkConfig); if (operationType == OperationType.Insert && !tableInfo.BulkConfig.SetOutputIdentity) { await SqlBulkOperation.InsertAsync(context, type, entities, tableInfo, progress, cancellationToken); } else if (operationType == OperationType.Read) { await SqlBulkOperation.ReadAsync(context, type, entities, tableInfo, progress, cancellationToken); } else if (operationType == OperationType.Truncate) { await SqlBulkOperation.TruncateAsync(context, tableInfo, cancellationToken); } else { await SqlBulkOperation.MergeAsync(context, type, entities, tableInfo, operationType, progress, cancellationToken); } } } }
private static async Task ExecuteWithGraphAsync_Impl(DbContext context, IEnumerable <object> entities, OperationType operationType, BulkConfig bulkConfig, Action <decimal> progress, CancellationToken?cancellationToken, bool isAsync = true) { if (operationType != OperationType.Insert && operationType != OperationType.InsertOrUpdate && operationType != OperationType.InsertOrUpdateDelete && operationType != OperationType.Update) { throw new InvalidBulkConfigException($"{nameof(BulkConfig)}.{nameof(BulkConfig.IncludeGraph)} only supports Insert or Update operations."); } // Sqlite bulk merge adapter does not support multiple objects of the same type with a zero value primary key if (SqlAdaptersMapping.GetDatabaseType(context) == DbServer.Sqlite) { throw new NotSupportedException("Sqlite is not currently supported due to its BulkInsert implementation."); } // If this is set to false, won't be able to propogate new primary keys to the relationships bulkConfig.SetOutputIdentity = true; // If this is set to false, wont' be able to support some code first model types as EFCore uses shadow properties when a relationship's foreign keys arent explicitly defined bulkConfig.EnableShadowProperties = true; var rootGraphItems = GraphUtil.GetOrderedGraph(context, entities); if (rootGraphItems == null) { return; } // Inserting an entity graph must be done within a transaction otherwise the database could end up in a bad state var hasExistingTransaction = context.Database.CurrentTransaction != null; var transaction = context.Database.CurrentTransaction ?? (isAsync ? await context.Database.BeginTransactionAsync() : context.Database.BeginTransaction()); try { foreach (var actionGraphItem in rootGraphItems) { var entitiesToAction = GetUniqueEntities(context, actionGraphItem).ToList(); var tableInfo = TableInfo.CreateInstance(context, actionGraphItem.EntityClrType, entitiesToAction, operationType, bulkConfig); if (isAsync) { await SqlBulkOperation.MergeAsync(context, actionGraphItem.EntityClrType, entitiesToAction, tableInfo, operationType, progress, cancellationToken.Value); } else { SqlBulkOperation.Merge(context, actionGraphItem.EntityClrType, entitiesToAction, tableInfo, operationType, progress); } // Loop through the dependants and update their foreign keys with the PK values of the just inserted / merged entities foreach (var graphEntity in actionGraphItem.Entities) { var entity = graphEntity.Entity; var parentEntity = graphEntity.ParentEntity; // If the parent entity is null its the root type of the object graph. if (parentEntity is null) { foreach (var navigation in actionGraphItem.Relationships) { // If this relationship requires the parents value to exist if (navigation.ParentNavigation.IsDependentToPrincipal() == false) { foreach (var navGraphEntity in navigation.Entities) { if (navGraphEntity.ParentEntity != entity) { continue; } SetForeignKeyForRelationship(context, navigation.ParentNavigation, navGraphEntity.Entity, entity); } } } } else { var navigation = actionGraphItem.ParentNavigation; if (navigation.IsDependentToPrincipal()) { SetForeignKeyForRelationship(context, navigation, dependent: parentEntity, principal: entity); } else { SetForeignKeyForRelationship(context, navigation, dependent: entity, principal: parentEntity); } } } } if (hasExistingTransaction == false) { if (isAsync) { await transaction.CommitAsync(); } else { transaction.Commit(); } } } finally { if (hasExistingTransaction == false) { if (isAsync) { await transaction.DisposeAsync(); } else { transaction.Dispose(); } } } }
private static async Task ExecuteWithGraphAsync(DbContext context, IEnumerable <object> entities, OperationType operationType, BulkConfig bulkConfig, Action <decimal> progress, CancellationToken cancellationToken, bool isAsync) { if (operationType != OperationType.Insert && operationType != OperationType.InsertOrUpdate && operationType != OperationType.InsertOrUpdateDelete && operationType != OperationType.Update) { throw new InvalidBulkConfigException($"{nameof(BulkConfig)}.{nameof(BulkConfig.IncludeGraph)} only supports Insert or Update operations."); } // Sqlite bulk merge adapter does not support multiple objects of the same type with a zero value primary key if (SqlAdaptersMapping.GetDatabaseType(context) == DbServer.Sqlite) { throw new NotSupportedException("Sqlite is not currently supported due to its BulkInsert implementation."); } bulkConfig.PreserveInsertOrder = true; // Required for SetOutputIdentity ('true' is default but here explicitly assigned again in case it was changed to 'false' in BulkConfing) bulkConfig.SetOutputIdentity = true; // If this is set to false, won't be able to propogate new primary keys to the relationships // If this is set to false, wont' be able to support some code first model types as EFCore uses shadow properties when a relationship's foreign keys arent explicitly defined bulkConfig.EnableShadowProperties = true; var graphNodes = GraphUtil.GetTopologicallySortedGraph(context, entities); if (graphNodes == null) { return; } // Inserting an entity graph must be done within a transaction otherwise the database could end up in a bad state var hasExistingTransaction = context.Database.CurrentTransaction != null; var transaction = context.Database.CurrentTransaction ?? (isAsync ? await context.Database.BeginTransactionAsync() : context.Database.BeginTransaction()); try { // Group the graph nodes by entity type so we can merge them into the database in batches, in the correct order of dependency (topological order) var graphNodesGroupedByType = graphNodes.GroupBy(y => y.Entity.GetType()); foreach (var graphNodeGroup in graphNodesGroupedByType) { // It is possible the object graph contains duplicate entities (by primary key) but the entities are different object instances in memory. // This an happen when deserializing a nested JSON tree for example. So filter out the duplicates. var entitiesToAction = GetUniqueEntities(context, graphNodeGroup.Select(y => y.Entity)).ToList(); var entityClrType = graphNodeGroup.Key; var tableInfo = TableInfo.CreateInstance(context, entityClrType, entitiesToAction, operationType, bulkConfig); if (isAsync) { await SqlBulkOperation.MergeAsync(context, entityClrType, entitiesToAction, tableInfo, operationType, progress, cancellationToken); } else { SqlBulkOperation.Merge(context, entityClrType, entitiesToAction, tableInfo, operationType, progress); } // Set the foreign keys for dependents so they may be inserted on the next loop var dependentsOfSameType = SetForeignKeysForDependentsAndYieldSameTypeDependents(context, entityClrType, graphNodeGroup).ToList(); // If there are any dependents of the same type (parent child relationship), then save those dependent entities again to commit the fk values if (dependentsOfSameType.Any()) { var dependentTableInfo = TableInfo.CreateInstance(context, entityClrType, dependentsOfSameType, operationType, bulkConfig); if (isAsync) { await SqlBulkOperation.MergeAsync(context, entityClrType, dependentsOfSameType, dependentTableInfo, operationType, progress, cancellationToken); } else { SqlBulkOperation.Merge(context, entityClrType, dependentsOfSameType, dependentTableInfo, operationType, progress); } } } if (hasExistingTransaction == false) { if (isAsync) { await transaction.CommitAsync(); } else { transaction.Commit(); } } } finally { if (hasExistingTransaction == false) { if (isAsync) { await transaction.DisposeAsync(); } else { transaction.Dispose(); } } } }