private static async Task SaveChangesAsync(DbContext context, BulkConfig bulkConfig, Action <decimal> progress, CancellationToken cancellationToken, bool isAsync) { // 2 ways: // OPTION 1) iteration with Dic and Fast member // OPTION 2) using Node model (here setting FK still not implemented) int option = 1; if (bulkConfig == null) { bulkConfig = new BulkConfig { }; } if (bulkConfig.OnSaveChangesSetFK && bulkConfig.SetOutputIdentity == false) // When FK is set by DB then SetOutput is required { bulkConfig.SetOutputIdentity = true; } var entries = context.ChangeTracker.Entries(); var entriesGroupedByEntity = entries.GroupBy(a => new { EntityType = a.Entity.GetType(), a.State }, (entry, group) => new { entry.EntityType, EntityState = entry.State, Entities = group.Select(a => a.Entity).ToList() }); var entriesGroupedChanged = entriesGroupedByEntity.Where(a => EntityStateBulkMethodDict.Keys.Contains(a.EntityState) & a.Entities.Count >= 0); var entriesGroupedChangedSorted = entriesGroupedChanged.OrderBy(a => a.EntityState.ToString() != EntityState.Modified.ToString()).ToList(); if (entriesGroupedChangedSorted.Count == 0) { return; } if (isAsync) { await context.Database.OpenConnectionAsync(cancellationToken).ConfigureAwait(false); } else { context.Database.OpenConnection(); } var connection = context.GetUnderlyingConnection(bulkConfig); bool doExplicitCommit = false; if (context.Database.CurrentTransaction == null) { doExplicitCommit = true; } try { var transaction = context.Database.CurrentTransaction ?? context.Database.BeginTransaction(); if (option == 1) { Dictionary <string, Dictionary <string, FastProperty> > fastPropertyDicts = new Dictionary <string, Dictionary <string, FastProperty> >(); foreach (var entryGroup in entriesGroupedChangedSorted) { Type entityType = entryGroup.EntityType; entityType = (entityType.Namespace == "Castle.Proxies") ? entityType.BaseType : entityType; var entityModelType = context.Model.FindEntityType(entityType); var entityPropertyDict = new Dictionary <string, FastProperty>(); if (!fastPropertyDicts.ContainsKey(entityType.Name)) { var properties = entityModelType.GetProperties(); var navigationPropertiesInfo = entityModelType.GetNavigations().Select(x => x.PropertyInfo); foreach (var property in properties) { if (property.PropertyInfo != null) // skip Shadow Property { entityPropertyDict.Add(property.Name, new FastProperty(property.PropertyInfo)); } } foreach (var navigationPropertyInfo in navigationPropertiesInfo) { if (navigationPropertyInfo != null) { entityPropertyDict.Add(navigationPropertyInfo.Name, new FastProperty(navigationPropertyInfo)); } } fastPropertyDicts.Add(entityType.Name, entityPropertyDict); } else { entityPropertyDict = fastPropertyDicts[entityType.Name]; } if (bulkConfig.OnSaveChangesSetFK) { var navigations = entityModelType.GetNavigations().Where(x => !x.IsCollection && !x.TargetEntityType.IsOwned()); if (navigations.Count() > 0) { foreach (var navigation in navigations) { // when FK entity was not modified it will not be in Dict, but also FK is auto set so no need here if (fastPropertyDicts.ContainsKey(navigation.ClrType.Name)) // otherwise set it: { var parentPropertyDict = fastPropertyDicts[navigation.ClrType.Name]; var fkName = navigation.ForeignKey.Properties.FirstOrDefault().Name; var pkName = navigation.ForeignKey.PrincipalKey.Properties.FirstOrDefault().Name; foreach (var entity in entryGroup.Entities) { var parentEntity = entityPropertyDict[navigation.Name].Get(entity); var pkValue = parentPropertyDict[pkName].Get(parentEntity); entityPropertyDict[fkName].Set(entity, pkValue); } } } } } string methodName = EntityStateBulkMethodDict[entryGroup.EntityState].Key; if (isAsync) { await InvokeBulkMethod(context, entryGroup.Entities, entityType, methodName, bulkConfig, progress, cancellationToken, isAsync : true).ConfigureAwait(false); } else { InvokeBulkMethod(context, entryGroup.Entities, entityType, methodName, bulkConfig, progress, cancellationToken, isAsync: false).GetAwaiter().GetResult(); } } } else if (option == 2) { List <BulkMethodEntries> bulkMethodEntriesList = GetBulkMethodEntries(entries); foreach (var bulkMethod in bulkMethodEntriesList) { if (isAsync) { await InvokeBulkMethod(context, bulkMethod.Entries, bulkMethod.Type, bulkMethod.MethodName, bulkConfig, progress, cancellationToken, isAsync : true).ConfigureAwait(false); } else { InvokeBulkMethod(context, bulkMethod.Entries, bulkMethod.Type, bulkMethod.MethodName, bulkConfig, progress, cancellationToken, isAsync: false).GetAwaiter().GetResult(); } } } if (doExplicitCommit) { transaction.Commit(); context.ChangeTracker.AcceptAllChanges(); } } finally { if (doExplicitCommit) { if (isAsync) { await context.Database.CloseConnectionAsync().ConfigureAwait(false); } else { context.Database.CloseConnection(); } } } }
internal static SqliteConnection OpenAndGetSqliteConnection(DbContext context, BulkConfig bulkConfig) { var connection = bulkConfig.SqliteConnection ?? (SqliteConnection)context.Database.GetDbConnection(); if (connection.State != ConnectionState.Open) { connection.Open(); } return(connection); }
private static SqlBulkCopy GetSqlBulkCopy(SqlConnection sqlConnection, IDbContextTransaction transaction, BulkConfig config) { var sqlBulkCopyOptions = config.SqlBulkCopyOptions; if (transaction == null) { return(new SqlBulkCopy(sqlConnection, sqlBulkCopyOptions, null)); } else { var sqlTransaction = (SqlTransaction)transaction.GetUnderlyingTransaction(config); return(new SqlBulkCopy(sqlConnection, sqlBulkCopyOptions, sqlTransaction)); } }
public static void BulkInsert <T>(this DbContext context, IList <T> entities, BulkConfig bulkConfig = null) where T : class { DbContextBulkTransaction.Execute <T>(context, entities, OperationType.Insert, bulkConfig); }
public static void Execute(DbContext context, Type type, IList <object> entities, OperationType operationType, BulkConfig bulkConfig, Action <decimal> progress) { using (ActivitySources.StartExecuteActivity(operationType, entities.Count)) { if (operationType != OperationType.Truncate && entities.Count == 0) { return; } if (bulkConfig?.IncludeGraph == true) { DbContextBulkTransactionGraphUtil.ExecuteWithGraph(context, entities, operationType, bulkConfig, progress); } else { TableInfo tableInfo = TableInfo.CreateInstance(context, type, entities, operationType, bulkConfig); if (operationType == OperationType.Insert && !tableInfo.BulkConfig.SetOutputIdentity) { SqlBulkOperation.Insert(context, type, entities, tableInfo, progress); } else if (operationType == OperationType.Read) { SqlBulkOperation.Read(context, type, entities, tableInfo, progress); } else if (operationType == OperationType.Truncate) { SqlBulkOperation.Truncate(context, tableInfo); } else { SqlBulkOperation.Merge(context, type, entities, tableInfo, operationType, progress); } } } }
public static TableInfo CreateInstance(DbContext context, Type type, IList <object> entities, OperationType operationType, BulkConfig bulkConfig) { return(CreateInstance <object>(context, type, entities, operationType, bulkConfig)); }
public static TableInfo CreateInstance <T>(DbContext context, IList <T> entities, OperationType operationType, BulkConfig bulkConfig) { var tableInfo = new TableInfo { NumberOfEntities = entities.Count, BulkConfig = bulkConfig ?? new BulkConfig() }; bool isExplicitTransaction = context.Database.GetDbConnection().State == ConnectionState.Open; if (tableInfo.BulkConfig.UseTempDB == true && !isExplicitTransaction && (operationType != OperationType.Insert || tableInfo.BulkConfig.SetOutputIdentity)) { throw new InvalidOperationException("UseTempDB when set then BulkOperation has to be inside Transaction. More info in README of the library in GitHub."); // Otherwise throws exception: 'Cannot access destination table' (gets Dropped too early because transaction ends before operation is finished) } var isDeleteOperation = operationType == OperationType.Delete; tableInfo.LoadData <T>(context, isDeleteOperation); return(tableInfo); }
public static Task ExecuteAsync <T>(DbContext context, IList <T> entities, OperationType operationType, BulkConfig bulkConfig, Action <decimal> progress, CancellationToken cancellationToken) where T : class { if (entities.Count == 0) { return(Task.CompletedTask); } TableInfo tableInfo = TableInfo.CreateInstance(context, entities, operationType, bulkConfig); if (operationType == OperationType.Insert && !tableInfo.BulkConfig.SetOutputIdentity) { return(SqlBulkOperation.InsertAsync(context, entities, tableInfo, progress, cancellationToken)); } else if (operationType == OperationType.Read) { return(SqlBulkOperation.ReadAsync(context, entities, tableInfo, progress, cancellationToken)); } else { return(SqlBulkOperation.MergeAsync(context, entities, tableInfo, operationType, progress, cancellationToken)); } }
public static Task BulkReadAsync <T>(this DbContext context, IList <T> entities, BulkConfig bulkConfig = null, Action <decimal> progress = null) where T : class { return(DbContextBulkTransaction.ExecuteAsync(context, entities, OperationType.Read, bulkConfig, progress)); }
private static async Task ExecuteWithGraphAsync(DbContext context, IEnumerable <object> entities, OperationType operationType, BulkConfig bulkConfig, Action <decimal> progress, CancellationToken cancellationToken, bool isAsync) { if (operationType != OperationType.Insert && operationType != OperationType.InsertOrUpdate && operationType != OperationType.InsertOrUpdateDelete && operationType != OperationType.Update) { throw new InvalidBulkConfigException($"{nameof(BulkConfig)}.{nameof(BulkConfig.IncludeGraph)} only supports Insert or Update operations."); } // Sqlite bulk merge adapter does not support multiple objects of the same type with a zero value primary key if (SqlAdaptersMapping.GetDatabaseType(context) == DbServer.Sqlite) { throw new NotSupportedException("Sqlite is not currently supported due to its BulkInsert implementation."); } bulkConfig.PreserveInsertOrder = true; // Required for SetOutputIdentity ('true' is default but here explicitly assigned again in case it was changed to 'false' in BulkConfing) bulkConfig.SetOutputIdentity = true; // If this is set to false, won't be able to propogate new primary keys to the relationships // If this is set to false, wont' be able to support some code first model types as EFCore uses shadow properties when a relationship's foreign keys arent explicitly defined bulkConfig.EnableShadowProperties = true; var graphNodes = GraphUtil.GetTopologicallySortedGraph(context, entities); if (graphNodes == null) { return; } // Inserting an entity graph must be done within a transaction otherwise the database could end up in a bad state var hasExistingTransaction = context.Database.CurrentTransaction != null; var transaction = context.Database.CurrentTransaction ?? (isAsync ? await context.Database.BeginTransactionAsync() : context.Database.BeginTransaction()); try { // Group the graph nodes by entity type so we can merge them into the database in batches, in the correct order of dependency (topological order) var graphNodesGroupedByType = graphNodes.GroupBy(y => y.Entity.GetType()); foreach (var graphNodeGroup in graphNodesGroupedByType) { var entityClrType = graphNodeGroup.Key; var entityType = context.Model.FindEntityType(entityClrType); if (OwnedTypeUtil.IsOwnedInSameTableAsOwner(entityType)) { continue; } // It is possible the object graph contains duplicate entities (by primary key) but the entities are different object instances in memory. // This an happen when deserializing a nested JSON tree for example. So filter out the duplicates. var entitiesToAction = GetUniqueEntities(context, graphNodeGroup.Select(y => y.Entity)).ToList(); var tableInfo = TableInfo.CreateInstance(context, entityClrType, entitiesToAction, operationType, bulkConfig); if (isAsync) { await SqlBulkOperation.MergeAsync(context, entityClrType, entitiesToAction, tableInfo, operationType, progress, cancellationToken); } else { SqlBulkOperation.Merge(context, entityClrType, entitiesToAction, tableInfo, operationType, progress); } // Set the foreign keys for dependents so they may be inserted on the next loop var dependentsOfSameType = SetForeignKeysForDependentsAndYieldSameTypeDependents(context, entityClrType, graphNodeGroup).ToList(); // If there are any dependents of the same type (parent child relationship), then save those dependent entities again to commit the fk values if (dependentsOfSameType.Any()) { var dependentTableInfo = TableInfo.CreateInstance(context, entityClrType, dependentsOfSameType, operationType, bulkConfig); if (isAsync) { await SqlBulkOperation.MergeAsync(context, entityClrType, dependentsOfSameType, dependentTableInfo, operationType, progress, cancellationToken); } else { SqlBulkOperation.Merge(context, entityClrType, dependentsOfSameType, dependentTableInfo, operationType, progress); } } } if (hasExistingTransaction == false) { if (isAsync) { await transaction.CommitAsync(); } else { transaction.Commit(); } } } finally { if (hasExistingTransaction == false) { if (isAsync) { await transaction.DisposeAsync(); } else { transaction.Dispose(); } } } }
public static void Execute <T>(DbContext context, IList <T> entities, OperationType operationType, BulkConfig bulkConfig, Action <decimal> progress) where T : class { if (entities.Count == 0) { return; } TableInfo tableInfo = TableInfo.CreateInstance(context, entities, operationType, bulkConfig); if (operationType == OperationType.Insert && !tableInfo.BulkConfig.SetOutputIdentity) { SqlBulkOperation.Insert(context, entities, tableInfo, progress); } else if (operationType == OperationType.Read) { SqlBulkOperation.Read(context, entities, tableInfo, progress); } else { SqlBulkOperation.Merge(context, entities, tableInfo, operationType, progress); } }
public static async Task ExecuteWithGraphAsync(DbContext context, IEnumerable <object> entities, OperationType operationType, BulkConfig bulkConfig, Action <decimal> progress, CancellationToken cancellationToken) { await ExecuteWithGraphAsync(context, entities, operationType, bulkConfig, progress, cancellationToken, isAsync : true); }
public static void ExecuteWithGraph(DbContext context, IEnumerable <object> entities, OperationType operationType, BulkConfig bulkConfig, Action <decimal> progress) { ExecuteWithGraphAsync(context, entities, operationType, bulkConfig, progress, CancellationToken.None, isAsync: false).GetAwaiter().GetResult(); }
public static Task ExecuteAsync <T>(DbContext context, IList <T> entities, OperationType operationType, BulkConfig bulkConfig, Action <decimal> progress) where T : class { TableInfo tableInfo = TableInfo.CreateInstance(context, entities, operationType, bulkConfig); if (operationType == OperationType.Insert && !tableInfo.BulkConfig.SetOutputIdentity) { return(SqlBulkOperation.InsertAsync(context, entities, tableInfo, progress)); } else { return(SqlBulkOperation.MergeAsync(context, entities, tableInfo, operationType, progress)); } }
public static Task BulkInsertOrUpdateOrDeleteAsync <T>(this DbContext context, IList <T> entities, BulkConfig bulkConfig = null, Action <decimal> progress = null, Type type = null, CancellationToken cancellationToken = default) where T : class { return(DbContextBulkTransaction.ExecuteAsync(context, type, entities, OperationType.InsertOrUpdateDelete, bulkConfig, progress, cancellationToken)); }
public static DbTransaction GetUnderlyingTransaction(this IDbContextTransaction ctxTransaction, BulkConfig config) { var dbTransaction = ctxTransaction.GetDbTransaction(); if (config?.UnderlyingTransaction != null) { dbTransaction = config.UnderlyingTransaction(dbTransaction); } return(dbTransaction); }
public static TableInfo CreateInstance <T>(DbContext context, IList <T> entities, OperationType operationType, BulkConfig bulkConfig) { return(CreateInstance <T>(context, typeof(T), entities, operationType, bulkConfig)); }
public static Task BulkReadAsync(this DbContext context, Type entityType, IList <object> entities, BulkConfig bulkConfig = null, Action <decimal> progress = null, CancellationToken cancellationToken = default) { return(DbContextBulkTransaction.ExecuteAsync(context, entityType, entities, OperationType.Read, bulkConfig, progress, cancellationToken)); }
public static TableInfo CreateInstance <T>(DbContext context, IList <T> entities, OperationType operationType, BulkConfig bulkConfig) { var tableInfo = new TableInfo { NumberOfEntities = entities.Count, BulkConfig = bulkConfig ?? new BulkConfig() }; bool isExplicitTransaction = context.Database.GetDbConnection().State == ConnectionState.Open; if (tableInfo.BulkConfig.UseTempDB == true && !isExplicitTransaction && (operationType != OperationType.Insert || tableInfo.BulkConfig.SetOutputIdentity)) { tableInfo.BulkConfig.UseTempDB = false; // If BulkOps is not in explicit transaction then tempdb[#] can only be used with Insert, other Operations done with customTemp table. // Otherwise throws exception: 'Cannot access destination table' (gets Droped too early because transaction ends before operation is finished) } var isDeleteOperation = operationType == OperationType.Delete; tableInfo.LoadData <T>(context, isDeleteOperation); return(tableInfo); }
public static void BulkUpdate(this DbContext context, Type entityType, IList <object> entities, BulkConfig bulkConfig = null, Action <decimal> progress = null) { DbContextBulkTransaction.Execute(context, entityType, entities, OperationType.Update, bulkConfig, progress); }
public static TableInfo CreateInstance <T>(DbContext context, IList <T> entities, OperationType operationType, BulkConfig bulkConfig) { var tableInfo = new TableInfo { NumberOfEntities = entities.Count, BulkConfig = bulkConfig ?? new BulkConfig() }; if (operationType != OperationType.Insert) { tableInfo.BulkConfig.UseTempDB = false; // TempDB can only be used with Insert. // Other Operations done with customTemp table. // If using tempdb[#] throws exception: 'Cannot access destination table' (gets Droped too early, probably because transaction ends) } var isDeleteOperation = operationType == OperationType.Delete; tableInfo.LoadData <T>(context, isDeleteOperation); return(tableInfo); }
public static void Execute <T>(DbContext context, Type type, IList <T> entities, OperationType operationType, BulkConfig bulkConfig, Action <decimal> progress) where T : class { type ??= typeof(T); using (ActivitySources.StartExecuteActivity(operationType, entities.Count)) { if (entities.Count == 0 && operationType != OperationType.InsertOrUpdateOrDelete && operationType != OperationType.Truncate && operationType != OperationType.SaveChanges && (bulkConfig == null || bulkConfig.CustomSourceTableName == null)) { return; } if (operationType == OperationType.SaveChanges) { DbContextBulkTransactionSaveChanges.SaveChanges(context, bulkConfig, progress); return; } else if (bulkConfig?.IncludeGraph == true) { DbContextBulkTransactionGraphUtil.ExecuteWithGraph(context, entities, operationType, bulkConfig, progress); } else { TableInfo tableInfo = TableInfo.CreateInstance(context, type, entities, operationType, bulkConfig); if (operationType == OperationType.Insert && !tableInfo.BulkConfig.SetOutputIdentity && tableInfo.BulkConfig.CustomSourceTableName == null) { SqlBulkOperation.Insert(context, type, entities, tableInfo, progress); } else if (operationType == OperationType.Read) { SqlBulkOperation.Read(context, type, entities, tableInfo, progress); } else if (operationType == OperationType.Truncate) { SqlBulkOperation.Truncate(context, tableInfo); } else { SqlBulkOperation.Merge(context, type, entities, tableInfo, operationType, progress); } } } }
public static TableInfo CreateInstance <T>(DbContext context, IList <T> entities, OperationType operationType, BulkConfig bulkConfig) { var tableInfo = new TableInfo(); var isDeleteOperation = operationType == OperationType.Delete; tableInfo.NumberOfEntities = entities.Count; tableInfo.BulkConfig = bulkConfig ?? new BulkConfig(); tableInfo.LoadData <T>(context, isDeleteOperation); return(tableInfo); }
public static void BulkRead <T>(this DbContext context, IEnumerable <T> entities, BulkConfig bulkConfig = null, Action <decimal> progress = null) where T : class { DbContextBulkTransaction.Execute(context, entities.ToList(), OperationType.Read, bulkConfig, progress); }
public static async Task ExecuteAsync <T>(DbContext context, IList <T> entities, OperationType operationType, BulkConfig bulkConfig, Action <decimal> progress, CancellationToken cancellationToken) where T : class { using (ActivitySources.StartExecuteActivity(operationType, entities.Count)) { if (operationType != OperationType.Truncate && entities.Count == 0) { return; } if (bulkConfig?.IncludeGraph == true) { await DbContextBulkTransactionGraphUtil.ExecuteWithGraphAsync(context, entities, operationType, bulkConfig, progress, cancellationToken); } else { TableInfo tableInfo = TableInfo.CreateInstance(context, entities, operationType, bulkConfig); if (operationType == OperationType.Insert && !tableInfo.BulkConfig.SetOutputIdentity) { await SqlBulkOperation.InsertAsync(context, entities, tableInfo, progress, cancellationToken); } else if (operationType == OperationType.Read) { await SqlBulkOperation.ReadAsync(context, entities, tableInfo, progress, cancellationToken); } else if (operationType == OperationType.Truncate) { await SqlBulkOperation.TruncateAsync(context, tableInfo, cancellationToken); } else { await SqlBulkOperation.MergeAsync(context, entities, tableInfo, operationType, progress, cancellationToken); } } } }
// Async methods public static Task BulkInsertAsync <T>(this DbContext context, IEnumerable <T> entities, BulkConfig bulkConfig = null, Action <decimal> progress = null) where T : class { return(DbContextBulkTransaction.ExecuteAsync(context, entities.ToList(), OperationType.Insert, bulkConfig, progress)); }
internal static async Task <SqliteConnection> OpenAndGetSqliteConnectionAsync(DbContext context, BulkConfig bulkConfig, CancellationToken cancellationToken) { var connection = bulkConfig.SqliteConnection ?? (SqliteConnection)context.Database.GetDbConnection(); if (connection.State != ConnectionState.Open) { await connection.OpenAsync(cancellationToken).ConfigureAwait(false); } return(connection); }
// InsertOrUpdateOrDelete methods #region BulkInsertOrUpdateOrDelete public static void BulkInsertOrUpdateOrDelete <T>(this DbContext context, IList <T> entities, BulkConfig bulkConfig = null, Action <decimal> progress = null, Type type = null) where T : class { DbContextBulkTransaction.Execute(context, type, entities, OperationType.InsertOrUpdateDelete, bulkConfig, progress); }
internal static async Task <SqlConnection> OpenAndGetSqlConnectionAsync(DbContext context, BulkConfig config) { var connection = context.GetUnderlyingConnection(config); if (connection.State != ConnectionState.Open) { await connection.OpenAsync().ConfigureAwait(false); } return((SqlConnection)connection); }
public static async Task SaveChangesAsync(DbContext context, BulkConfig bulkConfig, Action <decimal> progress, CancellationToken cancellationToken) { await SaveChangesAsync(context, bulkConfig, progress, cancellationToken, isAsync : true).ConfigureAwait(false); }