private static async Task SaveChangesAsync(DbContext context, BulkConfig bulkConfig, Action <decimal> progress, CancellationToken cancellationToken, bool isAsync)
{
// 2 ways:
// OPTION 1) iteration with Dic and Fast member
// OPTION 2) using Node model (here setting FK still not implemented)
int option = 1;
if (bulkConfig == null)
{
bulkConfig = new BulkConfig {
};
}
if (bulkConfig.OnSaveChangesSetFK && bulkConfig.SetOutputIdentity == false) // When FK is set by DB then SetOutput is required
{
bulkConfig.SetOutputIdentity = true;
}
var entries = context.ChangeTracker.Entries();
var entriesGroupedByEntity = entries.GroupBy(a => new { EntityType = a.Entity.GetType(), a.State },
(entry, group) => new { entry.EntityType, EntityState = entry.State, Entities = group.Select(a => a.Entity).ToList() });
var entriesGroupedChanged = entriesGroupedByEntity.Where(a => EntityStateBulkMethodDict.Keys.Contains(a.EntityState) & a.Entities.Count >= 0);
var entriesGroupedChangedSorted = entriesGroupedChanged.OrderBy(a => a.EntityState.ToString() != EntityState.Modified.ToString()).ToList();
if (entriesGroupedChangedSorted.Count == 0)
{
return;
}
if (isAsync)
{
await context.Database.OpenConnectionAsync(cancellationToken).ConfigureAwait(false);
}
else
{
context.Database.OpenConnection();
}
var connection = context.GetUnderlyingConnection(bulkConfig);
bool doExplicitCommit = false;
if (context.Database.CurrentTransaction == null)
{
doExplicitCommit = true;
}
try
{
var transaction = context.Database.CurrentTransaction ?? context.Database.BeginTransaction();
if (option == 1)
{
Dictionary <string, Dictionary <string, FastProperty> > fastPropertyDicts = new Dictionary <string, Dictionary <string, FastProperty> >();
foreach (var entryGroup in entriesGroupedChangedSorted)
{
Type entityType = entryGroup.EntityType;
entityType = (entityType.Namespace == "Castle.Proxies") ? entityType.BaseType : entityType;
var entityModelType = context.Model.FindEntityType(entityType);
var entityPropertyDict = new Dictionary <string, FastProperty>();
if (!fastPropertyDicts.ContainsKey(entityType.Name))
{
var properties = entityModelType.GetProperties();
var navigationPropertiesInfo = entityModelType.GetNavigations().Select(x => x.PropertyInfo);
foreach (var property in properties)
{
if (property.PropertyInfo != null) // skip Shadow Property
{
entityPropertyDict.Add(property.Name, new FastProperty(property.PropertyInfo));
}
}
foreach (var navigationPropertyInfo in navigationPropertiesInfo)
{
if (navigationPropertyInfo != null)
{
entityPropertyDict.Add(navigationPropertyInfo.Name, new FastProperty(navigationPropertyInfo));
}
}
fastPropertyDicts.Add(entityType.Name, entityPropertyDict);
}
else
{
entityPropertyDict = fastPropertyDicts[entityType.Name];
}
if (bulkConfig.OnSaveChangesSetFK)
{
var navigations = entityModelType.GetNavigations().Where(x => !x.IsCollection && !x.TargetEntityType.IsOwned());
if (navigations.Count() > 0)
{
foreach (var navigation in navigations)
{
// when FK entity was not modified it will not be in Dict, but also FK is auto set so no need here
if (fastPropertyDicts.ContainsKey(navigation.ClrType.Name)) // otherwise set it:
{
var parentPropertyDict = fastPropertyDicts[navigation.ClrType.Name];
var fkName = navigation.ForeignKey.Properties.FirstOrDefault().Name;
var pkName = navigation.ForeignKey.PrincipalKey.Properties.FirstOrDefault().Name;
foreach (var entity in entryGroup.Entities)
{
var parentEntity = entityPropertyDict[navigation.Name].Get(entity);
var pkValue = parentPropertyDict[pkName].Get(parentEntity);
entityPropertyDict[fkName].Set(entity, pkValue);
}
}
}
}
}
string methodName = EntityStateBulkMethodDict[entryGroup.EntityState].Key;
if (isAsync)
{
await InvokeBulkMethod(context, entryGroup.Entities, entityType, methodName, bulkConfig, progress, cancellationToken, isAsync : true).ConfigureAwait(false);
}
else
{
InvokeBulkMethod(context, entryGroup.Entities, entityType, methodName, bulkConfig, progress, cancellationToken, isAsync: false).GetAwaiter().GetResult();
}
}
}
else if (option == 2)
{
List <BulkMethodEntries> bulkMethodEntriesList = GetBulkMethodEntries(entries);
foreach (var bulkMethod in bulkMethodEntriesList)
{
if (isAsync)
{
await InvokeBulkMethod(context, bulkMethod.Entries, bulkMethod.Type, bulkMethod.MethodName, bulkConfig, progress, cancellationToken, isAsync : true).ConfigureAwait(false);
}
else
{
InvokeBulkMethod(context, bulkMethod.Entries, bulkMethod.Type, bulkMethod.MethodName, bulkConfig, progress, cancellationToken, isAsync: false).GetAwaiter().GetResult();
}
}
}
if (doExplicitCommit)
{
transaction.Commit();
context.ChangeTracker.AcceptAllChanges();
}
}
finally
{
if (doExplicitCommit)
{
if (isAsync)
{
await context.Database.CloseConnectionAsync().ConfigureAwait(false);
}
else
{
context.Database.CloseConnection();
}
}
}
}
internal static SqliteConnection OpenAndGetSqliteConnection(DbContext context, BulkConfig bulkConfig)
{
var connection = bulkConfig.SqliteConnection ?? (SqliteConnection)context.Database.GetDbConnection();
if (connection.State != ConnectionState.Open)
{
connection.Open();
}
return(connection);
}
private static SqlBulkCopy GetSqlBulkCopy(SqlConnection sqlConnection, IDbContextTransaction transaction, BulkConfig config)
{
var sqlBulkCopyOptions = config.SqlBulkCopyOptions;
if (transaction == null)
{
return(new SqlBulkCopy(sqlConnection, sqlBulkCopyOptions, null));
}
else
{
var sqlTransaction = (SqlTransaction)transaction.GetUnderlyingTransaction(config);
return(new SqlBulkCopy(sqlConnection, sqlBulkCopyOptions, sqlTransaction));
}
}
public static void BulkInsert <T>(this DbContext context, IList <T> entities, BulkConfig bulkConfig = null) where T : class
{
DbContextBulkTransaction.Execute <T>(context, entities, OperationType.Insert, bulkConfig);
}
public static void Execute(DbContext context, Type type, IList <object> entities, OperationType operationType, BulkConfig bulkConfig, Action <decimal> progress)
{
using (ActivitySources.StartExecuteActivity(operationType, entities.Count))
{
if (operationType != OperationType.Truncate && entities.Count == 0)
{
return;
}
if (bulkConfig?.IncludeGraph == true)
{
DbContextBulkTransactionGraphUtil.ExecuteWithGraph(context, entities, operationType, bulkConfig, progress);
}
else
{
TableInfo tableInfo = TableInfo.CreateInstance(context, type, entities, operationType, bulkConfig);
if (operationType == OperationType.Insert && !tableInfo.BulkConfig.SetOutputIdentity)
{
SqlBulkOperation.Insert(context, type, entities, tableInfo, progress);
}
else if (operationType == OperationType.Read)
{
SqlBulkOperation.Read(context, type, entities, tableInfo, progress);
}
else if (operationType == OperationType.Truncate)
{
SqlBulkOperation.Truncate(context, tableInfo);
}
else
{
SqlBulkOperation.Merge(context, type, entities, tableInfo, operationType, progress);
}
}
}
}
public static TableInfo CreateInstance(DbContext context, Type type, IList <object> entities, OperationType operationType, BulkConfig bulkConfig)
{
return(CreateInstance <object>(context, type, entities, operationType, bulkConfig));
}
public static TableInfo CreateInstance <T>(DbContext context, IList <T> entities, OperationType operationType, BulkConfig bulkConfig)
{
var tableInfo = new TableInfo
{
NumberOfEntities = entities.Count,
BulkConfig = bulkConfig ?? new BulkConfig()
};
bool isExplicitTransaction = context.Database.GetDbConnection().State == ConnectionState.Open;
if (tableInfo.BulkConfig.UseTempDB == true && !isExplicitTransaction && (operationType != OperationType.Insert || tableInfo.BulkConfig.SetOutputIdentity))
{
throw new InvalidOperationException("UseTempDB when set then BulkOperation has to be inside Transaction. More info in README of the library in GitHub.");
// Otherwise throws exception: 'Cannot access destination table' (gets Dropped too early because transaction ends before operation is finished)
}
var isDeleteOperation = operationType == OperationType.Delete;
tableInfo.LoadData <T>(context, isDeleteOperation);
return(tableInfo);
}
public static Task ExecuteAsync <T>(DbContext context, IList <T> entities, OperationType operationType, BulkConfig bulkConfig, Action <decimal> progress, CancellationToken cancellationToken) where T : class
{
if (entities.Count == 0)
{
return(Task.CompletedTask);
}
TableInfo tableInfo = TableInfo.CreateInstance(context, entities, operationType, bulkConfig);
if (operationType == OperationType.Insert && !tableInfo.BulkConfig.SetOutputIdentity)
{
return(SqlBulkOperation.InsertAsync(context, entities, tableInfo, progress, cancellationToken));
}
else if (operationType == OperationType.Read)
{
return(SqlBulkOperation.ReadAsync(context, entities, tableInfo, progress, cancellationToken));
}
else
{
return(SqlBulkOperation.MergeAsync(context, entities, tableInfo, operationType, progress, cancellationToken));
}
}
public static Task BulkReadAsync <T>(this DbContext context, IList <T> entities, BulkConfig bulkConfig = null, Action <decimal> progress = null) where T : class
{
return(DbContextBulkTransaction.ExecuteAsync(context, entities, OperationType.Read, bulkConfig, progress));
}
private static async Task ExecuteWithGraphAsync(DbContext context, IEnumerable <object> entities, OperationType operationType, BulkConfig bulkConfig, Action <decimal> progress, CancellationToken cancellationToken, bool isAsync)
{
if (operationType != OperationType.Insert &&
operationType != OperationType.InsertOrUpdate &&
operationType != OperationType.InsertOrUpdateDelete &&
operationType != OperationType.Update)
{
throw new InvalidBulkConfigException($"{nameof(BulkConfig)}.{nameof(BulkConfig.IncludeGraph)} only supports Insert or Update operations.");
}
// Sqlite bulk merge adapter does not support multiple objects of the same type with a zero value primary key
if (SqlAdaptersMapping.GetDatabaseType(context) == DbServer.Sqlite)
{
throw new NotSupportedException("Sqlite is not currently supported due to its BulkInsert implementation.");
}
bulkConfig.PreserveInsertOrder = true; // Required for SetOutputIdentity ('true' is default but here explicitly assigned again in case it was changed to 'false' in BulkConfing)
bulkConfig.SetOutputIdentity = true; // If this is set to false, won't be able to propogate new primary keys to the relationships
// If this is set to false, wont' be able to support some code first model types as EFCore uses shadow properties when a relationship's foreign keys arent explicitly defined
bulkConfig.EnableShadowProperties = true;
var graphNodes = GraphUtil.GetTopologicallySortedGraph(context, entities);
if (graphNodes == null)
{
return;
}
// Inserting an entity graph must be done within a transaction otherwise the database could end up in a bad state
var hasExistingTransaction = context.Database.CurrentTransaction != null;
var transaction = context.Database.CurrentTransaction ?? (isAsync ? await context.Database.BeginTransactionAsync() : context.Database.BeginTransaction());
try
{
// Group the graph nodes by entity type so we can merge them into the database in batches, in the correct order of dependency (topological order)
var graphNodesGroupedByType = graphNodes.GroupBy(y => y.Entity.GetType());
foreach (var graphNodeGroup in graphNodesGroupedByType)
{
var entityClrType = graphNodeGroup.Key;
var entityType = context.Model.FindEntityType(entityClrType);
if (OwnedTypeUtil.IsOwnedInSameTableAsOwner(entityType))
{
continue;
}
// It is possible the object graph contains duplicate entities (by primary key) but the entities are different object instances in memory.
// This an happen when deserializing a nested JSON tree for example. So filter out the duplicates.
var entitiesToAction = GetUniqueEntities(context, graphNodeGroup.Select(y => y.Entity)).ToList();
var tableInfo = TableInfo.CreateInstance(context, entityClrType, entitiesToAction, operationType, bulkConfig);
if (isAsync)
{
await SqlBulkOperation.MergeAsync(context, entityClrType, entitiesToAction, tableInfo, operationType, progress, cancellationToken);
}
else
{
SqlBulkOperation.Merge(context, entityClrType, entitiesToAction, tableInfo, operationType, progress);
}
// Set the foreign keys for dependents so they may be inserted on the next loop
var dependentsOfSameType = SetForeignKeysForDependentsAndYieldSameTypeDependents(context, entityClrType, graphNodeGroup).ToList();
// If there are any dependents of the same type (parent child relationship), then save those dependent entities again to commit the fk values
if (dependentsOfSameType.Any())
{
var dependentTableInfo = TableInfo.CreateInstance(context, entityClrType, dependentsOfSameType, operationType, bulkConfig);
if (isAsync)
{
await SqlBulkOperation.MergeAsync(context, entityClrType, dependentsOfSameType, dependentTableInfo, operationType, progress, cancellationToken);
}
else
{
SqlBulkOperation.Merge(context, entityClrType, dependentsOfSameType, dependentTableInfo, operationType, progress);
}
}
}
if (hasExistingTransaction == false)
{
if (isAsync)
{
await transaction.CommitAsync();
}
else
{
transaction.Commit();
}
}
}
finally
{
if (hasExistingTransaction == false)
{
if (isAsync)
{
await transaction.DisposeAsync();
}
else
{
transaction.Dispose();
}
}
}
}
public static void Execute <T>(DbContext context, IList <T> entities, OperationType operationType, BulkConfig bulkConfig, Action <decimal> progress) where T : class
{
if (entities.Count == 0)
{
return;
}
TableInfo tableInfo = TableInfo.CreateInstance(context, entities, operationType, bulkConfig);
if (operationType == OperationType.Insert && !tableInfo.BulkConfig.SetOutputIdentity)
{
SqlBulkOperation.Insert(context, entities, tableInfo, progress);
}
else if (operationType == OperationType.Read)
{
SqlBulkOperation.Read(context, entities, tableInfo, progress);
}
else
{
SqlBulkOperation.Merge(context, entities, tableInfo, operationType, progress);
}
}
public static async Task ExecuteWithGraphAsync(DbContext context, IEnumerable <object> entities, OperationType operationType, BulkConfig bulkConfig, Action <decimal> progress, CancellationToken cancellationToken)
{
await ExecuteWithGraphAsync(context, entities, operationType, bulkConfig, progress, cancellationToken, isAsync : true);
}
public static void ExecuteWithGraph(DbContext context, IEnumerable <object> entities, OperationType operationType, BulkConfig bulkConfig, Action <decimal> progress)
{
ExecuteWithGraphAsync(context, entities, operationType, bulkConfig, progress, CancellationToken.None, isAsync: false).GetAwaiter().GetResult();
}
public static Task ExecuteAsync <T>(DbContext context, IList <T> entities, OperationType operationType, BulkConfig bulkConfig, Action <decimal> progress) where T : class
{
TableInfo tableInfo = TableInfo.CreateInstance(context, entities, operationType, bulkConfig);
if (operationType == OperationType.Insert && !tableInfo.BulkConfig.SetOutputIdentity)
{
return(SqlBulkOperation.InsertAsync(context, entities, tableInfo, progress));
}
else
{
return(SqlBulkOperation.MergeAsync(context, entities, tableInfo, operationType, progress));
}
}
public static Task BulkInsertOrUpdateOrDeleteAsync <T>(this DbContext context, IList <T> entities, BulkConfig bulkConfig = null, Action <decimal> progress = null, Type type = null, CancellationToken cancellationToken = default) where T : class
{
return(DbContextBulkTransaction.ExecuteAsync(context, type, entities, OperationType.InsertOrUpdateDelete, bulkConfig, progress, cancellationToken));
}
public static DbTransaction GetUnderlyingTransaction(this IDbContextTransaction ctxTransaction, BulkConfig config)
{
var dbTransaction = ctxTransaction.GetDbTransaction();
if (config?.UnderlyingTransaction != null)
{
dbTransaction = config.UnderlyingTransaction(dbTransaction);
}
return(dbTransaction);
}
public static TableInfo CreateInstance <T>(DbContext context, IList <T> entities, OperationType operationType, BulkConfig bulkConfig)
{
return(CreateInstance <T>(context, typeof(T), entities, operationType, bulkConfig));
}
public static Task BulkReadAsync(this DbContext context, Type entityType, IList <object> entities, BulkConfig bulkConfig = null, Action <decimal> progress = null, CancellationToken cancellationToken = default)
{
return(DbContextBulkTransaction.ExecuteAsync(context, entityType, entities, OperationType.Read, bulkConfig, progress, cancellationToken));
}
public static TableInfo CreateInstance <T>(DbContext context, IList <T> entities, OperationType operationType, BulkConfig bulkConfig)
{
var tableInfo = new TableInfo
{
NumberOfEntities = entities.Count,
BulkConfig = bulkConfig ?? new BulkConfig()
};
bool isExplicitTransaction = context.Database.GetDbConnection().State == ConnectionState.Open;
if (tableInfo.BulkConfig.UseTempDB == true && !isExplicitTransaction && (operationType != OperationType.Insert || tableInfo.BulkConfig.SetOutputIdentity))
{
tableInfo.BulkConfig.UseTempDB = false;
// If BulkOps is not in explicit transaction then tempdb[#] can only be used with Insert, other Operations done with customTemp table.
// Otherwise throws exception: 'Cannot access destination table' (gets Droped too early because transaction ends before operation is finished)
}
var isDeleteOperation = operationType == OperationType.Delete;
tableInfo.LoadData <T>(context, isDeleteOperation);
return(tableInfo);
}
public static void BulkUpdate(this DbContext context, Type entityType, IList <object> entities, BulkConfig bulkConfig = null, Action <decimal> progress = null)
{
DbContextBulkTransaction.Execute(context, entityType, entities, OperationType.Update, bulkConfig, progress);
}
public static TableInfo CreateInstance <T>(DbContext context, IList <T> entities, OperationType operationType, BulkConfig bulkConfig)
{
var tableInfo = new TableInfo
{
NumberOfEntities = entities.Count,
BulkConfig = bulkConfig ?? new BulkConfig()
};
if (operationType != OperationType.Insert)
{
tableInfo.BulkConfig.UseTempDB = false; // TempDB can only be used with Insert.
// Other Operations done with customTemp table.
// If using tempdb[#] throws exception: 'Cannot access destination table' (gets Droped too early, probably because transaction ends)
}
var isDeleteOperation = operationType == OperationType.Delete;
tableInfo.LoadData <T>(context, isDeleteOperation);
return(tableInfo);
}
public static void Execute <T>(DbContext context, Type type, IList <T> entities, OperationType operationType, BulkConfig bulkConfig, Action <decimal> progress) where T : class
{
type ??= typeof(T);
using (ActivitySources.StartExecuteActivity(operationType, entities.Count))
{
if (entities.Count == 0 &&
operationType != OperationType.InsertOrUpdateOrDelete &&
operationType != OperationType.Truncate &&
operationType != OperationType.SaveChanges &&
(bulkConfig == null || bulkConfig.CustomSourceTableName == null))
{
return;
}
if (operationType == OperationType.SaveChanges)
{
DbContextBulkTransactionSaveChanges.SaveChanges(context, bulkConfig, progress);
return;
}
else if (bulkConfig?.IncludeGraph == true)
{
DbContextBulkTransactionGraphUtil.ExecuteWithGraph(context, entities, operationType, bulkConfig, progress);
}
else
{
TableInfo tableInfo = TableInfo.CreateInstance(context, type, entities, operationType, bulkConfig);
if (operationType == OperationType.Insert && !tableInfo.BulkConfig.SetOutputIdentity && tableInfo.BulkConfig.CustomSourceTableName == null)
{
SqlBulkOperation.Insert(context, type, entities, tableInfo, progress);
}
else if (operationType == OperationType.Read)
{
SqlBulkOperation.Read(context, type, entities, tableInfo, progress);
}
else if (operationType == OperationType.Truncate)
{
SqlBulkOperation.Truncate(context, tableInfo);
}
else
{
SqlBulkOperation.Merge(context, type, entities, tableInfo, operationType, progress);
}
}
}
}
public static TableInfo CreateInstance <T>(DbContext context, IList <T> entities, OperationType operationType, BulkConfig bulkConfig)
{
var tableInfo = new TableInfo();
var isDeleteOperation = operationType == OperationType.Delete;
tableInfo.NumberOfEntities = entities.Count;
tableInfo.BulkConfig = bulkConfig ?? new BulkConfig();
tableInfo.LoadData <T>(context, isDeleteOperation);
return(tableInfo);
}
public static void BulkRead <T>(this DbContext context, IEnumerable <T> entities, BulkConfig bulkConfig = null, Action <decimal> progress = null) where T : class
{
DbContextBulkTransaction.Execute(context, entities.ToList(), OperationType.Read, bulkConfig, progress);
}
public static async Task ExecuteAsync <T>(DbContext context, IList <T> entities, OperationType operationType, BulkConfig bulkConfig, Action <decimal> progress, CancellationToken cancellationToken) where T : class
{
using (ActivitySources.StartExecuteActivity(operationType, entities.Count))
{
if (operationType != OperationType.Truncate && entities.Count == 0)
{
return;
}
if (bulkConfig?.IncludeGraph == true)
{
await DbContextBulkTransactionGraphUtil.ExecuteWithGraphAsync(context, entities, operationType, bulkConfig, progress, cancellationToken);
}
else
{
TableInfo tableInfo = TableInfo.CreateInstance(context, entities, operationType, bulkConfig);
if (operationType == OperationType.Insert && !tableInfo.BulkConfig.SetOutputIdentity)
{
await SqlBulkOperation.InsertAsync(context, entities, tableInfo, progress, cancellationToken);
}
else if (operationType == OperationType.Read)
{
await SqlBulkOperation.ReadAsync(context, entities, tableInfo, progress, cancellationToken);
}
else if (operationType == OperationType.Truncate)
{
await SqlBulkOperation.TruncateAsync(context, tableInfo, cancellationToken);
}
else
{
await SqlBulkOperation.MergeAsync(context, entities, tableInfo, operationType, progress, cancellationToken);
}
}
}
}
// Async methods
public static Task BulkInsertAsync <T>(this DbContext context, IEnumerable <T> entities, BulkConfig bulkConfig = null, Action <decimal> progress = null) where T : class
{
return(DbContextBulkTransaction.ExecuteAsync(context, entities.ToList(), OperationType.Insert, bulkConfig, progress));
}
internal static async Task <SqliteConnection> OpenAndGetSqliteConnectionAsync(DbContext context, BulkConfig bulkConfig, CancellationToken cancellationToken)
{
var connection = bulkConfig.SqliteConnection ?? (SqliteConnection)context.Database.GetDbConnection();
if (connection.State != ConnectionState.Open)
{
await connection.OpenAsync(cancellationToken).ConfigureAwait(false);
}
return(connection);
}
// InsertOrUpdateOrDelete methods
#region BulkInsertOrUpdateOrDelete
public static void BulkInsertOrUpdateOrDelete <T>(this DbContext context, IList <T> entities, BulkConfig bulkConfig = null, Action <decimal> progress = null, Type type = null) where T : class
{
DbContextBulkTransaction.Execute(context, type, entities, OperationType.InsertOrUpdateDelete, bulkConfig, progress);
}
internal static async Task <SqlConnection> OpenAndGetSqlConnectionAsync(DbContext context, BulkConfig config)
{
var connection = context.GetUnderlyingConnection(config);
if (connection.State != ConnectionState.Open)
{
await connection.OpenAsync().ConfigureAwait(false);
}
return((SqlConnection)connection);
}
public static async Task SaveChangesAsync(DbContext context, BulkConfig bulkConfig, Action <decimal> progress, CancellationToken cancellationToken)
{
await SaveChangesAsync(context, bulkConfig, progress, cancellationToken, isAsync : true).ConfigureAwait(false);
}