public Dictionary <string, string> ConfigureBulkReadTableInfo(DbContext context) { InsertToTempTable = true; var previousPropertyColumnNamesDict = PropertyColumnNamesDict; BulkConfig.PropertiesToInclude = PrimaryKeys; PropertyColumnNamesDict = PropertyColumnNamesDict.Where(a => PrimaryKeys.Contains(a.Key)).ToDictionary(i => i.Key, i => i.Value); return(previousPropertyColumnNamesDict); }
public Dictionary<string, string> ConfigureBulkReadTableInfo(DbContext context) { InsertToTempTable = true; if (BulkConfig.UpdateByProperties == null || BulkConfig.UpdateByProperties.Count() == 0) CheckHasIdentity(context); var previousPropertyColumnNamesDict = PropertyColumnNamesDict; BulkConfig.PropertiesToInclude = PrimaryKeys; PropertyColumnNamesDict = PropertyColumnNamesDict.Where(a => PrimaryKeys.Contains(a.Key)).ToDictionary(i => i.Key, i => i.Value); return previousPropertyColumnNamesDict; }
protected void UpdateEntitiesIdentity <T>(IList <T> entities, IList <T> entitiesWithOutputIdentity) { if (BulkConfig.PreserveInsertOrder) // Updates PK in entityList { var accessor = TypeAccessor.Create(typeof(T), true); string identityPropertyName = PropertyColumnNamesDict.SingleOrDefault(a => a.Value == IdentityColumnName).Key; for (int i = 0; i < NumberOfEntities; i++) { accessor[entities[i], identityPropertyName] = accessor[entitiesWithOutputIdentity[i], identityPropertyName]; } } else // Clears entityList and then refills it with loaded entites from Db { entities.Clear(); ((List <T>)entities).AddRange(entitiesWithOutputIdentity); } }
public async Task LoadOutputDataAsync <T>(DbContext context, IList <T> entities, CancellationToken cancellationToken) where T : class { bool hasIdentity = PropertyColumnNamesDict.Any(a => a.Value == IdentityColumnName); if (BulkConfig.SetOutputIdentity && hasIdentity) { string sqlQuery = SqlQueryBuilder.SelectFromOutputTable(this); var entitiesWithOutputIdentity = await QueryOutputTableAsync <T>(context, sqlQuery).ToListAsync(cancellationToken).ConfigureAwait(false); UpdateEntitiesIdentity(entities, entitiesWithOutputIdentity); } if (BulkConfig.CalculateStats) { int numberUpdated = await GetNumberUpdatedAsync(context, cancellationToken).ConfigureAwait(false); BulkConfig.StatsInfo = new StatsInfo { StatsNumberUpdated = numberUpdated, StatsNumberInserted = entities.Count - numberUpdated }; } }
// Compiled queries created manually to avoid EF Memory leak bug when using EF with dynamic SQL: // https://github.com/borisdj/EFCore.BulkExtensions/issues/73 // Once the following Issue gets fixed(expected in EF 3.0) this can be replaced with code segment: DirectQuery // https://github.com/aspnet/EntityFrameworkCore/issues/12905 #region CompiledQuery public void LoadOutputData <T>(DbContext context, IList <T> entities) where T : class { bool hasIdentity = PropertyColumnNamesDict.Any(a => a.Value == IdentityColumnName); if (BulkConfig.SetOutputIdentity && hasIdentity) { string sqlQuery = SqlQueryBuilder.SelectFromOutputTable(this); var entitiesWithOutputIdentity = QueryOutputTable <T>(context, sqlQuery).ToList(); UpdateEntitiesIdentity(entities, entitiesWithOutputIdentity); } if (BulkConfig.CalculateStats) { string sqlQueryCount = SqlQueryBuilder.SelectCountIsUpdateFromOutputTable(this); int numberUpdated = GetNumberUpdated(context); BulkConfig.StatsInfo = new StatsInfo { StatsNumberUpdated = numberUpdated, StatsNumberInserted = entities.Count - numberUpdated }; } }
protected void UpdateEntitiesIdentity <T>(IList <T> entities, IList <T> entitiesWithOutputIdentity) { if (BulkConfig.PreserveInsertOrder) // Updates PK in entityList { var accessor = TypeAccessor.Create(typeof(T), true); string identityPropertyName = PropertyColumnNamesDict.SingleOrDefault(a => a.Value == IdentityColumnName).Key; for (int i = 0; i < NumberOfEntities; i++) { accessor[entities[i], identityPropertyName] = accessor[entitiesWithOutputIdentity[i], identityPropertyName]; if (TimeStampColumnName != null) // timestamp/rowversion is also generated by the SqlServer so if exist should ba updated as well { accessor[entities[i], TimeStampColumnName] = accessor[entitiesWithOutputIdentity[i], TimeStampColumnName]; } } } else // Clears entityList and then refills it with loaded entites from Db { entities.Clear(); ((List <T>)entities).AddRange(entitiesWithOutputIdentity); } }
public void LoadData <T>(DbContext context, IList <T> entities, bool loadOnlyPKColumn) { LoadOnlyPKColumn = loadOnlyPKColumn; var type = typeof(T); var entityType = context.Model.FindEntityType(type); if (entityType == null) { type = entities[0].GetType(); entityType = context.Model.FindEntityType(type); HasAbstractList = true; } if (entityType == null) { throw new InvalidOperationException($"DbContext does not contain EntitySet for Type: { type.Name }"); } var relationalData = entityType.Relational(); Schema = relationalData.Schema ?? "dbo"; TableName = relationalData.TableName; TempTableSufix = "Temp" + Guid.NewGuid().ToString().Substring(0, 8); // 8 chars of Guid as tableNameSufix to avoid same name collision with other tables bool AreSpecifiedUpdateByProperties = BulkConfig.UpdateByProperties?.Count() > 0; var primaryKeys = entityType.FindPrimaryKey().Properties.Select(a => a.Name).ToList(); HasSinglePrimaryKey = primaryKeys.Count == 1; PrimaryKeys = AreSpecifiedUpdateByProperties ? BulkConfig.UpdateByProperties : primaryKeys; var allProperties = entityType.GetProperties().AsEnumerable(); var ownedTypes = entityType.GetNavigations().Where(a => a.GetTargetType().IsOwned()); HasOwnedTypes = ownedTypes.Any(); OwnedTypesDict = ownedTypes.ToDictionary(a => a.Name, a => a); IdentityColumnName = allProperties.SingleOrDefault(a => a.IsPrimaryKey() && a.ClrType.Name.StartsWith("Int") && a.ValueGenerated == ValueGenerated.OnAdd)?.Name; // ValueGenerated equals OnAdd even for nonIdentity column like Guid so we only type int as second condition // timestamp/row version properties are only set by the Db, the property has a [Timestamp] Attribute or is configured in in FluentAPI with .IsRowVersion() // They can be identified by the columne type "timestamp" or .IsConcurrencyToken in combination with .ValueGenerated == ValueGenerated.OnAddOrUpdate string timestampDbTypeName = nameof(TimestampAttribute).Replace("Attribute", "").ToLower(); // = "timestamp"; var timeStampProperties = allProperties.Where(a => (a.IsConcurrencyToken && a.ValueGenerated == ValueGenerated.OnAddOrUpdate) || a.Relational().ColumnType == timestampDbTypeName); TimeStampColumnName = timeStampProperties.FirstOrDefault()?.Relational().ColumnName; // can be only One var allPropertiesExceptTimeStamp = allProperties.Except(timeStampProperties); var properties = allPropertiesExceptTimeStamp.Where(a => a.Relational().ComputedColumnSql == null); // TimeStamp prop. is last column in OutputTable since it is added later with varbinary(8) type in which Output can be inserted OutputPropertyColumnNamesDict = allPropertiesExceptTimeStamp.Concat(timeStampProperties).ToDictionary(a => a.Name, b => b.Relational().ColumnName.Replace("]", "]]")); // square brackets have to be escaped ColumnNameContainsSquareBracket = allPropertiesExceptTimeStamp.Concat(timeStampProperties).Any(a => a.Relational().ColumnName.Contains("]")); bool AreSpecifiedPropertiesToInclude = BulkConfig.PropertiesToInclude?.Count() > 0; bool AreSpecifiedPropertiesToExclude = BulkConfig.PropertiesToExclude?.Count() > 0; if (AreSpecifiedPropertiesToInclude) { if (AreSpecifiedUpdateByProperties) // Adds UpdateByProperties to PropertyToInclude if they are not already explicitly listed { foreach (var updateByProperty in BulkConfig.UpdateByProperties) { if (!BulkConfig.PropertiesToInclude.Contains(updateByProperty)) { BulkConfig.PropertiesToInclude.Add(updateByProperty); } } } else // Adds PrimaryKeys to PropertyToInclude if they are not already explicitly listed { foreach (var primaryKey in PrimaryKeys) { if (!BulkConfig.PropertiesToInclude.Contains(primaryKey)) { BulkConfig.PropertiesToInclude.Add(primaryKey); } } } } UpdateByPropertiesAreNullable = properties.Any(a => PrimaryKeys.Contains(a.Name) && a.IsNullable); if (AreSpecifiedPropertiesToInclude || AreSpecifiedPropertiesToExclude) { if (AreSpecifiedPropertiesToInclude && AreSpecifiedPropertiesToExclude) { throw new InvalidOperationException("Only one group of properties, either PropertiesToInclude or PropertiesToExclude can be specifed, specifying both not allowed."); } if (AreSpecifiedPropertiesToInclude) { properties = properties.Where(a => BulkConfig.PropertiesToInclude.Contains(a.Name)); } if (AreSpecifiedPropertiesToExclude) { properties = properties.Where(a => !BulkConfig.PropertiesToExclude.Contains(a.Name)); } } if (loadOnlyPKColumn) { PropertyColumnNamesDict = properties.Where(a => PrimaryKeys.Contains(a.Name)).ToDictionary(a => a.Name, b => b.Relational().ColumnName.Replace("]", "]]")); } else { PropertyColumnNamesDict = properties.ToDictionary(a => a.Name, b => b.Relational().ColumnName.Replace("]", "]]")); ShadowProperties = new HashSet <string>(properties.Where(p => p.IsShadowProperty).Select(p => p.Relational().ColumnName)); foreach (var property in properties.Where(p => p.GetValueConverter() != null)) { string columnName = property.Relational().ColumnName; ValueConverter converter = property.GetValueConverter(); ConvertibleProperties.Add(columnName, converter); } if (HasOwnedTypes) // Support owned entity property update. TODO: Optimize { foreach (var navgationProperty in ownedTypes) { var property = navgationProperty.PropertyInfo; Type navOwnedType = type.Assembly.GetType(property.PropertyType.FullName); var ownedEntityType = context.Model.FindEntityType(property.PropertyType); if (ownedEntityType == null) // when entity has more then one ownedType (e.g. Address HomeAddress, Address WorkAddress) or one ownedType is in multiple Entities like Audit is usually. { ownedEntityType = context.Model.GetEntityTypes().SingleOrDefault(a => a.DefiningNavigationName == property.Name && a.DefiningEntityType.Name == entityType.Name); } var ownedEntityProperties = ownedEntityType.GetProperties().ToList(); var ownedEntityPropertyNameColumnNameDict = new Dictionary <string, string>(); foreach (var ownedEntityProperty in ownedEntityProperties) { if (!ownedEntityProperty.IsPrimaryKey()) { string columnName = ownedEntityProperty.Relational().ColumnName; ownedEntityPropertyNameColumnNameDict.Add(ownedEntityProperty.Name, columnName); } } var ownedProperties = property.PropertyType.GetProperties(); foreach (var ownedProperty in ownedProperties) { if (ownedEntityPropertyNameColumnNameDict.ContainsKey(ownedProperty.Name)) { string columnName = ownedEntityPropertyNameColumnNameDict[ownedProperty.Name]; var ownedPropertyType = Nullable.GetUnderlyingType(ownedProperty.PropertyType) ?? ownedProperty.PropertyType; bool doAddProperty = true; if (AreSpecifiedPropertiesToInclude && !BulkConfig.PropertiesToInclude.Contains(columnName)) { doAddProperty = false; } if (AreSpecifiedPropertiesToExclude && BulkConfig.PropertiesToExclude.Contains(columnName)) { doAddProperty = false; } if (doAddProperty) { PropertyColumnNamesDict.Add(property.Name + "." + ownedProperty.Name, columnName); OutputPropertyColumnNamesDict.Add(property.Name + "." + ownedProperty.Name, columnName); } } } } } } }
public void LoadData <T>(DbContext context, bool loadOnlyPKColumn) { var entityType = context.Model.FindEntityType(typeof(T)); if (entityType == null) { throw new InvalidOperationException("DbContext does not contain EntitySet for Type: " + typeof(T).Name); } var relationalData = entityType.Relational(); Schema = relationalData.Schema ?? "dbo"; TableName = relationalData.TableName; TempTableSufix = "Temp" + Guid.NewGuid().ToString().Substring(0, 8); // 8 chars of Guid as tableNameSufix to avoid same name collision with other tables bool AreSpecifiedUpdateByProperties = BulkConfig.UpdateByProperties?.Count() > 0; PrimaryKeys = AreSpecifiedUpdateByProperties ? BulkConfig.UpdateByProperties : entityType.FindPrimaryKey().Properties.Select(a => a.Name).ToList(); HasSinglePrimaryKey = PrimaryKeys.Count == 1; var allProperties = entityType.GetProperties().AsEnumerable(); var allNavigationProperties = entityType.GetNavigations().Where(a => a.GetTargetType().IsOwned()); HasOwnedTypes = allNavigationProperties.Any(); // timestamp datatype can only be set by database, that's property having [Timestamp] Attribute but keep if one with [ConcurrencyCheck] var timeStampProperties = allProperties.Where(a => a.IsConcurrencyToken == true && a.ValueGenerated == ValueGenerated.OnAddOrUpdate && a.BeforeSaveBehavior == PropertySaveBehavior.Ignore); TimeStampColumn = timeStampProperties.FirstOrDefault()?.Relational().ColumnName; // expected to be only One var properties = allProperties.Except(timeStampProperties); OutputPropertyColumnNamesDict = properties.ToDictionary(a => a.Name, b => b.Relational().ColumnName); properties = properties.Where(a => a.Relational().ComputedColumnSql == null); bool AreSpecifiedPropertiesToInclude = BulkConfig.PropertiesToInclude?.Count() > 0; bool AreSpecifiedPropertiesToExclude = BulkConfig.PropertiesToExclude?.Count() > 0; if (AreSpecifiedPropertiesToInclude) { if (AreSpecifiedUpdateByProperties) // Adds UpdateByProperties to PropertyToInclude if they are not already explicitly listed { foreach (var updateByProperty in BulkConfig.UpdateByProperties) { if (!BulkConfig.PropertiesToInclude.Contains(updateByProperty)) { BulkConfig.PropertiesToInclude.Add(updateByProperty); } } } else // Adds PrimaryKeys to PropertyToInclude if they are not already explicitly listed { foreach (var primaryKey in PrimaryKeys) { if (!BulkConfig.PropertiesToInclude.Contains(primaryKey)) { BulkConfig.PropertiesToInclude.Add(primaryKey); } } } } UpdateByPropertiesAreNullable = properties.Any(a => PrimaryKeys.Contains(a.Name) && a.IsNullable); if (AreSpecifiedPropertiesToInclude || AreSpecifiedPropertiesToExclude) { if (AreSpecifiedPropertiesToInclude && AreSpecifiedPropertiesToExclude) { throw new InvalidOperationException("Only one group of properties, either PropertiesToInclude or PropertiesToExclude can be specifed, specifying both not allowed."); } if (AreSpecifiedPropertiesToInclude) { properties = properties.Where(a => BulkConfig.PropertiesToInclude.Contains(a.Name)); } if (AreSpecifiedPropertiesToExclude) { properties = properties.Where(a => !BulkConfig.PropertiesToExclude.Contains(a.Name)); } } if (loadOnlyPKColumn) { PropertyColumnNamesDict = properties.Where(a => PrimaryKeys.Contains(a.Name)).ToDictionary(a => a.Name, b => b.Relational().ColumnName); } else { PropertyColumnNamesDict = properties.ToDictionary(a => a.Name, b => b.Relational().ColumnName); ShadowProperties = new HashSet <string>(properties.Where(p => p.IsShadowProperty).Select(p => p.Relational().ColumnName)); foreach (var property in properties.Where(p => p.GetValueConverter() != null)) { ConvertibleProperties.Add(property.Relational().ColumnName, property.GetValueConverter()); } if (HasOwnedTypes) // Support owned entity property update. TODO: Optimize { var type = typeof(T); foreach (var navgationProperty in allNavigationProperties) { var property = navgationProperty.PropertyInfo; Type navOwnedType = type.Assembly.GetType(property.PropertyType.FullName); var ownedEntityType = context.Model.FindEntityType(property.PropertyType); if (ownedEntityType == null) // when entity has more then one ownedType (e.g. Address HomeAddress, Address WorkAddress) or one ownedType is in multiple Entities like Audit is usually. { ownedEntityType = context.Model.GetEntityTypes().SingleOrDefault(a => a.DefiningNavigationName == property.Name && a.DefiningEntityType.Name == entityType.Name); } var ownedEntityProperties = ownedEntityType.GetProperties().ToList(); var ownedEntityPropertyNameColumnNameDict = new Dictionary <string, string>(); foreach (var ownedEntityProperty in ownedEntityProperties) { if (!ownedEntityProperty.IsPrimaryKey()) { string columnName = ownedEntityProperty.Relational().ColumnName; ownedEntityPropertyNameColumnNameDict.Add(ownedEntityProperty.Name, columnName); } } var ownedProperties = property.PropertyType.GetProperties(); foreach (var ownedProperty in ownedProperties) { if (ownedEntityPropertyNameColumnNameDict.ContainsKey(ownedProperty.Name)) { string columnName = ownedEntityPropertyNameColumnNameDict[ownedProperty.Name]; var ownedPropertyType = Nullable.GetUnderlyingType(ownedProperty.PropertyType) ?? ownedProperty.PropertyType; PropertyColumnNamesDict.Add(property.Name + "." + ownedProperty.Name, columnName); OutputPropertyColumnNamesDict.Add(property.Name + "." + ownedProperty.Name, columnName); } } } } } }