public void Ctor_SetsData() { const int id = 42; const int num = 123; const bool flag = true; const string text = "test"; var date = new DateTime(1980, 1, 30); var data = new ComplexType { Id = id, IntProperty = num, BoolProperty = flag, StringProperty = text, DateTimeProperty = date }; var result = new BulkOperation(data, Operation.Index, "en", null, null, "test-en"); dynamic resultData = result.Data; Assert.Equal(id.ToString(), resultData.Id); Assert.Equal(num.ToString(), resultData.IntProperty); Assert.Equal(flag, resultData.BoolProperty); Assert.Equal(text, resultData.StringProperty); Assert.Equal(date, resultData.DateTimeProperty); }
public static void Main() { List <Contato> contatos = new List <Contato>(); string line; int counter = 0; string[] columns; string[] rows; var file = new StreamReader(@"{caminho do arquivo .csv}"); while ((line = file.ReadLine()) != null) { Console.WriteLine(line); if (counter == 0) { columns = line.Split(";"); } else { Contato contato = new Contato(); rows = line.Split(";"); contato.Id = Convert.ToInt32(rows[0]); contato.FName = rows[1]; contato.LName = rows[2]; contato.Title = rows[3]; contato.Email = rows[4]; contato.Mobile = rows[5]; contato.Address1 = rows[6]; contato.Address2 = rows[7]; contato.City = rows[8]; contato.Zip = Convert.ToInt32(rows[9]); contato.State = rows[10]; contato.CountryId = rows[11]; contato.Lang = rows[12]; contato.Birthdate = Convert.ToDateTime(rows[13]); contatos.Add(contato); } counter++; } var connection = new SqlConnection(@"{connection string}"); connection.Open(); var bulk = new BulkOperation <Contato>(connection); bulk.DestinationTableName = "Contatos"; bulk.ColumnInputExpression = c => new { c.FName, c.LName, c.Title, c.Email, c.Mobile, c.Address1, c.Address2, c.City, c.Zip, c.State, c.CountryId, c.Lang, c.Birthdate }; bulk.ColumnOutputExpression = c => c.Id; bulk.ColumnPrimaryKeyExpression = c => c.Id; bulk.BulkInsert(contatos); //bulk.BulkUpdate(contatos); }
public void Ctor_SetsMetaDataDataType() { var data = new ComplexType { Id = 42 }; var result = new BulkOperation(data, Operation.Index, "en", null, null, "test-en"); Assert.True(result.MetaData.DataType.IsAssignableFrom(typeof(ComplexType))); }
public void Ctor_TypeWithID_SetsMetaDataId() { var data = new ComplexType { Id = 42 }; var result = new BulkOperation(data, Operation.Index, "en", null, null, "test-en"); Assert.Equal("42", result.MetaData.Id); }
public override BulkOperation <TOperationEntity> Apply <TOperationEntity>( ICrudRequestConfig config, BulkOperation <TOperationEntity> operation) { operation = base.Apply(config, operation); operation.CaseSensitive = _keyCaseSensitivity; return(operation); }
public async Task DeleteMany(IEnumerable <T> items, string tableName) { DbConnection db = new SqlConnection(_connectionString); db.Open(); var bulkOperation = new BulkOperation(db); bulkOperation.DestinationTableName = tableName; await bulkOperation.BulkDeleteAsync(items); db.Close(); }
private void EdSystemPrimaryKeyMapping(BulkOperation operation) { if (operation is BulkOperation <EdSystem> ) { var bulk = (BulkOperation <EdSystem>)operation; bulk.ColumnPrimaryKeyExpression = c => c.EdsmId; } if (operation is BulkOperation <EdSystemCoordinates> ) { var bulk = (BulkOperation <EdSystemCoordinates>)operation; bulk.ColumnPrimaryKeyExpression = c => new { c.X, c.Y, c.Z }; } }
/// <summary> /// /// </summary> /// <typeparam name="T"></typeparam> /// <param name="action"></param> protected void Db <T>(Action <BulkOperation <T> > action) where T : class { using (var conn = new SqlConnection(ConnString)) { conn.Open(); var bulk = new BulkOperation <T>(conn) { BatchSize = 1000 }; action.Invoke(bulk); conn.Close(); conn.Dispose(); } }
public void BulkDelete(IEnumerable <TEntity> items) { CheckForObjectAlreadyDisposedOrNot(typeof(EntityFrameworkCodeFirstCommand <TEntity, TId>).FullName); ContractUtility.Requires <ArgumentNullException>(items.IsNotNull(), "items instance cannot be null"); ContractUtility.Requires <ArgumentOutOfRangeException>(items.IsNotEmpty(), "items count should be greater than 0"); DbConnection connection = _dbContext.Database.Connection; if (connection.State != ConnectionState.Open) { connection.Open(); } var bulkOperation = new BulkOperation <TEntity>(connection); ApplyAuditInfoRules(); bulkOperation.BulkDelete(items); }
public async Task UpdateManyAfterContinueRound(IEnumerable <GamePlayer> gamePlayers) { DbConnection db = new SqlConnection(_connectionString); db.Open(); var bulkOperation = new BulkOperation(db); bulkOperation.DestinationTableName = "GamePlayers"; bulkOperation.ColumnMappings.Add(new ColumnMapping("Id", true)); bulkOperation.ColumnMappings.Add(new ColumnMapping("RoundScore", "RoundScore")); bulkOperation.ColumnMappings.Add(new ColumnMapping("CardAmount", "CardAmount")); bulkOperation.ColumnMappings.Add(new ColumnMapping("BetPayCoefficient", "BetPayCoefficient")); await bulkOperation.BulkMergeAsync(gamePlayers); db.Close(); }
public async Task <List <Player> > CreateMany(List <Player> players) { DbConnection db = new SqlConnection(_connectionString); db.Open(); var bulkOperation = new BulkOperation(db); bulkOperation.DestinationTableName = "Players"; bulkOperation.ColumnMappings.Add(new ColumnMapping("Id", true)); bulkOperation.ColumnMappings.Add(new ColumnMapping("Name", "Name")); bulkOperation.ColumnMappings.Add(new ColumnMapping("Type", "Type")); bulkOperation.ColumnMappings.Add(new ColumnMapping("CreationDate", "CreationDate")); bulkOperation.ColumnMappings.Add("Id", ColumnMappingDirectionType.Output); await bulkOperation.BulkInsertAsync(players); db.Close(); return(players); }
private static void BulkWriteToDB(DataTable dataTable, string tableName) { try { MySqlConnection conn = DBUtils.GetDBConnection(); conn.Open(); using (var bulk = new BulkOperation(conn)) { bulk.DestinationTableName = tableName; bulk.BulkInsert(dataTable); } conn.Close(); } catch (Exception ex) { throw ex; } }
static void insertDataTable(DataTable dt, string tableName) { try { if (OpenConnection() == true) { using (BulkOperation bulk = new BulkOperation(conn)) { bulk.DestinationTableName = tableName; bulk.BulkMerge(dt); } CloseConnection(); } } catch (Exception ex) { Console.WriteLine("Ex raised at insertDataTable function: " + ex.Message); } }
/* * public int InsertDatatable(DataTable _dataTable) * { * lock(DBlock) * { * string sqlString = string.Format("SELECT * FROM {0} WHERE FALSE", _dataTable.TableName); * * using (MySqlConnection connection = establishConnection()) * { * using (MySqlCommand mySqlCommand = new MySqlCommand(sqlString, connection)) * { * connection.Open(); * MySqlTransaction mySqlTransaction = connection.BeginTransaction(IsolationLevel.ReadCommitted); * try * { * int count = 0; * MySqlDataAdapter dataAdapter = new MySqlDataAdapter(); * dataAdapter.SelectCommand = new MySqlCommand(sqlString, connection); * MySqlCommandBuilder builder = new MySqlCommandBuilder(dataAdapter); * builder.ConflictOption = ConflictOption.OverwriteChanges; * builder.SetAllValues = true; * count = dataAdapter.Update(_dataTable); * mySqlTransaction.Commit(); * _dataTable.AcceptChanges(); * dataAdapter.Dispose(); * builder.Dispose(); * * return count; * } * catch(Exception ex) * { * mySqlTransaction.Rollback(); * Console.WriteLine(ex.Message); * * return 0; * } * } * } * //以下是sqlBulkCopy 應用於SQL但不能用在MySQL * using (var sqlBulkCopy = new SqlBulkCopy(connectionString(), SqlBulkCopyOptions.UseInternalTransaction)) * { * //設定批次量及逾時 * sqlBulkCopy.BatchSize = 1000; * sqlBulkCopy.BulkCopyTimeout = 60; * //sqlBulkCopy.NotifyAfter = 10000; * //sqlBulkCopy.SqlRowsCopied += new SqlRowsCopiedEventHandler(OnSqlRowsCopied); * * //資料庫內的資料表名 * sqlBulkCopy.DestinationTableName = "plc"; * * sqlBulkCopy.ColumnMappings.Add("StationID", "StationID"); * sqlBulkCopy.ColumnMappings.Add("DateTime", "DateTime"); * sqlBulkCopy.ColumnMappings.Add("StationState", "StationState"); * sqlBulkCopy.ColumnMappings.Add("Temperature", "Temperature"); * sqlBulkCopy.ColumnMappings.Add("Pa", "Pa"); * sqlBulkCopy.ColumnMappings.Add("ValueType", "ValueType"); * sqlBulkCopy.ColumnMappings.Add("Value", "Value"); * sqlBulkCopy.ColumnMappings.Add("PositionState", "PositionState"); * sqlBulkCopy.ColumnMappings.Add("X", "X"); * sqlBulkCopy.ColumnMappings.Add("Y", "Y"); * sqlBulkCopy.ColumnMappings.Add("Z", "Z"); * sqlBulkCopy.ColumnMappings.Add("A", "A"); * sqlBulkCopy.ColumnMappings.Add("B", "B"); * sqlBulkCopy.ColumnMappings.Add("C", "C"); * * //開始寫入 * try * { * sqlBulkCopy.WriteToServer(_dataTable); * } * catch(Exception ex) * { * Console.WriteLine(ex.Message); * } * } * connection.Dispose(); */ //將Datatable可以直接insert進Database //對於中斷點可以再處理得更好 目前的test寫成了10秒內會輸入30萬筆左右的測試資料進資料庫 public void InsertTable(DataTable _datatable) { lock (DBlock) { //需要每個月初定期下載更新版本來延長license MySqlConnection connection = establishConnection(); connection.Open(); DataTable dt = new DataTable("plc"); dt = _datatable; MySqlDataAdapter da = new MySqlDataAdapter("SELECT * FROM plc", connection); MySqlCommandBuilder cb = new MySqlCommandBuilder(da); da.Fill(dt); var bulk = new BulkOperation(connection); bulk.DestinationTableName = "plc"; bulk.BulkInsert(dt); connection.Close(); } }
public static TimeSpan Insert(List <Temporary_LoadTestForTransactionBulkInsert> trx) { var timer = new Stopwatch(); using var context = new YourContext(); var bulk = new BulkOperation <Temporary_LoadTestForTransactionBulkInsert>(context.Database.GetDbConnection()); if (bulk.Connection.State != System.Data.ConnectionState.Open) { bulk.Connection.Open(); } timer.Start(); bulk.ColumnInputExpression = c => new { c.Amount, c.BonusId, c.CreationTime, c.CurrencyId, c.DeviceType, c.GameId, c.OperationType, c.OriginalTxId, c.Rake, c.RoundId, c.WinType, c.SessionId, c.ProviderTransactionId, c.PlayerId }; bulk.ColumnOutputExpression = c => c.Id; bulk.BulkMerge(trx); timer.Stop(); return(timer.Elapsed); }
public override BulkOperation <TOperationEntity> Apply <TOperationEntity>( IRequestConfig config, BulkOperation <TOperationEntity> operation) { operation = base.Apply(config, operation); if (IgnoredColumns.Count > 0) { if (operation.IgnoreOnInsertExpression != null) { foreach (var member in ((NewExpression)operation.IgnoreOnInsertExpression.Body).Members) { IgnoredColumns.Add(member); } } operation.IgnoreOnInsertExpression = CreateNewExpression <TOperationEntity>(IgnoredColumns); } return(operation); }
public static void Configure <TEntity>(this BulkOperation <TEntity> operation, BulkConfigurationType type, DataContext <TEntity> context) where TEntity : class { var typeConfigs = BulkConfigurationManager.Configurations(type); if (!typeConfigs.ContainsKey(context.Configuration.RequestType)) { typeConfigs[context.Configuration.RequestType] = new BulkRequestEntityConfigurationMap(); } var requestConfigs = typeConfigs[context.Configuration.RequestType]; if (!requestConfigs.ContainsKey(typeof(TEntity))) { requestConfigs[typeof(TEntity)] = new DefaultBulkConfiguration <TEntity>(); } var configurations = new List <IBulkConfiguration>(); foreach (var tRequest in context.Configuration.RequestType.BuildTypeHierarchyDown()) { if (!typeConfigs.TryGetValue(tRequest, out requestConfigs)) { continue; } foreach (var tEntity in typeof(TEntity).BuildTypeHierarchyDown()) { if (requestConfigs.TryGetValue(tEntity, out var entityConfigs)) { configurations.Add(entityConfigs); } } } foreach (var configuration in configurations) { configuration.Apply(context.Configuration, operation); } }
private void BodyPrimaryKeyMapping(BulkOperation operation) { if (operation is BulkOperation <CelestialBody> ) { var bulk = (BulkOperation <CelestialBody>)operation; bulk.ColumnPrimaryKeyExpression = c => c.EdsmId; } if (operation is BulkOperation <CelestialBodyAtmosphereComposition> ) { var bulk = (BulkOperation <CelestialBodyAtmosphereComposition>)operation; bulk.ColumnPrimaryKeyExpression = c => new { c.BodyId, c.Key }; } if (operation is BulkOperation <CelestialBodyBelt> ) { var bulk = (BulkOperation <CelestialBodyBelt>)operation; bulk.ColumnPrimaryKeyExpression = c => new { c.BodyId, c.Name }; } if (operation is BulkOperation <CelestialBodyMaterial> ) { var bulk = (BulkOperation <CelestialBodyMaterial>)operation; bulk.ColumnPrimaryKeyExpression = c => new { c.BodyId, c.Key }; } if (operation is BulkOperation <CelestialBodyParent> ) { var bulk = (BulkOperation <CelestialBodyParent>)operation; bulk.ColumnPrimaryKeyExpression = c => new { c.BodyId, c.Key }; } if (operation is BulkOperation <CelestialBodyRing> ) { var bulk = (BulkOperation <CelestialBodyRing>)operation; bulk.ColumnPrimaryKeyExpression = c => new { c.BodyId, c.Name }; } if (operation is BulkOperation <CelestialBodySolidComposition> ) { var bulk = (BulkOperation <CelestialBodySolidComposition>)operation; bulk.ColumnPrimaryKeyExpression = c => new { c.BodyId, c.Key }; } }
/// <summary> /// Initializes a new instance of the <see cref="BulkOperation" /> class. /// </summary> /// <param name="key">The key.</param> /// <param name="bulkAction">The bulk action.</param> public static BulkOperation Register(string key, Action <object> bulkAction) { var data = CallContext.LogicalGetData(CallContextKey) as Dictionary <string, BulkOperation>; if (data == null) { data = new Dictionary <string, BulkOperation>( ); CallContext.LogicalSetData(CallContextKey, data); } BulkOperation bulkOperation; if (!data.TryGetValue(key, out bulkOperation)) { bulkOperation = new BulkOperation(key, bulkAction); data[key] = bulkOperation; } bulkOperation.RefCount++; return(bulkOperation); }
public BulkRegistrationDocumentType CreateOperationRecord() { var taxpayerIdentificationNumber = MapTaxpayerIdentificationNumber(); var bulkRegistrationHeader = new BulkRegistrationHeaderType { ElectronicPostmark = new BulkRegistrationHeaderTypeElectronicPostmark { CSPID = Contractor.CspCode, Value = DateTime.UtcNow }, FilingType = BulkOperation.FilingType, TIN = taxpayerIdentificationNumber }; var bulkRegistrationType = BulkOperation.CreateBulkRegistrationType(this); var document = new BulkRegistrationDocumentType { BulkRegistrationHeader = bulkRegistrationHeader, DocumentType = BulkOperation.DocumentType, Item = bulkRegistrationType }; return(document); }
public override BulkOperation <TOperationEntity> Apply <TOperationEntity>( IRequestConfig config, BulkOperation <TOperationEntity> operation) { operation = base.Apply(config, operation); operation.AllowDuplicateKeys = _allowDuplicateKeys; operation.CaseSensitive = _keyCaseSensitivity; if (IgnoredColumns.Count > 0) { if (operation.IgnoreOnUpdateExpression != null) { foreach (var member in ((NewExpression)operation.IgnoreOnUpdateExpression.Body).Members) { IgnoredColumns.Add(member); } } operation.IgnoreOnUpdateExpression = CreateNewExpression <TOperationEntity>(IgnoredColumns); } return(operation); }
/// <summary>Maps.</summary> /// <param name="bulkOperation">The bulk operation.</param> /// <param name="config">The configuration.</param> public void Map(BulkOperation bulkOperation, DapperPlusEntityMapper config) { var isMapModified = config.IsMapModified(); var isKeyModified = config.IsKeyModified(); var isOutputModified = config.IsOutputModified(); var isIdentityModified = config.IsIdentityModified(); var isIgnoreModified = config.IsIgnoreModified(); var isModified = config._columnMappings == null || isMapModified.HasValue && isMapModified.Value || isKeyModified.HasValue && isKeyModified.Value || isOutputModified.HasValue && isOutputModified.Value || isIdentityModified.HasValue && isIdentityModified.Value || isIgnoreModified.HasValue && isIgnoreModified.Value; if (!isModified) { return; } config._mapAccessors = config.GetMapAccessors(); config._keyAccessors = config.GetKeyAccessors(); config._outputAccessors = config.GetOutputAccessors(); config._identityAccessors = config.GetIdentityAccessors(); config._ignoreAccessors = config.GetIgnoreAccessors(); // SET AutoMap value AutoMap(config); var columnMappings = new List <DapperPlusColumnMapping>(); config._columnMappings = columnMappings; foreach (var accessor in config._mapAccessors) { if (accessor.IsCalculated) { // NOT Supported yet } else { columnMappings.Add(new DapperPlusColumnMapping { SourceName = accessor.ToString(), DestinationName = accessor.Member, Input = true }); } } foreach (var accessor in config._outputAccessors) { var columnMapping = columnMappings.Find(x => x.SourceName == accessor.ToString() && x.DestinationName == accessor.Member); if (columnMapping != null) { columnMapping.Output = true; } else { // CHECK smart mapping if (string.IsNullOrEmpty(accessor.Member) && columnMappings.Count(x => x.SourceName == accessor.ToString()) == 1) { columnMapping = columnMappings.Find(x => x.SourceName == accessor.ToString()); columnMapping.Output = true; } else { columnMappings.Add(new DapperPlusColumnMapping { SourceName = accessor.ToString(), DestinationName = accessor.Member, Output = true }); } } } foreach (var accessor in config._keyAccessors) { var columnMapping = columnMappings.Find(x => x.SourceName == accessor.ToString() && x.DestinationName == accessor.Member); if (columnMapping != null) { columnMapping.IsPrimaryKey = true; } else { // CHECK smart mapping if (string.IsNullOrEmpty(accessor.Member) && columnMappings.Count(x => x.SourceName == accessor.ToString()) == 1) { columnMapping = columnMappings.Find(x => x.SourceName == accessor.ToString()); columnMapping.IsPrimaryKey = true; } else { columnMappings.Add(new DapperPlusColumnMapping { SourceName = accessor.ToString(), DestinationName = accessor.Member, IsPrimaryKey = true }); } } } foreach (var accessor in config._identityAccessors) { var columnMapping = columnMappings.Find(x => x.SourceName == accessor.ToString() && x.DestinationName == accessor.Member); if (columnMapping != null) { columnMapping.IsIdentity = true; } else { // CHECK smart mapping if (string.IsNullOrEmpty(accessor.Member) && columnMappings.Count(x => x.SourceName == accessor.ToString()) == 1) { columnMapping = columnMappings.Find(x => x.SourceName == accessor.ToString()); columnMapping.IsIdentity = true; } else { columnMappings.Add(new DapperPlusColumnMapping { SourceName = accessor.ToString(), DestinationName = accessor.Member, IsIdentity = true }); } } } config._isIdentityModified = false; config._isIgnoreModified = false; config._isKeyModified = false; config._isMapModified = false; config._isOutputModified = false; }
/// <summary>Executes this object.</summary> public void Execute() { if (IsExecuted) { return; } if (DataSource == null) { IsExecuted = true; return; } var bulkOperation = new BulkOperation { Connection = (DbConnection)Connection }; var mapperKey = ""; var enumerableDataSource = DataSource as IEnumerable <object>; if (enumerableDataSource != null) { var first = enumerableDataSource.FirstOrDefault(); if (first == null) { IsExecuted = true; return; } var enumerableFirst = first as IEnumerable <object>; if (enumerableFirst != null) { // List<List<Entity>> var list = enumerableDataSource.Where(x => x != null).Cast <IEnumerable <object> >().SelectMany(x => x).ToList(); var firstElement = list[0]; // GET mapper key mapperKey = DapperPlusManager.GetFullMapperKey(firstElement.GetType(), Key); // Convert DataSource to List<Entity> var castMethod = typeof(Enumerable).GetMethod("Cast"); castMethod = castMethod.MakeGenericMethod(firstElement.GetType()); var toListMethod = typeof(Enumerable).GetMethod("ToList"); toListMethod = toListMethod.MakeGenericMethod(firstElement.GetType()); var obj = castMethod.Invoke(null, new[] { list }); obj = toListMethod.Invoke(null, new[] { obj }); DataSource = obj; } else if (DataSource.GetType().GetGenericArguments()[0] == typeof(object)) { // List<object> => List<Entity> var type = first.GetType(); // GET mapper key mapperKey = DapperPlusManager.GetFullMapperKey(type, Key); // Convert DataSource to List<Entity> var castMethod = typeof(Enumerable).GetMethod("Cast"); castMethod = castMethod.MakeGenericMethod(type); var toListMethod = typeof(Enumerable).GetMethod("ToList"); toListMethod = toListMethod.MakeGenericMethod(type); var obj = castMethod.Invoke(null, new[] { DataSource }); obj = toListMethod.Invoke(null, new[] { obj }); DataSource = obj; } else { // List<Entity> // GET mapper key mapperKey = DapperPlusManager.GetFullMapperKey(first.GetType(), Key); } } else { // item // GET mapper key mapperKey = DapperPlusManager.GetFullMapperKey(DataSource.GetType(), Key); // CREATE list var list = new List <object> { DataSource }; // Convert List<object> to List<Entity> var castMethod = typeof(Enumerable).GetMethod("Cast"); castMethod = castMethod.MakeGenericMethod(DataSource.GetType()); var toListMethod = typeof(Enumerable).GetMethod("ToList"); toListMethod = toListMethod.MakeGenericMethod(DataSource.GetType()); var obj = castMethod.Invoke(null, new[] { list }); obj = toListMethod.Invoke(null, new[] { obj }); DataSource = obj; } DapperPlusEntityMapper config; if (!DapperPlusManager.MapperCache.TryGetValue(mapperKey, out config)) { var elementType = DataSource.GetType().GetElementType(); var constructor = typeof(DapperPlusEntityMapper <>).MakeGenericType(elementType).GetConstructor(new Type[0]); config = (DapperPlusEntityMapper)constructor.Invoke(new object[0]); } bulkOperation.DataSource = DataSource; bulkOperation.AllowDuplicateKeys = true; bulkOperation.CaseSensitive = CaseSensitiveType.DestinationInsensitive; if (Kind == DapperPlusActionKind.Insert) { ApplyConfig(bulkOperation, config._configInsert); bulkOperation.BulkInsert(); } else if (Kind == DapperPlusActionKind.Update) { ApplyConfig(bulkOperation, config._configInsert); bulkOperation.BulkUpdate(); } else if (Kind == DapperPlusActionKind.Delete) { ApplyConfig(bulkOperation, config._configInsert); bulkOperation.BulkDelete(); } else if (Kind == DapperPlusActionKind.Merge) { ApplyConfig(bulkOperation, config._configInsert); bulkOperation.BulkMerge(); } IsExecuted = true; }
/// <summary>Applies the configuration.</summary> /// <param name="bulkOperation">The bulk operation.</param> /// <param name="config">The configuration.</param> public void ApplyConfig(BulkOperation bulkOperation, DapperPlusEntityMapper config) { if (config == null) { return; } // Verify Column Mappings Map(bulkOperation, config); // Batch { var batchDelayInterval = config.BatchDelayInterval(); var batchSize = config.BatchSize(); var batchTimeout = config.BatchTimeout(); if (batchDelayInterval.HasValue) { bulkOperation.BatchDelayInterval = batchDelayInterval.Value; } if (batchSize.HasValue) { bulkOperation.BatchSize = batchSize.Value; } if (batchTimeout.HasValue) { bulkOperation.BatchTimeout = batchTimeout.Value; } } // Destination { var table = config.Table(); if (!string.IsNullOrEmpty(table)) { bulkOperation.DestinationTableName = table; } } // SqlServer { var sqlBulkCopyOptions = config.SqlBulkCopyOptions(); if (sqlBulkCopyOptions.HasValue) { bulkOperation.SqlBulkCopyOptions = sqlBulkCopyOptions.Value; } } // TemproaryTable { var temporaryTableBatchByTable = config.TemporaryTableBatchByTable(); var temporaryTableInsertBatchSize = config.TemporaryTableInsertBatchSize(); var temporaryTableMinRecord = config.TemporaryTableMinRecord(); var temporaryTableSchemaName = config.TemporaryTableSchemaName(); if (temporaryTableBatchByTable.HasValue) { bulkOperation.TemporaryTableBatchByTable = temporaryTableBatchByTable.Value; } if (temporaryTableInsertBatchSize.HasValue) { bulkOperation.TemporaryTableInsertBatchSize = temporaryTableInsertBatchSize.Value; } if (temporaryTableMinRecord.HasValue) { bulkOperation.TemporaryTableMinRecord = temporaryTableMinRecord.Value; } if (!string.IsNullOrEmpty(temporaryTableSchemaName)) { bulkOperation.TemporaryTableSchemaName = temporaryTableSchemaName; } } // Transient { var retryCount = config.RetryCount(); var retryInterval = config.RetryInterval(); if (retryCount.HasValue) { bulkOperation.RetryCount = retryCount.Value; } if (retryInterval.HasValue) { bulkOperation.RetryInterval = new TimeSpan(0, 0, 0, 0, retryInterval.Value); } } // Column Mapping { foreach (var column in config._columnMappings) { var bulkMapping = new ColumnMapping { SourceName = column.SourceName, DestinationName = column.DestinationName }; if (column.IsPrimaryKey) { bulkMapping.IsPrimaryKey = true; } if (column.Input && column.Output) { bulkMapping.Direction = ColumnMappingDirectionType.InputOutput; } else if (column.Output) { bulkMapping.Direction = ColumnMappingDirectionType.Output; } if (column.IsIdentity && (config == config._masterConfig._configInsert) || (config == config._masterConfig._configMerge)) { bulkMapping.Direction = ColumnMappingDirectionType.Output; } bulkOperation.ColumnMappings.Add(bulkMapping); } } }
public virtual BulkOperation <TOperationEntity> Apply <TOperationEntity>( IRequestConfig config, BulkOperation <TOperationEntity> operation) where TOperationEntity : class { var keyColumn = operation.ColumnMappings.Find(x => x.IsPrimaryKey); if (KeyColumnMapping != null) { if (keyColumn != null) { operation.ColumnMappings.Remove(keyColumn); } } else { KeyColumnMapping = keyColumn != null ? UpcastExpression(keyColumn.SourceExpression) : GetDefaultKeyMapping(config); } if (keyColumn == null) { if (KeyColumnMapping == null) { var message = $"No key column has been declared for bulk request '{config.RequestType}' " + $"and entity '{typeof(TOperationEntity)}'."; throw new BadConfigurationException(message); } operation.ColumnMappings.Add(DowncastExpression <TOperationEntity>(KeyColumnMapping), true); } if (ColumnMappings.Count == 0) { AddDefaultColumnMappings(operation.ColumnMappings); } foreach (var mapping in ColumnMappings) { operation.ColumnMappings.Add(DowncastExpression <TOperationEntity>(mapping), false); } if (OutputExpression == null) { OutputExpression = KeyColumnMapping; if (operation.ColumnOutputExpression == null) { operation.ColumnOutputExpression = DowncastExpression <TOperationEntity>(OutputExpression); } } if (BatchSize.HasValue) { operation.BatchSize = BatchSize.Value; } if (BatchTimeout.HasValue) { operation.BatchTimeout = BatchTimeout.Value; } return(operation); }
/// <summary>Executes this object.</summary> public void Execute() { if (IsExecuted) return; if (DataSource == null) { IsExecuted = true; return; } var bulkOperation = new BulkOperation { Connection = (DbConnection) Connection }; var mapperKey = ""; var enumerableDataSource = DataSource as IEnumerable<object>; if (enumerableDataSource != null) { var first = enumerableDataSource.FirstOrDefault(); if (first == null) { IsExecuted = true; return; } var enumerableFirst = first as IEnumerable<object>; if (enumerableFirst != null) { // List<List<Entity>> var list = enumerableDataSource.Where(x => x != null).Cast<IEnumerable<object>>().SelectMany(x => x).ToList(); var firstElement = list[0]; // GET mapper key mapperKey = DapperPlusManager.GetFullMapperKey(firstElement.GetType(), Key); // Convert DataSource to List<Entity> var castMethod = typeof (Enumerable).GetMethod("Cast"); castMethod = castMethod.MakeGenericMethod(firstElement.GetType()); var toListMethod = typeof (Enumerable).GetMethod("ToList"); toListMethod = toListMethod.MakeGenericMethod(firstElement.GetType()); var obj = castMethod.Invoke(null, new[] {list}); obj = toListMethod.Invoke(null, new[] {obj}); DataSource = obj; } else if (DataSource.GetType().GetGenericArguments()[0] == typeof (object)) { // List<object> => List<Entity> var type = first.GetType(); // GET mapper key mapperKey = DapperPlusManager.GetFullMapperKey(type, Key); // Convert DataSource to List<Entity> var castMethod = typeof (Enumerable).GetMethod("Cast"); castMethod = castMethod.MakeGenericMethod(type); var toListMethod = typeof (Enumerable).GetMethod("ToList"); toListMethod = toListMethod.MakeGenericMethod(type); var obj = castMethod.Invoke(null, new[] {DataSource}); obj = toListMethod.Invoke(null, new[] {obj}); DataSource = obj; } else { // List<Entity> // GET mapper key mapperKey = DapperPlusManager.GetFullMapperKey(first.GetType(), Key); } } else { // item // GET mapper key mapperKey = DapperPlusManager.GetFullMapperKey(DataSource.GetType(), Key); // CREATE list var list = new List<object> {DataSource}; // Convert List<object> to List<Entity> var castMethod = typeof (Enumerable).GetMethod("Cast"); castMethod = castMethod.MakeGenericMethod(DataSource.GetType()); var toListMethod = typeof (Enumerable).GetMethod("ToList"); toListMethod = toListMethod.MakeGenericMethod(DataSource.GetType()); var obj = castMethod.Invoke(null, new[] {list}); obj = toListMethod.Invoke(null, new[] {obj}); DataSource = obj; } DapperPlusEntityMapper config; if (!DapperPlusManager.MapperCache.TryGetValue(mapperKey, out config)) { var elementType = DataSource.GetType().GetElementType(); var constructor = typeof (DapperPlusEntityMapper<>).MakeGenericType(elementType).GetConstructor(new Type[0]); config = (DapperPlusEntityMapper) constructor.Invoke(new object[0]); } bulkOperation.DataSource = DataSource; bulkOperation.AllowDuplicateKeys = true; bulkOperation.CaseSensitive = CaseSensitiveType.DestinationInsensitive; if (Kind == DapperPlusActionKind.Insert) { ApplyConfig(bulkOperation, config._configInsert); bulkOperation.BulkInsert(); } else if (Kind == DapperPlusActionKind.Update) { ApplyConfig(bulkOperation, config._configInsert); bulkOperation.BulkUpdate(); } else if (Kind == DapperPlusActionKind.Delete) { ApplyConfig(bulkOperation, config._configInsert); bulkOperation.BulkDelete(); } else if (Kind == DapperPlusActionKind.Merge) { ApplyConfig(bulkOperation, config._configInsert); bulkOperation.BulkMerge(); } IsExecuted = true; }
/// <summary>Executes this object.</summary> public void Execute() { if (IsExecuted) { return; } if (DataSource == null) { IsExecuted = true; return; } var bulkOperation = new BulkOperation { Connection = (DbConnection)Connection }; var mapperKey = ""; var enumerableDataSource = DataSource as IEnumerable <object>; if (enumerableDataSource != null) { var first = enumerableDataSource.FirstOrDefault(); if (first == null) { IsExecuted = true; return; } var enumerableFirst = first as IEnumerable <object>; if (enumerableFirst != null) { // List<List<Entity>> var list = enumerableDataSource.Where(x => x != null).Cast <IEnumerable <object> >().SelectMany(x => x).ToList(); var firstElement = list[0]; // GET mapper key mapperKey = DapperPlusManager.GetFullMapperKey(firstElement.GetType(), Key); // Convert DataSource to List<Entity> var castMethod = typeof(Enumerable).GetMethod("Cast"); castMethod = castMethod.MakeGenericMethod(firstElement.GetType()); var toListMethod = typeof(Enumerable).GetMethod("ToList"); toListMethod = toListMethod.MakeGenericMethod(firstElement.GetType()); var obj = castMethod.Invoke(null, new[] { list }); obj = toListMethod.Invoke(null, new[] { obj }); DataSource = obj; } else if (DataSource.GetType().GetGenericArguments()[0] == typeof(object)) { // List<object> => List<Entity> var type = first.GetType(); // GET mapper key mapperKey = DapperPlusManager.GetFullMapperKey(type, Key); // Convert DataSource to List<Entity> var castMethod = typeof(Enumerable).GetMethod("Cast"); castMethod = castMethod.MakeGenericMethod(type); var toListMethod = typeof(Enumerable).GetMethod("ToList"); toListMethod = toListMethod.MakeGenericMethod(type); var obj = castMethod.Invoke(null, new[] { DataSource }); obj = toListMethod.Invoke(null, new[] { obj }); DataSource = obj; } else { // List<Entity> // GET mapper key mapperKey = DapperPlusManager.GetFullMapperKey(first.GetType(), Key); } } else { // item // GET mapper key mapperKey = DapperPlusManager.GetFullMapperKey(DataSource.GetType(), Key); // CREATE list var list = new List <object> { DataSource }; // Convert List<object> to List<Entity> var castMethod = typeof(Enumerable).GetMethod("Cast"); castMethod = castMethod.MakeGenericMethod(DataSource.GetType()); var toListMethod = typeof(Enumerable).GetMethod("ToList"); toListMethod = toListMethod.MakeGenericMethod(DataSource.GetType()); var obj = castMethod.Invoke(null, new[] { list }); obj = toListMethod.Invoke(null, new[] { obj }); DataSource = obj; } DapperPlusEntityMapper config; if (!DapperPlusManager.MapperCache.TryGetValue(mapperKey, out config)) { // CHECK for Entity Framework Proxy Type if (mapperKey.StartsWith("zzz_proxy;")) { var mapperProxy = new List <DapperPlusEntityMapper>(); // Try to find if one mapping could correspond foreach (var keyValue in DapperPlusManager.MapperCache) { var key = keyValue.Key; var prefix = string.IsNullOrEmpty(Key) ? "zzz_null" : Key; // MUST start with the same suffix if (!key.StartsWith(prefix)) { continue; } var suffix = key.Split('.').Last().Split('+').Last(); var mapperSuffix = mapperKey.Split(';').Last(); if (suffix.Length < 20) { // MUST BE Equal if (suffix != mapperSuffix) { continue; } } else { // MUST START with same name but only one! if (!suffix.StartsWith(mapperSuffix)) { continue; } } mapperProxy.Add(keyValue.Value); } if (mapperProxy.Count == 1) { config = mapperProxy[0]; } } if (config == null) { if (DapperPlusManager.ThrowErrorIfNotMapped) { StringBuilder sb = new StringBuilder(); sb.AppendLine("Mapping Not Found!"); sb.AppendLine("Current MapperKey: " + mapperKey); sb.AppendLine("Possible Mapping:"); foreach (var keyValue in DapperPlusManager.MapperCache) { sb.AppendLine(" - " + keyValue.Key); } throw new Exception(sb.ToString()); } else { var type = DataSource.GetType(); var elementType = type.GetGenericArguments()[0]; var constructor = typeof(DapperPlusEntityMapper <>).MakeGenericType(elementType).GetConstructor(new Type[0]); config = (DapperPlusEntityMapper)constructor.Invoke(new object[0]); } } } bulkOperation._isDapperPlus = true; bulkOperation.DataSource = DataSource; bulkOperation.AllowDuplicateKeys = true; bulkOperation.CaseSensitive = CaseSensitiveType.DestinationInsensitive; if (Kind == DapperPlusActionKind.Insert) { ApplyConfig(bulkOperation, config._configInsert); bulkOperation.BulkInsert(); } else if (Kind == DapperPlusActionKind.Update) { ApplyConfig(bulkOperation, config._configInsert); bulkOperation.BulkUpdate(); } else if (Kind == DapperPlusActionKind.Delete) { ApplyConfig(bulkOperation, config._configInsert); bulkOperation.BulkDelete(); } else if (Kind == DapperPlusActionKind.Merge) { ApplyConfig(bulkOperation, config._configInsert); bulkOperation.BulkMerge(); } IsExecuted = true; }
//Called upon board creation, saves squares to DB public bool SaveBoard(Model sudoku) { DataTable gridDataTable = new DataTable("grid_square"); gridDataTable.Columns.Add(new DataColumn("gridID", typeof(string))); gridDataTable.Columns.Add(new DataColumn("columnNum", typeof(string))); gridDataTable.Columns.Add(new DataColumn("rowNum", typeof(string))); gridDataTable.Columns.Add(new DataColumn("entryNum", typeof(string))); DataTable blankDataTable = new DataTable("start_empty"); blankDataTable.Columns.Add(new DataColumn("gridID", typeof(string))); blankDataTable.Columns.Add(new DataColumn("columnNum", typeof(string))); blankDataTable.Columns.Add(new DataColumn("rowNum", typeof(string))); for (int cIndex = 0; cIndex < sudoku.SolutionBoard.GetLength(0); cIndex++) { for (int rIndex = 0; rIndex < sudoku.SolutionBoard.GetLength(1); rIndex++) { //Add each square to the grid data table gridDataTable.Rows.Add(new string[] { sudoku.GameId.ToString(), cIndex.ToString(), rIndex.ToString(), sudoku.SolutionBoard[cIndex, rIndex].Substring(0, 1) }); //If the square is blank in the play board, add it to the blank data table if (sudoku.PlayBoard[cIndex, rIndex] == "--") { blankDataTable.Rows.Add(new string[] { sudoku.GameId.ToString(), cIndex.ToString(), rIndex.ToString() }); } } } using (MySqlConnection connection = new MySqlConnection(Helper.ConnectionVal("SudokuCloudDB"))) { connection.Open(); connection.Query($"INSERT INTO full_grid (gridID, playerID, difficulty, hintsRemaining) VALUES ({sudoku.GameId}, 1, {sudoku.Difficulty}, {sudoku.Hints})"); var bulkGrid = new BulkOperation(connection); bulkGrid.BulkInsert(gridDataTable); var bulkBlank = new BulkOperation(connection); bulkBlank.BulkInsert(blankDataTable); connection.Close(); } //Iterate over solution board and call SaveSquare on each square //for (int cIndex = 0; cIndex < sudoku.SolutionBoard.GetLength(0); cIndex++) //{ // for (int rIndex = 0; rIndex < sudoku.SolutionBoard.GetLength(1); rIndex++) // { // try // { // int entry = int.Parse(sudoku.SolutionBoard[cIndex, rIndex].Substring(0, 1)); // if (!SaveSquare(sudoku, false, cIndex, rIndex, entry)) // { // return false; // } // //Check this location in the play board. If it's blank // if (sudoku.PlayBoard[cIndex, rIndex] == "--") // { // SaveSquare(sudoku, true, cIndex, rIndex, 0); // } // } // catch (FormatException) // { // Console.WriteLine("ERROR: Fault saving grid entry. " + cIndex + "-" + rIndex + " is not a number."); // return false; // } // } //} return(true); }
/// <summary>Maps.</summary> /// <param name="bulkOperation">The bulk operation.</param> /// <param name="config">The configuration.</param> public void Map(BulkOperation bulkOperation, DapperPlusEntityMapper config) { var isMapModified = config.IsMapModified(); var isKeyModified = config.IsKeyModified(); var isOutputModified = config.IsOutputModified(); var isIdentityModified = config.IsIdentityModified(); var isIgnoreModified = config.IsIgnoreModified(); var isModified = isMapModified.HasValue && isMapModified.Value || isKeyModified.HasValue && isKeyModified.Value || isOutputModified.HasValue && isOutputModified.Value || isIdentityModified.HasValue && isIdentityModified.Value || isIgnoreModified.HasValue && isIgnoreModified.Value; if (!isModified) return; config._mapAccessors = config.GetMapAccessors(); config._keyAccessors = config.GetKeyAccessors(); config._outputAccessors = config.GetOutputAccessors(); config._identityAccessors = config.GetIdentityAccessors(); config._ignoreAccessors = config.GetIgnoreAccessors(); // SET AutoMap value AutoMap(config); var columnMappings = new List<DapperPlusColumnMapping>(); config._columnMappings = columnMappings; foreach (var accessor in config._mapAccessors) { if (accessor.IsCalculated) { // NOT Supported yet } else { columnMappings.Add(new DapperPlusColumnMapping {SourceName = accessor.ToString(), DestinationName = accessor.Member, Input = true}); } } foreach (var accessor in config._outputAccessors) { var columnMapping = columnMappings.Find(x => x.SourceName == accessor.ToString() && x.DestinationName == accessor.Member); if (columnMapping != null) { columnMapping.Output = true; } else { columnMappings.Add(new DapperPlusColumnMapping {SourceName = accessor.ToString(), DestinationName = accessor.Member, Output = true}); } } foreach (var accessor in config._keyAccessors) { var columnMapping = columnMappings.Find(x => x.SourceName == accessor.ToString() && x.DestinationName == accessor.Member); if (columnMapping != null) { columnMapping.IsPrimaryKey = true; } else { columnMappings.Add(new DapperPlusColumnMapping {SourceName = accessor.ToString(), DestinationName = accessor.Member, IsPrimaryKey = true}); } } foreach (var accessor in config._identityAccessors) { var columnMapping = columnMappings.Find(x => x.SourceName == accessor.ToString() && x.DestinationName == accessor.Member); if (columnMapping != null) { columnMapping.IsIdentity = true; } else { columnMappings.Add(new DapperPlusColumnMapping {SourceName = accessor.ToString(), DestinationName = accessor.Member, IsIdentity = true}); } } config._isIdentityModified = false; config._isIgnoreModified = false; config._isKeyModified = false; config._isMapModified = false; config._isOutputModified = false; }
public async Task CreateConfigIfNotExistsAsync2(int sqlConfigId, SqlConfigDetailDto[] listdata) { try { var mapperData = ObjectMapper.Map <SqlConfigDetail[]>(listdata); var oldConfig = GetColumnConfigBySqlId(sqlConfigId).ToArray(); var oldConfigMap = oldConfig.Select(x => x.Code); var newConfig = listdata.Select(x => x.Code); using (var conn = new SqlConnection(this.connectionString)) { if (conn.State == ConnectionState.Closed) { await conn.OpenAsync(); } if (oldConfig.Length > 0 && oldConfigMap != newConfig) { using (var bulk = new BulkOperation <SqlConfigDetail>(conn)) { bulk.DestinationTableName = "SqlConfigDetail"; bulk.InsertIfNotExists = true; bulk.ColumnPrimaryKeyExpression = c => c.Id; bulk.BatchSize = 10000; bulk.BatchTimeout = 10000; bulk.ColumnPrimaryKeyExpression = c => new { c.Code }; //bulk.AutoMapValueFactory = //bulk.ColumnInputExpression = c => new //{ // c.LastModificationTime, // c.LastModifierUserId, // c.IsDisplay, // c.IsSum, // c.GroupLevel, // c.Width, // c.Name, // c.GroupSort, // c.Type, // c.Format, // c.ColNum //}; await bulk.BulkInsertAsync(mapperData); } } else { using (var bulk = new BulkOperation <SqlConfigDetail>(conn)) { bulk.DestinationTableName = "SqlConfigDetail"; bulk.InsertIfNotExists = true; bulk.ColumnPrimaryKeyExpression = c => c.Id; bulk.BatchSize = 10000; bulk.BatchTimeout = 10000; //bulk.AutoMapValueFactory = bulk.ColumnInputExpression = c => new { c.LastModificationTime, c.LastModifierUserId, c.IsDisplay, c.IsSum, c.GroupLevel, c.Width, c.Name, c.GroupSort, c.Type, c.Format, c.ColNum }; await bulk.BulkUpdateAsync(mapperData); } } } } catch (Exception ex) { System.Diagnostics.Debug.WriteLine("Error: ", ex); } }
// TODO Multi-Get API public static async Task Bulk(this ElasticsearchClient client, BulkOperation bulkOperation) { await Bulk(client, bulkOperation.BulkOperationItems); }