protected virtual async Task OnPostBulkInsert(SqlConnection sqlConnection, TargetTable target, T[] insertedData) { if (m_postBulkInsert != null) { await m_postBulkInsert(sqlConnection, m_targetTable, insertedData).ConfigureAwait(false); } }
public DbBulkInserter(TargetTable targetTable, DataflowOptions options, int bulkSize = 8192, string dbBulkInserterName = null, PostBulkInsertDelegate <T> postBulkInsert = null) : base(targetTable, options, bulkSize, dbBulkInserterName, postBulkInsert) { m_longConnection = new SqlConnection(targetTable.ConnectionString); m_longConnection.Open(); m_transaction = m_longConnection.BeginTransaction(); }
/// <summary> /// Constructs an instance of DbBulkInserter /// </summary> /// <param name="targetTable">Information about the database target and mapping label</param> /// <param name="options">Options to use for this dataflow</param> /// <param name="bulkSize">The bulk size to insert in a batch. Default to 8192.</param> /// <param name="dbBulkInserterName">A given name of this bulk inserter (would be nice for logging)</param> /// <param name="postBulkInsert">A delegate that enables you to inject some customized work whenever a bulk insert is done</param> public DbBulkInserterBase(TargetTable targetTable, DataflowOptions options, int bulkSize = 4096 * 2, string dbBulkInserterName = null, PostBulkInsertDelegate <T> postBulkInsert = null) : base(options) { this.m_targetTable = targetTable; this.m_typeAccessor = TypeAccessorManager <T> .GetAccessorForTable(targetTable); this.m_bulkSize = bulkSize; this.m_dbBulkInserterName = dbBulkInserterName; this.m_postBulkInsert = postBulkInsert; this.m_batchBlock = new BatchBlock <T>(bulkSize, options.ToGroupingBlockOption()); var bulkInsertOption = options.ToExecutionBlockOption(); //Action block deal with array references if (bulkInsertOption.BoundedCapacity != DataflowBlockOptions.Unbounded) { bulkInsertOption.BoundedCapacity = bulkInsertOption.BoundedCapacity / bulkSize; } this.m_logger = Utils.GetNamespaceLogger(); this.m_actionBlock = new ActionBlock <T[]>( async array => { m_logger.Debug(h => h("{3} starts bulk-inserting {0} {1} to db table {2}", array.Length, typeof(T).Name, targetTable.TableName, this.FullName)); try { await this.DumpToDBAsync(array, targetTable); } catch (Exception e) { m_logger.Error($"{this.FullName} failed bulk-inserting {array.Length} {typeof(T).Name} to db table {targetTable.TableName}", e); throw; } m_logger.Debug(h => h("{3} bulk-inserted {0} {1} to db table {2}", array.Length, typeof(T).Name, targetTable.TableName, this.FullName)); } , bulkInsertOption); this.m_batchBlock.LinkTo(this.m_actionBlock, this.m_defaultLinkOption); this.RegisterChild(this.m_batchBlock); this.RegisterChild(this.m_actionBlock); this.m_timer = new Timer( state => { this.TriggerBatch(); }, null, TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(10)); }
public TypeAccessor(TargetTable target) { m_destLabel = target.DestLabel; m_connectionString = target.ConnectionString; m_destinationTablename = string.IsNullOrWhiteSpace(target.TableName) ? typeof(T).Name : target.TableName; m_schemaTable = new Lazy <DataTable>(this.GetSchemaTable); m_properties = new Dictionary <int, Func <T, object> >(); m_dbColumnMappings = new List <DBColumnMapping>(); m_classLogger = LogManager.GetLogger(this.GetType().Namespace + "." + this.GetType().GetFriendlyName()); CreateTypeVisitor(); }
protected async virtual Task DumpToDBAsync(T[] data, TargetTable targetTable) { m_logger.Debug(h => h("{3} starts bulk-inserting {0} {1} to db table {2}", data.Length, typeof(T).Name, targetTable.TableName, this.FullName)); using (var bulkReader = new BulkDataReader <T>(m_typeAccessor, data)) { using (var conn = new SqlConnection(targetTable.ConnectionString)) { await conn.OpenAsync().ConfigureAwait(false); var transaction = conn.BeginTransaction(); try { using (var bulkCopy = new SqlBulkCopy(conn, SqlBulkCopyOptions.TableLock, transaction)) { foreach (SqlBulkCopyColumnMapping map in bulkReader.ColumnMappings) { bulkCopy.ColumnMappings.Add(map); } bulkCopy.DestinationTableName = targetTable.TableName; bulkCopy.BulkCopyTimeout = (int)TimeSpan.FromMinutes(30).TotalMilliseconds; bulkCopy.BatchSize = m_bulkSize; // Write from the source to the destination. await bulkCopy.WriteToServerAsync(bulkReader).ConfigureAwait(false); } } catch (Exception e) { if (e is NullReferenceException) { m_logger.ErrorFormat( "{0} NullReferenceException occurred in bulk insertion. This is probably caused by forgetting assigning value to a [NoNullCheck] attribute when constructing your object.", this.FullName); } m_logger.ErrorFormat("{0} Bulk insertion failed. Rolling back all changes...", this.FullName, e); transaction.Rollback(); m_logger.InfoFormat("{0} Changes successfully rolled back", this.FullName); //As this is an unrecoverable exception, rethrow it throw new AggregateException(e); } transaction.Commit(); await this.OnPostBulkInsert(conn, targetTable, data).ConfigureAwait(false); } } m_logger.Info(h => h("{3} bulk-inserted {0} {1} to db table {2}", data.Length, typeof(T).Name, targetTable.TableName, this.FullName)); }
protected override async Task DumpToDBAsync(T[] data, TargetTable targetTable) { m_logger.Debug(h => h("{3} starts bulk-inserting {0} {1} to db table {2}", data.Length, typeof(T).Name, targetTable.TableName, this.FullName)); using (var bulkReader = new BulkDataReader <T>(m_typeAccessor, data)) { try { using (var bulkCopy = new SqlBulkCopy(m_longConnection, SqlBulkCopyOptions.TableLock, m_transaction)) { foreach (SqlBulkCopyColumnMapping map in bulkReader.ColumnMappings) { bulkCopy.ColumnMappings.Add(map); } bulkCopy.DestinationTableName = targetTable.TableName; bulkCopy.BulkCopyTimeout = (int)TimeSpan.FromMinutes(30).TotalMilliseconds; bulkCopy.BatchSize = m_bulkSize; // Write from the source to the destination. await bulkCopy.WriteToServerAsync(bulkReader).ConfigureAwait(false); } } catch (Exception e) { if (e is NullReferenceException) { m_logger.ErrorFormat( "{0} NullReferenceException occurred in bulk insertion. This is probably caused by forgetting assigning value to a [NoNullCheck] attribute when constructing your object.", this.FullName); } m_logger.ErrorFormat("{0} Bulk insertion error in the first place", e, this.FullName); //As this is an unrecoverable exception, rethrow it throw new AggregateException(e); } await this.OnPostBulkInsert(m_longConnection, targetTable, data).ConfigureAwait(false); } m_logger.Info(h => h("{3} bulk-inserted {0} {1} to db table {2}", data.Length, typeof(T).Name, targetTable.TableName, this.FullName)); }
/// <summary> /// Constructs an instance of DbBulkInserter /// </summary> /// <param name="targetTable">Information about the database target and mapping label</param> /// <param name="options">Options to use for this dataflow</param> /// <param name="bulkSize">The bulk size to insert in a batch. Default to 8192.</param> /// <param name="dbBulkInserterName">A given name of this bulk inserter (would be nice for logging)</param> /// <param name="postBulkInsert">A delegate that enables you to inject some customized work whenever a bulk insert is done</param> public DbBulkInserterBase(TargetTable targetTable, DataflowOptions options, int bulkSize = 4096 * 2, string dbBulkInserterName = null, PostBulkInsertDelegate <T> postBulkInsert = null) : base(options) { this.m_targetTable = targetTable; this.m_typeAccessor = TypeAccessorManager <T> .GetAccessorForTable(targetTable); this.m_bulkSize = bulkSize; this.m_dbBulkInserterName = dbBulkInserterName; this.m_postBulkInsert = postBulkInsert; this.m_batchBlock = new BatchBlock <T>(bulkSize, options.ToGroupingBlockOption()); var bulkInsertOption = options.ToExecutionBlockOption(); //Action block deal with array references if (bulkInsertOption.BoundedCapacity != DataflowBlockOptions.Unbounded) { bulkInsertOption.BoundedCapacity = bulkInsertOption.BoundedCapacity / bulkSize; } this.m_actionBlock = new ActionBlock <T[]>(array => this.DumpToDBAsync(array, targetTable), bulkInsertOption); this.m_batchBlock.LinkTo(this.m_actionBlock, this.m_defaultLinkOption); this.m_logger = Utils.GetNamespaceLogger(); this.RegisterChild(this.m_batchBlock); this.RegisterChild(this.m_actionBlock); this.m_timer = new Timer( state => { this.TriggerBatch(); }, null, TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(10)); }
protected bool Equals(TargetTable other) { return string.Equals(this.ConnectionString, other.ConnectionString) && string.Equals(this.TableName, other.TableName) && string.Equals(this.DestLabel, other.DestLabel); }
/// <summary> /// If the typeAccessor exists, just return it; else create a new one with parameter: destLabel, connectionString, /// dataTableName /// </summary> public static TypeAccessor <T> GetAccessorForTable(TargetTable target) { return(s_accessors.GetOrAdd(target, t => new Lazy <TypeAccessor <T> >(() => new TypeAccessor <T>(t))).Value); }
public EagerDbBulkInserter(TargetTable targetTable, DataflowOptions options, int bulkSize = 8192, string dbBulkInserterName = null, PostBulkInsertDelegate <T> postBulkInsert = null) : base(targetTable, options, bulkSize, dbBulkInserterName, postBulkInsert) { }
protected abstract Task DumpToDBAsync(T[] data, TargetTable targetTable);
protected bool Equals(TargetTable other) { return(string.Equals(this.ConnectionString, other.ConnectionString) && string.Equals(this.TableName, other.TableName) && string.Equals(this.DestLabel, other.DestLabel)); }