/// <summary> /// Constructs an instance of DbBulkInserter /// </summary> /// <param name="connectionString">The connection string to the output database</param> /// <param name="destTable">The table name in database to bulk insert into</param> /// <param name="options">Options to use for this dataflow</param> /// <param name="destLabel">The mapping label to help choose among all column mappings</param> /// <param name="bulkSize">The bulk size to insert in a batch. Default to 8192.</param> /// <param name="dbBulkInserterName">A given name of this bulk inserter (would be nice for logging)</param> /// <param name="postBulkInsert">A delegate that enables you to inject some customized work whenever a bulk insert is done</param> public DbBulkInserterBase( string connectionString, string destTable, DataflowOptions options, string destLabel, int bulkSize = 4096 * 2, string dbBulkInserterName = null, PostBulkInsertDelegate <T> postBulkInsert = null) : this(new TargetTable(destLabel, connectionString, destTable), options, bulkSize, dbBulkInserterName, postBulkInsert) { }
/// <summary> /// Constructs an instance of DbBulkInserter /// </summary> /// <param name="targetTable">Information about the database target and mapping label</param> /// <param name="options">Options to use for this dataflow</param> /// <param name="bulkSize">The bulk size to insert in a batch. Default to 8192.</param> /// <param name="dbBulkInserterName">A given name of this bulk inserter (would be nice for logging)</param> /// <param name="postBulkInsert">A delegate that enables you to inject some customized work whenever a bulk insert is done</param> public DbBulkInserterBase(TargetTable targetTable, DataflowOptions options, int bulkSize = 4096 * 2, string dbBulkInserterName = null, PostBulkInsertDelegate <T> postBulkInsert = null) : base(options) { this.m_targetTable = targetTable; this.m_typeAccessor = TypeAccessorManager <T> .GetAccessorForTable(targetTable); this.m_bulkSize = bulkSize; this.m_dbBulkInserterName = dbBulkInserterName; this.m_postBulkInsert = postBulkInsert; this.m_batchBlock = new BatchBlock <T>(bulkSize, options.ToGroupingBlockOption()); var bulkInsertOption = options.ToExecutionBlockOption(); //Action block deal with array references if (bulkInsertOption.BoundedCapacity != DataflowBlockOptions.Unbounded) { bulkInsertOption.BoundedCapacity = bulkInsertOption.BoundedCapacity / bulkSize; } this.m_logger = Utils.GetNamespaceLogger(); this.m_actionBlock = new ActionBlock <T[]>( async array => { m_logger.Debug(h => h("{3} starts bulk-inserting {0} {1} to db table {2}", array.Length, typeof(T).Name, targetTable.TableName, this.FullName)); try { await this.DumpToDBAsync(array, targetTable); } catch (Exception e) { m_logger.Error($"{this.FullName} failed bulk-inserting {array.Length} {typeof(T).Name} to db table {targetTable.TableName}", e); throw; } m_logger.Debug(h => h("{3} bulk-inserted {0} {1} to db table {2}", array.Length, typeof(T).Name, targetTable.TableName, this.FullName)); } , bulkInsertOption); this.m_batchBlock.LinkTo(this.m_actionBlock, this.m_defaultLinkOption); this.RegisterChild(this.m_batchBlock); this.RegisterChild(this.m_actionBlock); this.m_timer = new Timer( state => { this.TriggerBatch(); }, null, TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(10)); }
/// <summary> /// Constructs a MultiDbBulkInserter instance. /// </summary> /// <param name="options">The option for this dataflow</param> /// <param name="dispatchFunc">The dispatch function to decide which child flow the incoming objects will be delivered to</param> /// <param name="connectionGetter">Connection string generator for child flows</param> /// <param name="destTable">The table name in database to bulk insert into for every child</param> /// <param name="destLabel">The mapping label to help choose among all column mappings</param> /// <param name="bulkSize">The bulk size to insert in a batch. Default to 8192.</param> /// <param name="displayName">A given name of this multi bulk inserter (would be nice for logging)</param> /// <param name="postBulkInsert">A delegate that enables you to inject some customized work whenever a bulk insert is done</param> public MultiDbBulkInserter(DataflowOptions options, Func <T, int> dispatchFunc, Func <int, string> connectionGetter, string destTable, string destLabel, int bulkSize = 4096 * 2, string displayName = null, PostBulkInsertDelegate <T> postBulkInsert = null) : base(dispatchFunc, options) { m_options = options; m_connectionGetter = connectionGetter; m_destTable = destTable; m_destLabel = destLabel; m_bulkSize = bulkSize; m_displayName = displayName; this.m_postBulkInsert = postBulkInsert; }
/// <summary> /// Constructs an instance of DbBulkInserter /// </summary> /// <param name="targetTable">Information about the database target and mapping label</param> /// <param name="options">Options to use for this dataflow</param> /// <param name="bulkSize">The bulk size to insert in a batch. Default to 8192.</param> /// <param name="dbBulkInserterName">A given name of this bulk inserter (would be nice for logging)</param> /// <param name="postBulkInsert">A delegate that enables you to inject some customized work whenever a bulk insert is done</param> public DbBulkInserterBase(TargetTable targetTable, DataflowOptions options, int bulkSize = 4096 * 2, string dbBulkInserterName = null, PostBulkInsertDelegate <T> postBulkInsert = null) : base(options) { this.m_targetTable = targetTable; this.m_typeAccessor = TypeAccessorManager <T> .GetAccessorForTable(targetTable); this.m_bulkSize = bulkSize; this.m_dbBulkInserterName = dbBulkInserterName; this.m_postBulkInsert = postBulkInsert; this.m_batchBlock = new BatchBlock <T>(bulkSize, options.ToGroupingBlockOption()); var bulkInsertOption = options.ToExecutionBlockOption(); //Action block deal with array references if (bulkInsertOption.BoundedCapacity != DataflowBlockOptions.Unbounded) { bulkInsertOption.BoundedCapacity = bulkInsertOption.BoundedCapacity / bulkSize; } this.m_actionBlock = new ActionBlock <T[]>(array => this.DumpToDBAsync(array, targetTable), bulkInsertOption); this.m_batchBlock.LinkTo(this.m_actionBlock, this.m_defaultLinkOption); this.m_logger = Utils.GetNamespaceLogger(); this.RegisterChild(this.m_batchBlock); this.RegisterChild(this.m_actionBlock); this.m_timer = new Timer( state => { this.TriggerBatch(); }, null, TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(10)); }
public EagerDbBulkInserter(TargetTable targetTable, DataflowOptions options, int bulkSize = 8192, string dbBulkInserterName = null, PostBulkInsertDelegate <T> postBulkInsert = null) : base(targetTable, options, bulkSize, dbBulkInserterName, postBulkInsert) { }
public EagerDbBulkInserter(string connectionString, string destTable, DataflowOptions options, string destLabel, int bulkSize = 8192, string dbBulkInserterName = null, PostBulkInsertDelegate <T> postBulkInsert = null) : base(connectionString, destTable, options, destLabel, bulkSize, dbBulkInserterName, postBulkInsert) { }
public DbBulkInserter(TargetTable targetTable, DataflowOptions options, int bulkSize = 8192, string dbBulkInserterName = null, PostBulkInsertDelegate <T> postBulkInsert = null) : base(targetTable, options, bulkSize, dbBulkInserterName, postBulkInsert) { m_longConnection = new SqlConnection(targetTable.ConnectionString); m_longConnection.Open(); m_transaction = m_longConnection.BeginTransaction(); }