protected virtual async Task OnPostBulkInsert(SqlConnection sqlConnection, TargetTable target, T[] insertedData)
{
if (m_postBulkInsert != null)
{
await m_postBulkInsert(sqlConnection, m_targetTable, insertedData).ConfigureAwait(false);
}
}
public DbBulkInserter(TargetTable targetTable, DataflowOptions options, int bulkSize = 8192, string dbBulkInserterName = null, PostBulkInsertDelegate <T> postBulkInsert = null)
: base(targetTable, options, bulkSize, dbBulkInserterName, postBulkInsert)
{
m_longConnection = new SqlConnection(targetTable.ConnectionString);
m_longConnection.Open();
m_transaction = m_longConnection.BeginTransaction();
}
/// <summary>
/// Constructs an instance of DbBulkInserter
/// </summary>
/// <param name="targetTable">Information about the database target and mapping label</param>
/// <param name="options">Options to use for this dataflow</param>
/// <param name="bulkSize">The bulk size to insert in a batch. Default to 8192.</param>
/// <param name="dbBulkInserterName">A given name of this bulk inserter (would be nice for logging)</param>
/// <param name="postBulkInsert">A delegate that enables you to inject some customized work whenever a bulk insert is done</param>
public DbBulkInserterBase(TargetTable targetTable,
DataflowOptions options,
int bulkSize = 4096 * 2,
string dbBulkInserterName = null,
PostBulkInsertDelegate <T> postBulkInsert = null)
: base(options)
{
this.m_targetTable = targetTable;
this.m_typeAccessor = TypeAccessorManager <T> .GetAccessorForTable(targetTable);
this.m_bulkSize = bulkSize;
this.m_dbBulkInserterName = dbBulkInserterName;
this.m_postBulkInsert = postBulkInsert;
this.m_batchBlock = new BatchBlock <T>(bulkSize, options.ToGroupingBlockOption());
var bulkInsertOption = options.ToExecutionBlockOption();
//Action block deal with array references
if (bulkInsertOption.BoundedCapacity != DataflowBlockOptions.Unbounded)
{
bulkInsertOption.BoundedCapacity = bulkInsertOption.BoundedCapacity / bulkSize;
}
this.m_logger = Utils.GetNamespaceLogger();
this.m_actionBlock = new ActionBlock <T[]>(
async array =>
{
m_logger.Debug(h => h("{3} starts bulk-inserting {0} {1} to db table {2}", array.Length, typeof(T).Name, targetTable.TableName, this.FullName));
try
{
await this.DumpToDBAsync(array, targetTable);
}
catch (Exception e)
{
m_logger.Error($"{this.FullName} failed bulk-inserting {array.Length} {typeof(T).Name} to db table {targetTable.TableName}", e);
throw;
}
m_logger.Debug(h => h("{3} bulk-inserted {0} {1} to db table {2}", array.Length, typeof(T).Name, targetTable.TableName, this.FullName));
}
, bulkInsertOption);
this.m_batchBlock.LinkTo(this.m_actionBlock, this.m_defaultLinkOption);
this.RegisterChild(this.m_batchBlock);
this.RegisterChild(this.m_actionBlock);
this.m_timer = new Timer(
state =>
{
this.TriggerBatch();
}, null, TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(10));
}
public TypeAccessor(TargetTable target)
{
m_destLabel = target.DestLabel;
m_connectionString = target.ConnectionString;
m_destinationTablename = string.IsNullOrWhiteSpace(target.TableName)
? typeof(T).Name
: target.TableName;
m_schemaTable = new Lazy <DataTable>(this.GetSchemaTable);
m_properties = new Dictionary <int, Func <T, object> >();
m_dbColumnMappings = new List <DBColumnMapping>();
m_classLogger = LogManager.GetLogger(this.GetType().Namespace + "." + this.GetType().GetFriendlyName());
CreateTypeVisitor();
}
protected async virtual Task DumpToDBAsync(T[] data, TargetTable targetTable)
{
m_logger.Debug(h => h("{3} starts bulk-inserting {0} {1} to db table {2}", data.Length, typeof(T).Name, targetTable.TableName, this.FullName));
using (var bulkReader = new BulkDataReader <T>(m_typeAccessor, data))
{
using (var conn = new SqlConnection(targetTable.ConnectionString))
{
await conn.OpenAsync().ConfigureAwait(false);
var transaction = conn.BeginTransaction();
try
{
using (var bulkCopy = new SqlBulkCopy(conn, SqlBulkCopyOptions.TableLock, transaction))
{
foreach (SqlBulkCopyColumnMapping map in bulkReader.ColumnMappings)
{
bulkCopy.ColumnMappings.Add(map);
}
bulkCopy.DestinationTableName = targetTable.TableName;
bulkCopy.BulkCopyTimeout = (int)TimeSpan.FromMinutes(30).TotalMilliseconds;
bulkCopy.BatchSize = m_bulkSize;
// Write from the source to the destination.
await bulkCopy.WriteToServerAsync(bulkReader).ConfigureAwait(false);
}
}
catch (Exception e)
{
if (e is NullReferenceException)
{
m_logger.ErrorFormat(
"{0} NullReferenceException occurred in bulk insertion. This is probably caused by forgetting assigning value to a [NoNullCheck] attribute when constructing your object.", this.FullName);
}
m_logger.ErrorFormat("{0} Bulk insertion failed. Rolling back all changes...", this.FullName, e);
transaction.Rollback();
m_logger.InfoFormat("{0} Changes successfully rolled back", this.FullName);
//As this is an unrecoverable exception, rethrow it
throw new AggregateException(e);
}
transaction.Commit();
await this.OnPostBulkInsert(conn, targetTable, data).ConfigureAwait(false);
}
}
m_logger.Info(h => h("{3} bulk-inserted {0} {1} to db table {2}", data.Length, typeof(T).Name, targetTable.TableName, this.FullName));
}
protected override async Task DumpToDBAsync(T[] data, TargetTable targetTable)
{
m_logger.Debug(h => h("{3} starts bulk-inserting {0} {1} to db table {2}", data.Length, typeof(T).Name, targetTable.TableName, this.FullName));
using (var bulkReader = new BulkDataReader <T>(m_typeAccessor, data))
{
try
{
using (var bulkCopy = new SqlBulkCopy(m_longConnection, SqlBulkCopyOptions.TableLock, m_transaction))
{
foreach (SqlBulkCopyColumnMapping map in bulkReader.ColumnMappings)
{
bulkCopy.ColumnMappings.Add(map);
}
bulkCopy.DestinationTableName = targetTable.TableName;
bulkCopy.BulkCopyTimeout = (int)TimeSpan.FromMinutes(30).TotalMilliseconds;
bulkCopy.BatchSize = m_bulkSize;
// Write from the source to the destination.
await bulkCopy.WriteToServerAsync(bulkReader).ConfigureAwait(false);
}
}
catch (Exception e)
{
if (e is NullReferenceException)
{
m_logger.ErrorFormat(
"{0} NullReferenceException occurred in bulk insertion. This is probably caused by forgetting assigning value to a [NoNullCheck] attribute when constructing your object.",
this.FullName);
}
m_logger.ErrorFormat("{0} Bulk insertion error in the first place", e, this.FullName);
//As this is an unrecoverable exception, rethrow it
throw new AggregateException(e);
}
await this.OnPostBulkInsert(m_longConnection, targetTable, data).ConfigureAwait(false);
}
m_logger.Info(h => h("{3} bulk-inserted {0} {1} to db table {2}", data.Length, typeof(T).Name, targetTable.TableName, this.FullName));
}
/// <summary>
/// Constructs an instance of DbBulkInserter
/// </summary>
/// <param name="targetTable">Information about the database target and mapping label</param>
/// <param name="options">Options to use for this dataflow</param>
/// <param name="bulkSize">The bulk size to insert in a batch. Default to 8192.</param>
/// <param name="dbBulkInserterName">A given name of this bulk inserter (would be nice for logging)</param>
/// <param name="postBulkInsert">A delegate that enables you to inject some customized work whenever a bulk insert is done</param>
public DbBulkInserterBase(TargetTable targetTable,
DataflowOptions options,
int bulkSize = 4096 * 2,
string dbBulkInserterName = null,
PostBulkInsertDelegate <T> postBulkInsert = null)
: base(options)
{
this.m_targetTable = targetTable;
this.m_typeAccessor = TypeAccessorManager <T> .GetAccessorForTable(targetTable);
this.m_bulkSize = bulkSize;
this.m_dbBulkInserterName = dbBulkInserterName;
this.m_postBulkInsert = postBulkInsert;
this.m_batchBlock = new BatchBlock <T>(bulkSize, options.ToGroupingBlockOption());
var bulkInsertOption = options.ToExecutionBlockOption();
//Action block deal with array references
if (bulkInsertOption.BoundedCapacity != DataflowBlockOptions.Unbounded)
{
bulkInsertOption.BoundedCapacity = bulkInsertOption.BoundedCapacity / bulkSize;
}
this.m_actionBlock = new ActionBlock <T[]>(array => this.DumpToDBAsync(array, targetTable), bulkInsertOption);
this.m_batchBlock.LinkTo(this.m_actionBlock, this.m_defaultLinkOption);
this.m_logger = Utils.GetNamespaceLogger();
this.RegisterChild(this.m_batchBlock);
this.RegisterChild(this.m_actionBlock);
this.m_timer = new Timer(
state =>
{
this.TriggerBatch();
}, null, TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(10));
}
protected bool Equals(TargetTable other)
{
return string.Equals(this.ConnectionString, other.ConnectionString) && string.Equals(this.TableName, other.TableName) && string.Equals(this.DestLabel, other.DestLabel);
}
/// <summary>
/// If the typeAccessor exists, just return it; else create a new one with parameter: destLabel, connectionString,
/// dataTableName
/// </summary>
public static TypeAccessor <T> GetAccessorForTable(TargetTable target)
{
return(s_accessors.GetOrAdd(target, t => new Lazy <TypeAccessor <T> >(() => new TypeAccessor <T>(t))).Value);
}
public EagerDbBulkInserter(TargetTable targetTable, DataflowOptions options, int bulkSize = 8192, string dbBulkInserterName = null, PostBulkInsertDelegate <T> postBulkInsert = null)
: base(targetTable, options, bulkSize, dbBulkInserterName, postBulkInsert)
{
}
protected abstract Task DumpToDBAsync(T[] data, TargetTable targetTable);
protected bool Equals(TargetTable other)
{
return(string.Equals(this.ConnectionString, other.ConnectionString) && string.Equals(this.TableName, other.TableName) && string.Equals(this.DestLabel, other.DestLabel));
}
