public MultiDbBulkInserter(BlockContainerOptions options, Func <T, int> dispatchFunc, Func <int, string> connectionGetter, string destTable, string destLabel, int bulkSize = 4096 * 2, string dbBulkInserterName = null)
: base(options)
{
m_options = options;
m_bulkInserterMap = new ConcurrentDictionary <int, Lazy <DbBulkInserter <T> > >();
m_dispatchFunc = dispatchFunc;
m_connectionGetter = connectionGetter;
m_destTable = destTable;
m_bulkSize = bulkSize;
m_dispatchBlock = new ActionBlock <T>(item =>
{
int profileId = m_dispatchFunc(item);
var bulkInserter = m_bulkInserterMap.GetOrAdd(profileId, m_initer).Value;
bulkInserter.InputBlock.SafePost(item);
}, new ExecutionDataflowBlockOptions {
BoundedCapacity = m_containerOptions.RecommendedCapacity ?? int.MaxValue,
});
m_initer = p => new Lazy <DbBulkInserter <T> >(
() =>
{
var singleInserter = new DbBulkInserter <T>(
m_connectionGetter(p),
m_destTable,
m_options,
destLabel,
m_bulkSize, string.Format("{0}_{1}", this.Name, p));
//Register dynamically generated blocks to enable upstream propagation
this.RegisterChild(singleInserter);
return(singleInserter);
});
RegisterChild(m_dispatchBlock, t =>
{
//propagate completion to children as we don't have 'link'
if (t.Status == TaskStatus.RanToCompletion)
{
foreach (var kvPair in m_bulkInserterMap)
{
DbBulkInserter <T> singleProfileInserter = kvPair.Value.Value;
singleProfileInserter.InputBlock.Complete();
}
}
//no need to propagate errors as register handles that (given that dyamic blocks are registered)
});
}
public DbBulkInserter(string connectionString, string destTable, BlockContainerOptions options, string destLabel, int bulkSize = 4096 * 2, string dbBulkInserterName = null)
: base(options)
{
m_bulkSize = bulkSize;
m_dbBulkInserterName = dbBulkInserterName;
m_batchBlock = new BatchBlock <T>(bulkSize);
m_actionBlock = new ActionBlock <T[]>(async array =>
{
LogHelper.Logger.Debug(h => h("{3} starts bulk-inserting {0} {1} to db table {2}", array.Length, typeof(T).Name, destTable, this.Name));
await DumpToDB(array, destTable, connectionString, destLabel);
LogHelper.Logger.Info(h => h("{3} bulk-inserted {0} {1} to db table {2}", array.Length, typeof(T).Name, destTable, this.Name));
});
m_batchBlock.LinkTo(m_actionBlock, m_defaultLinkOption);
RegisterChild(m_batchBlock);
RegisterChild(m_actionBlock);
}