public async Task DataflowPipeBulkInsertBlockAsync(int seqNumber = 1, int orderCount = 4000)
{
if (!_isInitialize)
{
throw new InvalidOperationException("not initialize");
}
#region Block Method
(int blockCount, int minPerBlock, int maxPerBlock)blockBascInfo = BlockHelper.GetBasciBlockInfo(orderCount);
int[] blockInfos = BlockHelper.GetBlockInfo(messageCount: orderCount, blockCount: blockBascInfo.blockCount, minPerBlock: blockBascInfo.minPerBlock, maxPerBlock: blockBascInfo.maxPerBlock);
#endregion
int boundedCapacity = blockInfos.Sum(b => b);
Debug.Assert(orderCount == boundedCapacity);
var logger = _loggerFactory.CreateLogger($"DataflowPipeBulkInser-{seqNumber}:{orderCount}");
try
{
var dataflowPipeBulkInserter = this.ServiceProvider.GetRequiredService <IDataflowPipeBulkInserter <OrderDto, OrderDto> >();
var contosoDataSourceFactory = this.ServiceProvider.GetRequiredService <IDataSourceFactory <IContosoDataSource> >();
var orderDataSource = contosoDataSourceFactory.Current.OrderDataSource;
_totalCount = 0;
var transportTimeWatcher = Stopwatch.StartNew();
TimeSpan totalTransportTime = TimeSpan.Zero;
var executionTimeWatcher = Stopwatch.StartNew();
logger.LogInformation($"----begin dataflow pipe bulk insert {orderCount} orders,now:{DateTime.Now.TimeOfDay}----");
int start = 0;
for (int i = 0; i < blockInfos.Count(); i++)
{
int insertOrderCount = blockInfos[i];
var orders = OrderJsonProvider.CreateOrders(start, insertOrderCount);
start += insertOrderCount;
transportTimeWatcher.Restart();
var dbOrders = await orderDataSource.DataflowPipeBulkInsertOrdersAsync(orders);
totalTransportTime += transportTimeWatcher.Elapsed;
transportTimeWatcher.Reset();
if (dbOrders?.Count() > 0)
{
await ProcessDataflowPipeOrdersAsync(dbOrders);
}
}
logger
.LogInformation($"----dataflow pipe bulk insert {orderCount} orders,cost time:\"{executionTimeWatcher.Elapsed}\",transport time:{ totalTransportTime },count/time(sec):{Math.Ceiling(orderCount / totalTransportTime.TotalSeconds)},now:\"{DateTime.Now.TimeOfDay}\"----");
}
catch (Exception ex)
{
logger.LogError($"Error while dataflow pipe bulk insert: {ex.Message}");
}
}
public async Task PipeBulkInsertBlockCoreAsync(AsyncCountdownEvent signals, int index, int orderCount)
{
#region Block Method
(int blockCount, int minPerBlock, int maxPerBlock)blockBascInfo = BlockHelper.GetBasciBlockInfo(orderCount);
int[] blockInfos = BlockHelper.GetBlockInfo(messageCount: orderCount, blockCount: blockBascInfo.blockCount, minPerBlock: blockBascInfo.minPerBlock, maxPerBlock: blockBascInfo.maxPerBlock);
#endregion
int boundedCapacity = blockInfos.Sum(b => b);
Debug.Assert(orderCount == boundedCapacity);
var logger = _loggerFactory.CreateLogger($"PipeBulkInserter-{index}");
try
{
var contosoDataSourceFactory = this.ServiceProvider.GetRequiredService <IDataSourceFactory <IContosoDataSource> >();
var orderDataSource = contosoDataSourceFactory.Current.OrderDataSource;
var transportTimeWatcher = Stopwatch.StartNew();
TimeSpan totalTransportTime = TimeSpan.Zero;
var executionTimeWatcher = Stopwatch.StartNew();
logger.LogInformation($"----begin pipe bulk insert {orderCount} orders,now:{DateTime.Now.TimeOfDay}----");
int start = 0;
for (int i = 0; i < blockInfos.Count(); i++)
{
int insertOrderCount = blockInfos[i];
var orders = OrderJsonProvider.CreateOrders(start, insertOrderCount);
start += insertOrderCount;
transportTimeWatcher.Restart();
var dbOrders = await orderDataSource.PipeBulkInsertOrdersAsync(orders);
totalTransportTime += transportTimeWatcher.Elapsed;
transportTimeWatcher.Reset();
//if (dbOrders?.Count() > 0)
//{
// await ProcessPipeOrdersAsync(dbOrders);
//}
}
logger
.LogInformation($"----pipe bulk insert {orderCount} orders,cost time:\"{executionTimeWatcher.Elapsed}\",transport time:{ totalTransportTime },count/time(sec):{Math.Ceiling(orderCount / totalTransportTime.TotalSeconds)},now:\"{DateTime.Now.TimeOfDay}\"----");
signals?.Signal();
}
catch (Exception ex)
{
logger.LogError($"Error while pipe bulk insert orders of {nameof(PipeBulkInsertLoopCoreAsync)}: {ex.Message}");
}
}
public async Task DataflowBulkInsertBlockRetryTasksCoreAsync(AsyncCountdownEvent signals, int index, int count, int orderCount)
{
var logger = _loggerFactory.CreateLogger($"DataflowBulkInserter-TaskCount:{index}");
int totalOrderCount = count * orderCount;
try
{
var dataflowBulkInserter = this.ServiceProvider.GetRequiredService<IDataflowBulkInserter<OrderDto, OrderDto>>();
var contosoDataSourceFactory = this.ServiceProvider.GetRequiredService<IDataSourceFactory<IContosoDataSource>>();
var orderDataSource = contosoDataSourceFactory.Current.OrderDataSource;
// await Task.Delay(TimeSpan.FromMilliseconds(2000));
var transportTimeWatcher = Stopwatch.StartNew();
TimeSpan totalTransportTime = TimeSpan.Zero;
var executionTimeWatcher = Stopwatch.StartNew();
// logger.LogInformation($"----begin dataflow bulk insert { totalOrderCount} orders,now:{DateTime.Now.TimeOfDay}----");
int start = 0;
for (int i = 0; i < count; i++)
{
var orders = OrderJsonProvider.CreateOrders(start, orderCount);
start += orderCount;
transportTimeWatcher.Restart();
var dbOrders = await orderDataSource.DataflowBulkInsertOrdersAsync(orders);
totalTransportTime += transportTimeWatcher.Elapsed;
transportTimeWatcher.Reset();
if (dbOrders?.Count() > 0)
{
await ProcessDataflowOrdersAsync(dbOrders);
}
}
//logger
// .LogInformation($"----dataflow bulk insert {totalOrderCount} orders,cost time:\"{executionTimeWatcher.Elapsed}\",transport time:{ totalTransportTime },count/time(sec):{Math.Ceiling(totalOrderCount / totalTransportTime.TotalSeconds)},now:\"{DateTime.Now.TimeOfDay}\"----");
signals?.Signal();
}
catch (Exception ex)
{
logger.LogError($"Error while dataflow bulk insert orders of {nameof(DataflowBulkInsertBlockRetryTasksCoreAsync)}: {ex.Message}");
}
}
public async Task PipeBulkInsertLoopCoreAsync(AsyncCountdownEvent signals, int index, int count, int orderCount)
{
var logger = _loggerFactory.CreateLogger($"PipeBulkInserter-{index}");
try
{
var contosoDataSourceFactory = this.ServiceProvider.GetRequiredService <IDataSourceFactory <IContosoDataSource> >();
var orderDataSource = contosoDataSourceFactory.Current.OrderDataSource;
var transportTimeWatcher = Stopwatch.StartNew();
TimeSpan totalTransportTime = TimeSpan.Zero;
var executionTimeWatcher = Stopwatch.StartNew();
logger.LogInformation($"----begin pipe bulk insert {orderCount} orders,now:{DateTime.Now.TimeOfDay}----");
int start = 0;
for (int i = 0; i < count; i++)
{
var orders = OrderJsonProvider.CreateOrders(start, orderCount);
start += orderCount;
transportTimeWatcher.Restart();
var dbOrders = await orderDataSource.PipeBulkInsertOrdersAsync(orders);
totalTransportTime += transportTimeWatcher.Elapsed;
transportTimeWatcher.Reset();
//if (dbOrders?.Count() > 0)
//{
// await ProcessPipeOrdersAsync(dbOrders);
//}
}
logger
.LogInformation($"----pipe bulk insert {orderCount} orders,cost time:\"{executionTimeWatcher.Elapsed}\",transport time:{ totalTransportTime },count/time(sec):{Math.Ceiling(orderCount / totalTransportTime.TotalSeconds)},now:\"{DateTime.Now.TimeOfDay}\"----");
signals?.Signal();
}
catch (Exception ex)
{
logger.LogError($"Error while pipe bulk insert orders of {nameof(PipeBulkInsertLoopCoreAsync)}: {ex.Message}");
}
}
public Task InitializeDataflowBulkInsertBlockTimerAsync(double frequency = 2000d, int maxRetryCount = 10, int orderCount = 1000)
{
if (!_isInitialize)
{
throw new InvalidOperationException("not initialize");
}
_frequency = frequency;
_period = TimeSpan.FromMilliseconds(frequency);
_maxRetryCount = maxRetryCount;
_totalOrderCount = _maxRetryCount * orderCount;
var logger = _loggerFactory.CreateLogger($"DataflowBulkInser-Timer:{_period.Milliseconds}-OrderCount:{ _totalOrderCount}");
int currentRetryCount = 0;
try
{
var contosoDataSourceFactory = this.ServiceProvider.GetRequiredService <IDataSourceFactory <IContosoDataSource> >();
var orderDataSource = contosoDataSourceFactory.Current.OrderDataSource;
_transportTimeWatcher = Stopwatch.StartNew();
_totalTransportTime = TimeSpan.Zero;
_executionTimeWatcher = Stopwatch.StartNew();
int start = 0;
//dueTime:调用callback之前延迟的时间量（以毫秒为单位），指定 Timeout.Infinite 以防止计时器开始计时。指定零 (0) 以立即启动计时器
//period:定时的时间时隔，以毫秒为单位
_timer = new Timer(async(state) =>
{
var orders = OrderJsonProvider.CreateOrders(start, orderCount);
start += orderCount;
_transportTimeWatcher.Restart();
var dbOrders = await orderDataSource.DataflowBulkInsertOrdersAsync(orders);
_totalTransportTime += _transportTimeWatcher.Elapsed;
_transportTimeWatcher.Reset();
//currentRetryTime = transportTimeWatcher.Elapsed;
//if (currentRetryTime >= maxRetryTime)
//{
// _timer.Change(Timeout.Infinite, Timeout.Infinite);
//}
currentRetryCount++;
if (currentRetryCount >= _maxRetryCount)
{
logger
.LogInformation($"----dataflow bulk insert {_totalOrderCount} orders,cost time:\"{_executionTimeWatcher.Elapsed}\",transport time:{ _totalTransportTime },count/time(sec):{Math.Ceiling(_totalOrderCount / _totalTransportTime.TotalSeconds)},now:\"{DateTime.Now.TimeOfDay}\"----");
_timer.Change(Timeout.Infinite, Timeout.Infinite);
_timer.Dispose();
}
},
this, Timeout.Infinite, Timeout.Infinite);
}
catch (Exception ex)
{
logger.LogError($"Error while dataflow bulk insert orders of {nameof(InitializeDataflowBulkInsertBlockTimerAsync)}: {ex.Message}");
}
return(Task.CompletedTask);
}
public async Task DataflowBulkInsertBlockRetryAsync(int seqNumber = 1, int maxRetryCount = 200, int orderCount = 1000)
{
if (!_isInitialize)
{
throw new InvalidOperationException("not initialize");
}
#region Block Method
(int blockCount, int minPerBlock, int maxPerBlock)blockBascInfo = BlockHelper.GetBasciBlockInfo(orderCount);
int[] blockInfos = BlockHelper.GetBlockInfo(messageCount: orderCount, blockCount: blockBascInfo.blockCount, minPerBlock: blockBascInfo.minPerBlock, maxPerBlock: blockBascInfo.maxPerBlock);
#endregion
int boundedCapacity = blockInfos.Sum(b => b);
Debug.Assert(orderCount == boundedCapacity);
var logger = _loggerFactory.CreateLogger($"DataflowBulkInser-{seqNumber}:{maxRetryCount * orderCount}");
int currentRetryCount = 0;
try
{
var dataflowBulkInserter = this.ServiceProvider.GetRequiredService <IDataflowBulkInserter <OrderDto, OrderDto> >();
var contosoDataSourceFactory = this.ServiceProvider.GetRequiredService <IDataSourceFactory <IContosoDataSource> >();
var orderDataSource = contosoDataSourceFactory.Current.OrderDataSource;
TimeSpan period = TimeSpan.FromMilliseconds(50);
_totalCount = 0;
var transportTimeWatcher = Stopwatch.StartNew();
TimeSpan totalTransportTime = TimeSpan.Zero;
var executionTimeWatcher = Stopwatch.StartNew();
logger.LogInformation($"----begin dataflow bulk insert {maxRetryCount * orderCount} orders,now:{DateTime.Now.TimeOfDay}----");
while (true)
{
if (_cancellationTokenSource.Token.IsCancellationRequested)
{
_cancellationTokenSource.Token.ThrowIfCancellationRequested();
// _logger?.LogCritical($"message was cancelled");
break;
}
int start = 0;
for (int i = 0; i < blockInfos.Count(); i++)
{
int insertOrderCount = blockInfos[i];
var orders = OrderJsonProvider.CreateOrders(start, insertOrderCount);
start += insertOrderCount;
transportTimeWatcher.Restart();
var dbOrders = await orderDataSource.DataflowBulkInsertOrdersAsync(orders);
totalTransportTime += transportTimeWatcher.Elapsed;
transportTimeWatcher.Reset();
//if (dbOrders?.Count() > 0)
//{
// await ProcessDataflowOrdersAsync(dbOrders);
//}
}
// await Task.Delay(TimeSpan.FromMilliseconds(200));
if (maxRetryCount == 99)
{
await Task.Delay(period);
}
currentRetryCount++;
if (currentRetryCount >= maxRetryCount)
{
//_cancellationTokenSource.CancelAfter(TimeSpan.FromMilliseconds(200));
break;
}
}
logger
.LogInformation($"----dataflow bulk insert {maxRetryCount * orderCount} orders,cost time:\"{executionTimeWatcher.Elapsed}\",transport time:{ totalTransportTime },count/time(sec):{Math.Ceiling(maxRetryCount * orderCount / totalTransportTime.TotalSeconds)},now:\"{DateTime.Now.TimeOfDay}\"----");
}
catch (Exception ex)
{
logger.LogError($"Error while dataflow bulk insert: {ex.Message}");
}
}
