private async Task ProcessPipelinesAsync(CancellationToken ct, Action <string> cancelPipeline)
{
const int queueLengthQuantifier = 5;
var readCommandChannel =
Channel.CreateBounded <ReadCommand>(_dataMoverSettings.ReadConcurrency * queueLengthQuantifier);
var insertCommandChannel =
Channel.CreateBounded <InsertCommand>(_dataMoverSettings.InsertConcurrency * queueLengthQuantifier);
async Task RunCommandGeneratorWorker()
{
try
{
_logger.LogDebug("{@EventType}", "Pipeline_SchemaReadStarted");
await foreach (var readCommand in _readCommandGenerator
.GetReadCommandsAsync(ct).WithCancellation(ct))
{
await readCommandChannel.Writer.WaitToWriteAsync(ct);
await readCommandChannel.Writer.WriteAsync(readCommand, ct);
}
_logger.LogDebug("{@EventType}", "Pipeline_SchemaReadCompleted");
}
catch (Exception e)
{
cancelPipeline(nameof(RunCommandGeneratorWorker));
_logger.LogError(e, "{@EventType}", "Pipeline_SchemaReadFailed");
throw;
}
finally
{
if (!readCommandChannel.Writer.TryComplete())
{
_logger.LogWarning("Failed to complete writing to readCommandChannel");
}
}
}
async Task RunReadWorker(int n)
{
while (await readCommandChannel.Reader.WaitToReadAsync(ct) && !ct.IsCancellationRequested)
{
object debugPayload = null;
try
{
var command = await readCommandChannel.Reader.ReadAsync(ct);
debugPayload = new
{
TableName = command.TableSchema.Name,
command.FromPrimaryKey,
command.ToPrimaryKey
};
_logger.LogDebug("{@EventType} {@Payload}", "Pipeline_ReadStarted", debugPayload);
var batch = await _sourceDataReader.ReadBatchAsync(command, ct);
await insertCommandChannel.Writer.WaitToWriteAsync(ct);
await insertCommandChannel.Writer.WriteAsync(new InsertCommand
{
TableName = command.TableSchema.Name,
Batch = batch
}, ct);
_logger.LogDebug("{@EventType} {@Payload}", "Pipeline_ReadCompleted", debugPayload);
}
catch (ChannelClosedException)
{
_logger.LogDebug(
$"Catching channel 'readCommandChannel' closed exception by read worker {n}. This should be considered normal.");
}
catch (Exception e)
{
cancelPipeline(nameof(RunReadWorker));
_logger.LogError(e, "{@EventType} {@Payload}, bubbling up exception",
"Pipeline_ReadFailed", debugPayload);
throw;
}
}
}
async Task RunInsertWorker(int n)
{
while (await insertCommandChannel.Reader.WaitToReadAsync(ct) && !ct.IsCancellationRequested)
{
object debugPayload = null;
try
{
var command = await insertCommandChannel.Reader.ReadAsync(ct);
debugPayload = new
{
command.TableName,
};
_logger.LogDebug("{@EventType} {@Payload}", "Pipeline_InsertStarted", debugPayload);
await _publisher.PublishAsync(command, ct);
_logger.LogDebug("{@EventType} {@Payload}", "Pipeline_InsertCompleted", debugPayload);
}
catch (ChannelClosedException)
{
_logger.LogDebug(
$"Catching channel 'insertCommandChannel' closed exception by writer {n}. This should be considered normal.");
}
catch (Exception e)
{
cancelPipeline(nameof(RunInsertWorker));
_logger.LogError(e, "{@EventType} {@Payload}, bubbling up exception",
"Pipeline_InsertFailed", debugPayload);
throw;
}
}
}
var allPipelineTasks = new List <Task>();
var commandGeneratorWorker = RunCommandGeneratorWorker();
allPipelineTasks.Add(commandGeneratorWorker);
var readWorkers = Enumerable.Range(0, _dataMoverSettings.ReadConcurrency)
.Select(RunReadWorker)
.ToList();
allPipelineTasks.AddRange(readWorkers);
var insertWorkers = Enumerable.Range(0, _dataMoverSettings.InsertConcurrency)
.Select(RunInsertWorker)
.ToList();
allPipelineTasks.AddRange(insertWorkers);
var firstStage = readWorkers.Concat(new[] { commandGeneratorWorker }).ToList();
try
{
await Task.WhenAll(firstStage);
}
catch
{
}
if (!insertCommandChannel.Writer.TryComplete())
{
_logger.LogWarning("Failed to complete writing to insertCommandChannel");
}
try
{
await Task.WhenAll(insertWorkers);
}
catch
{
}
if (allPipelineTasks.Any(t => t.IsFaulted))
{
_logger.LogError("{@EventType}",
"Pipeline_Failed");
throw new Exception("Pipeline has errors");
}
_logger.LogInformation("{@EventType}",
"Pipeline_Finished");
}