public async Task ExecuteAsync(IDataSourceAdapter source, IDataSinkAdapter sink, ITransferStatistics statistics, CancellationToken cancellation)
{
Guard.NotNull("source", source);
Guard.NotNull("sink", sink);
Guard.NotNull("statistics", statistics);
var writeTasks = Enumerable
.Range(0, sink.MaxDegreeOfParallelism)
.Select(i => (Task)Task.FromResult<object>(null))
.ToArray();
var fatalExceptions = new List<Exception>();
var readOutput = new ReadOutputByRef();
IDataItem dataItem;
while (!cancellation.IsCancellationRequested)
{
readOutput.Wipe();
try
{
dataItem = await source.ReadNextAsync(readOutput, cancellation);
}
catch (NonFatalReadException nonFatalException)
{
statistics.AddError(readOutput.DataItemId, nonFatalException);
continue;
}
catch (Exception exception)
{
fatalExceptions.Add(exception);
break;
}
if (dataItem == null || cancellation.IsCancellationRequested)
break;
var completed = await Task.WhenAny(writeTasks);
writeTasks[Array.IndexOf(writeTasks, completed)] =
TransferDataItem(sink, readOutput.DataItemId, dataItem, statistics, cancellation);
}
// Report completion to the sink
try
{
await sink.CompleteAsync(cancellation);
}
catch (Exception exception)
{
fatalExceptions.Add(exception);
}
// Wait for all on-going writes to complete
for (var index = 0; index < writeTasks.Length; ++index)
await writeTasks[index];
// Throw fatal exceptions, if any
if (fatalExceptions.Any())
throw new AggregateException(fatalExceptions);
}
public Sidecar(IDataSourceAdapter dataSource, IComputeStep computeStep, IDataSinkAdapter dataSink,
IOrchestratorServiceClient orchestrator, ILogger <Sidecar> logger, IConfiguration configuration,
ActivitySource source,
IDataAdapterProvider adapterProvider)
{
_dataSource = dataSource;
_computeStep = computeStep;
_dataSink = dataSink;
_orchestrator = orchestrator;
_logger = logger;
_source = source;
_adapterProvider = adapterProvider;
this._inputPath = configuration["Sidecar:InputPath"];
}
public async Task ExecuteAsync(IDataSourceAdapter source, IDataSinkAdapter sink, ITransferStatistics statistics, CancellationToken cancellation)
{
Guard.NotNull("source", source);
Guard.NotNull("sink", sink);
try
{
statistics.Start();
await TransferData(source, sink, statistics, cancellation);
}
finally
{
statistics.Stop();
}
}
private static async Task TransferDataItem(IDataSinkAdapter sink, string dataItemId, IDataItem dataItem, ITransferStatistics statistics, CancellationToken cancellation)
{
try
{
await sink.WriteAsync(dataItem, cancellation);
statistics.AddTransferred();
}
catch (AggregateException aggregateException)
{
foreach (var exception in aggregateException.Flatten().InnerExceptions)
statistics.AddError(dataItemId, exception);
}
catch (Exception exception)
{
statistics.AddError(dataItemId, exception);
}
}
protected async Task WriteDataAsync(IDataSinkAdapter sinkAdapter, IEnumerable<IDataItem> data)
{
Task[] writeTasks = Enumerable
.Range(0, sinkAdapter.MaxDegreeOfParallelism)
.Select(i => (Task)Task.FromResult<object>(null))
.ToArray();
foreach (var dataItem in data)
{
var completed = await Task.WhenAny(writeTasks);
await completed;
writeTasks[Array.IndexOf(writeTasks, completed)] =
sinkAdapter.WriteAsync(dataItem, CancellationToken.None);
}
await sinkAdapter.CompleteAsync(CancellationToken.None);
for (var index = 0; index < writeTasks.Length; ++index)
await writeTasks[index];
}
private static async Task TransferDataItem(IDataSinkAdapter sink, string dataItemId, IDataItem dataItem, ITransferStatistics statistics, CancellationToken cancellation)
{
try
{
await sink.WriteAsync(dataItem, cancellation);
statistics.AddTransferred();
}
catch (AggregateException aggregateException)
{
foreach (var exception in aggregateException.Flatten().InnerExceptions)
{
statistics.AddError(dataItemId, exception);
}
}
catch (Exception exception)
{
statistics.AddError(dataItemId, exception);
}
}
protected async Task WriteDataAsync(IDataSinkAdapter sinkAdapter, IEnumerable <IDataItem> data)
{
Task[] writeTasks = Enumerable
.Range(0, sinkAdapter.MaxDegreeOfParallelism)
.Select(i => (Task)Task.FromResult <object>(null))
.ToArray();
foreach (var dataItem in data)
{
var completed = await Task.WhenAny(writeTasks);
await completed;
writeTasks[Array.IndexOf(writeTasks, completed)] =
sinkAdapter.WriteAsync(dataItem, CancellationToken.None);
}
await sinkAdapter.CompleteAsync(CancellationToken.None);
for (var index = 0; index < writeTasks.Length; ++index)
{
await writeTasks[index];
}
}
private static async Task TransferData(IDataSourceAdapter source, IDataSinkAdapter sink, ITransferStatistics statistics, CancellationToken cancellation)
{
var writeTasks = Enumerable
.Range(0, sink.MaxDegreeOfParallelism)
.Select(i => (Task)Task.FromResult <object>(null))
.ToArray();
var fatalExceptions = new List <Exception>();
var readOutput = new ReadOutputByRef();
IDataItem dataItem;
while (!cancellation.IsCancellationRequested)
{
readOutput.Wipe();
try
{
dataItem = await source.ReadNextAsync(readOutput, cancellation);
}
catch (NonFatalReadException nonFatalException)
{
statistics.AddError(readOutput.DataItemId, nonFatalException);
continue;
}
catch (Exception exception)
{
fatalExceptions.Add(exception);
break;
}
if (dataItem == null || cancellation.IsCancellationRequested)
{
break;
}
var completed = await Task.WhenAny(writeTasks);
writeTasks[Array.IndexOf(writeTasks, completed)] =
TransferDataItem(sink, readOutput.DataItemId, dataItem, statistics, cancellation);
}
// Report completion to the sink
try
{
await sink.CompleteAsync(cancellation);
}
catch (Exception exception)
{
fatalExceptions.Add(exception);
}
// Wait for all on-going writes to complete
for (var index = 0; index < writeTasks.Length; ++index)
{
await writeTasks[index];
}
// Throw fatal exceptions, if any
if (fatalExceptions.Any())
{
throw new AggregateException(fatalExceptions);
}
}
