public async Task ExecuteAsync(IDataSourceAdapter source, IDataSinkAdapter sink, ITransferStatistics statistics, CancellationToken cancellation) { Guard.NotNull("source", source); Guard.NotNull("sink", sink); Guard.NotNull("statistics", statistics); var writeTasks = Enumerable .Range(0, sink.MaxDegreeOfParallelism) .Select(i => (Task)Task.FromResult<object>(null)) .ToArray(); var fatalExceptions = new List<Exception>(); var readOutput = new ReadOutputByRef(); IDataItem dataItem; while (!cancellation.IsCancellationRequested) { readOutput.Wipe(); try { dataItem = await source.ReadNextAsync(readOutput, cancellation); } catch (NonFatalReadException nonFatalException) { statistics.AddError(readOutput.DataItemId, nonFatalException); continue; } catch (Exception exception) { fatalExceptions.Add(exception); break; } if (dataItem == null || cancellation.IsCancellationRequested) break; var completed = await Task.WhenAny(writeTasks); writeTasks[Array.IndexOf(writeTasks, completed)] = TransferDataItem(sink, readOutput.DataItemId, dataItem, statistics, cancellation); } // Report completion to the sink try { await sink.CompleteAsync(cancellation); } catch (Exception exception) { fatalExceptions.Add(exception); } // Wait for all on-going writes to complete for (var index = 0; index < writeTasks.Length; ++index) await writeTasks[index]; // Throw fatal exceptions, if any if (fatalExceptions.Any()) throw new AggregateException(fatalExceptions); }
public Sidecar(IDataSourceAdapter dataSource, IComputeStep computeStep, IDataSinkAdapter dataSink, IOrchestratorServiceClient orchestrator, ILogger <Sidecar> logger, IConfiguration configuration, ActivitySource source, IDataAdapterProvider adapterProvider) { _dataSource = dataSource; _computeStep = computeStep; _dataSink = dataSink; _orchestrator = orchestrator; _logger = logger; _source = source; _adapterProvider = adapterProvider; this._inputPath = configuration["Sidecar:InputPath"]; }
public async Task ExecuteAsync(IDataSourceAdapter source, IDataSinkAdapter sink, ITransferStatistics statistics, CancellationToken cancellation) { Guard.NotNull("source", source); Guard.NotNull("sink", sink); try { statistics.Start(); await TransferData(source, sink, statistics, cancellation); } finally { statistics.Stop(); } }
private static async Task TransferDataItem(IDataSinkAdapter sink, string dataItemId, IDataItem dataItem, ITransferStatistics statistics, CancellationToken cancellation) { try { await sink.WriteAsync(dataItem, cancellation); statistics.AddTransferred(); } catch (AggregateException aggregateException) { foreach (var exception in aggregateException.Flatten().InnerExceptions) statistics.AddError(dataItemId, exception); } catch (Exception exception) { statistics.AddError(dataItemId, exception); } }
protected async Task WriteDataAsync(IDataSinkAdapter sinkAdapter, IEnumerable<IDataItem> data) { Task[] writeTasks = Enumerable .Range(0, sinkAdapter.MaxDegreeOfParallelism) .Select(i => (Task)Task.FromResult<object>(null)) .ToArray(); foreach (var dataItem in data) { var completed = await Task.WhenAny(writeTasks); await completed; writeTasks[Array.IndexOf(writeTasks, completed)] = sinkAdapter.WriteAsync(dataItem, CancellationToken.None); } await sinkAdapter.CompleteAsync(CancellationToken.None); for (var index = 0; index < writeTasks.Length; ++index) await writeTasks[index]; }
private static async Task TransferDataItem(IDataSinkAdapter sink, string dataItemId, IDataItem dataItem, ITransferStatistics statistics, CancellationToken cancellation) { try { await sink.WriteAsync(dataItem, cancellation); statistics.AddTransferred(); } catch (AggregateException aggregateException) { foreach (var exception in aggregateException.Flatten().InnerExceptions) { statistics.AddError(dataItemId, exception); } } catch (Exception exception) { statistics.AddError(dataItemId, exception); } }
protected async Task WriteDataAsync(IDataSinkAdapter sinkAdapter, IEnumerable <IDataItem> data) { Task[] writeTasks = Enumerable .Range(0, sinkAdapter.MaxDegreeOfParallelism) .Select(i => (Task)Task.FromResult <object>(null)) .ToArray(); foreach (var dataItem in data) { var completed = await Task.WhenAny(writeTasks); await completed; writeTasks[Array.IndexOf(writeTasks, completed)] = sinkAdapter.WriteAsync(dataItem, CancellationToken.None); } await sinkAdapter.CompleteAsync(CancellationToken.None); for (var index = 0; index < writeTasks.Length; ++index) { await writeTasks[index]; } }
private static async Task TransferData(IDataSourceAdapter source, IDataSinkAdapter sink, ITransferStatistics statistics, CancellationToken cancellation) { var writeTasks = Enumerable .Range(0, sink.MaxDegreeOfParallelism) .Select(i => (Task)Task.FromResult <object>(null)) .ToArray(); var fatalExceptions = new List <Exception>(); var readOutput = new ReadOutputByRef(); IDataItem dataItem; while (!cancellation.IsCancellationRequested) { readOutput.Wipe(); try { dataItem = await source.ReadNextAsync(readOutput, cancellation); } catch (NonFatalReadException nonFatalException) { statistics.AddError(readOutput.DataItemId, nonFatalException); continue; } catch (Exception exception) { fatalExceptions.Add(exception); break; } if (dataItem == null || cancellation.IsCancellationRequested) { break; } var completed = await Task.WhenAny(writeTasks); writeTasks[Array.IndexOf(writeTasks, completed)] = TransferDataItem(sink, readOutput.DataItemId, dataItem, statistics, cancellation); } // Report completion to the sink try { await sink.CompleteAsync(cancellation); } catch (Exception exception) { fatalExceptions.Add(exception); } // Wait for all on-going writes to complete for (var index = 0; index < writeTasks.Length; ++index) { await writeTasks[index]; } // Throw fatal exceptions, if any if (fatalExceptions.Any()) { throw new AggregateException(fatalExceptions); } }