public PipelineFacts()
{
_httpDocument = Substitute.For <IHttpDocument>();
_httpDocument.ClassifyAsync(Arg.Any <string>()).Returns(new ClassificationResult());
_httpDocument.ExtractAsync(Arg.Any <string>()).Returns(new ExtractionResults());
_httpDocument.DeleteAsync();
_documentFactory
.CreateDocumentAsync(Arg.Any <Document>())
.Returns(_httpDocument);
_sut = new Pipeline(_documentFactory, 10);
}
public async Task <IHttpDocument> CreateDocumentAsync(Document source, CancellationToken cancellationToken = default)
{
var httpDocument = await _wrappedDocumentFactory
.CreateDocumentAsync(source, cancellationToken)
.ConfigureAwait(false);
Logger.Info(
"Created Waives document {DocumentId} from '{DocumentSourceId}'",
httpDocument.Id,
source.SourceId);
return(httpDocument);
}
/// <summary>
/// Start processing the documents in the pipeline.
/// </summary>
/// <param name="cancellationToken">
/// The token to monitor for cancellation requests. The default value is
/// <see cref="CancellationToken.None"/>.
/// </param>
/// <returns>A <see cref="Task"/> which completes when processing of all the documents
/// in the pipeline is complete.
/// </returns>
public async Task RunAsync(CancellationToken cancellationToken = default)
{
var taskCompletion = new TaskCompletionSource <bool>(
TaskCreationOptions.RunContinuationsAsynchronously);
void OnPipelineComplete()
{
try
{
_onPipelineCompletedUserAction();
Logger.Info("Pipeline complete");
taskCompletion.SetResult(true);
}
catch (Exception e)
{
taskCompletion.TrySetException(e);
}
}
void OnPipelineError(Exception e)
{
Logger.Error(e, "An error occurred processing the pipeline");
taskCompletion.TrySetException(e);
}
void OnDocumentException(Exception exception, Document document)
{
try
{
_onDocumentError(new DocumentError(document, exception));
}
catch (Exception e)
{
Logger.Error(e, "An error occurred when calling the error handler");
taskCompletion.TrySetException(e);
}
}
async Task <WaivesDocument> CreateDocument(Document d, CancellationToken ct)
{
Logger.Info("Started processing '{DocumentSourceId}'", d.SourceId);
var httpDocument = await _documentFactory.CreateDocumentAsync(d, ct).ConfigureAwait(false);
return(new WaivesDocument(d, httpDocument));
}
async Task DeleteDocument(WaivesDocument d)
{
try
{
await d.HttpDocument.DeleteAsync().ConfigureAwait(false);
Logger.Info(
"Deleted document {DocumentId}. Processing of '{DocumentSourceId}' complete.",
d.Id,
d.Source.SourceId);
}
catch (Exception e)
{
Logger.Error(e, "An error occurred when deleting '{DocumentId}''", d.Id);
taskCompletion.TrySetException(e);
}
}
Logger.Info("Pipeline started");
var documentProcessor = new DocumentProcessor(
CreateDocument,
_documentActions,
DeleteDocument,
OnDocumentException);
var pipelineObserver = new ConcurrentPipelineObserver(
documentProcessor,
OnPipelineComplete,
OnPipelineError,
_maxConcurrency,
cancellationToken);
var connection = _documentSource.Subscribe(pipelineObserver);
try
{
await taskCompletion.Task;
}
finally
{
connection.Dispose();
}
}
