public Task StartAsync(CancellationToken cancellationToken)
{
var recognizer = new Recognizer(_logger, _configuration);
Directory
.EnumerateFiles(_configuration.Value.ImagesPath, "*.jpg", SearchOption.TopDirectoryOnly)
.AsParallel()
.WithDegreeOfParallelism(10)
.ForAll(file => CreateWorker(new Input(file, ++_queuedFiles)).GetAwaiter().GetResult());
_logger.LogInformation($"Total # files queued: {_queuedFiles}");
async Task CreateWorker(Input input)
{
_logger.LogInformation($"Queued {input.FilePath} ({_queuedFiles})");
try
{
var result = await recognizer.RecognizeAsync(input);
await _resultWriter.PersistResultAsync(result);
_logger.LogInformation($"Transformed {result.Input.FilePath} ({result.Input.FileIndex}).");
}
catch (Exception e)
{
_logger.LogError(e, $"Error processing {input.FilePath} ({input.FileIndex}): {e.Message}");
}
}
Console.WriteLine("Finished. Press any key to exit.");
Console.ReadKey();
return(Task.CompletedTask);
}
private async Task ProcessQueuedFileAsync(Input input)
{
_logger.LogInformation($"Transforming {input.FilePath} ({input.FileIndex}).");
try
{
var result = await _recognizer.RecognizeAsync(input);
await _resultWriter.PersistResultAsync(result);
_logger.LogInformation($"Transformed {result.Input.FilePath} ({result.Input.FileIndex}).");
}
catch (Exception e)
{
_logger.LogError(e, $"Error processing {input.FilePath} ({input.FileIndex}): {e.Message}");
}
}
public async Task StartAsync(CancellationToken cancellationToken)
{
var recognizer = new Recognizer(_logger, _configuration);
var tasks = new List <Task>();
// Start the producer
foreach (var file in Directory.EnumerateFiles(_configuration.Value.ImagesPath, "*.jpg", SearchOption.TopDirectoryOnly))
{
tasks.Add(CreateWorker(new Input(file, ++_queuedFiles)));
_logger.LogInformation($"Queued {file} ({_queuedFiles})");
}
_logger.LogInformation($"Total # files queued: {_queuedFiles}");
// Wait for process to complete
await Task.WhenAll(tasks);
Console.WriteLine("Finished. Press any key to exit.");
Console.ReadKey();
// Set up the consumer
async Task CreateWorker(Input input)
{
await _semaphoreSlim.WaitAsync();
_logger.LogInformation($"Transforming {input.FilePath} ({input.FileIndex}).");
try
{
var result = await recognizer.RecognizeAsync(input);
await _resultWriter.PersistResultAsync(result);
_logger.LogInformation($"Transformed {result.Input.FilePath} ({result.Input.FileIndex}).");
}
catch (Exception e)
{
_logger.LogError(e, $"Error processing {input.FilePath} ({input.FileIndex}): {e.Message}");
}
finally
{
_semaphoreSlim.Release();
}
}
}
public async Task StartAsync(CancellationToken cancellationToken)
{
var tcs = new TaskCompletionSource <Exception>();
var recognizer = new Recognizer(_logger, _configuration);
// Set up the consumer
using (_fileSubject
.AsObservable()
.Select(input =>
Observable.Empty <Result>()
.Delay(TimeSpan.FromSeconds(1))
.Concat(Observable.FromAsync(() => recognizer.RecognizeAsync(input))))
.Merge(10)
.Subscribe(async result =>
{
await _resultWriter.PersistResultAsync(result);
_logger.LogInformation($"Transformed {result.Input.FilePath} ({result.Input.FileIndex}).");
}, e => tcs.TrySetResult(e),
() => tcs.TrySetResult(null)))
{
// Start the producer
foreach (var file in Directory.EnumerateFiles(_configuration.Value.ImagesPath, "*.jpg",
SearchOption.TopDirectoryOnly))
{
_fileSubject.OnNext(new Input(file, ++_queuedFiles));
_logger.LogInformation($"Queued {file} ({_queuedFiles})");
}
_logger.LogInformation($"Total # files queued: {_queuedFiles}");
// Signal completion of producer
_fileSubject.OnCompleted();
// Wait for pipeline to drain
var exception = await tcs.Task;
if (exception != null)
{
_logger.LogError(exception, exception.Message);
}
Console.WriteLine("Finished. Press any key to exit.");
Console.ReadKey();
}
}
public async Task StartAsync(CancellationToken cancellationToken)
{
var recognizer = new Recognizer(_logger, _configuration);
// Set up the consumer (DataFlow pipeline)
var transformer = new TransformBlock <Input, Result>(async input =>
{
_logger.LogInformation($"Transforming {input.FilePath} ({input.FileIndex}).");
return(await recognizer.RecognizeAsync(input));
}, new ExecutionDataflowBlockOptions
{
CancellationToken = cancellationToken,
MaxDegreeOfParallelism = 10
});
var writer = new ActionBlock <Result>(async result =>
{
await _resultWriter.PersistResultAsync(result);
_logger.LogInformation($"Transformed {result.Input.FilePath} ({result.Input.FileIndex}).");
}, new ExecutionDataflowBlockOptions
{
CancellationToken = cancellationToken,
MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded
});
transformer.LinkTo(writer, new DataflowLinkOptions {
PropagateCompletion = true
});
// Start the producer
QueueImageFiles(_configuration.Value.ImagesPath, transformer);
// Signal completion
transformer.Complete();
// Wait for pipeline to drain
await writer.Completion;
Console.WriteLine("Finished. Press any key to exit.");
Console.ReadKey();
}