static async Task Run(UploadOptions uploadOptions) { var services = new ServiceCollection(); services.AddLogging(builder => { builder.SetMinimumLevel(uploadOptions.LogLevel); builder.AddConsole(); }); if (uploadOptions.UsePlate2Format) { services.AddTransient <IWorkItemGenerator, PlateFile2WorkItemGenerator>(); } else { services.AddTransient <IWorkItemGenerator, PlateFileWorkItemGenerator>(); } services.AddSingleton <AzurePlateTilePyramid>(); services.AddSingleton <Processor>(); services.AddSingleton(new AzurePlateTilePyramidOptions { CreateContainer = true, SkipIfExists = uploadOptions.SkipExisting, }); services.AddAzureClients(builder => { builder.AddBlobServiceClient(uploadOptions.Storage); if (uploadOptions.Interactive) { builder.UseCredential(new InteractiveBrowserCredential()); } else { builder.UseCredential(new DefaultAzureCredential()); } }); var options = new ProcessorOptions { BaseUrl = uploadOptions.BaseUrl, Files = uploadOptions.File.Select(p => p.FullName) }; using var container = services.BuildServiceProvider(); await container.GetRequiredService <Processor>().ProcessAsync(options, default); }
public async Task ProcessAsync(ProcessorOptions options, CancellationToken token) { var files = Channel.CreateUnbounded <string>(new UnboundedChannelOptions { SingleWriter = true }); foreach (var file in options.Files) { while (!files.Writer.TryWrite(file)) { } } files.Writer.Complete(); var tasks = Channel.CreateBounded <Func <int, int, Task> >(new BoundedChannelOptions(10000) { FullMode = BoundedChannelFullMode.Wait, SingleReader = false, SingleWriter = false }); int total = 0; var fileProcessing = Enumerable.Range(0, options.FileProcessorCount).Select(async _ => { while (!files.Reader.Completion.IsCompleted) { var file = await files.Reader.ReadAsync(token); _logger.LogInformation("Adding {File}", file); foreach (var generator in _generators) { foreach (var task in generator.GenerateWorkItems(file, options.BaseUrl, token)) { await tasks.Writer.WriteAsync(task, token); Interlocked.Increment(ref total); } } } }).ToList(); var c = 0; var uploadProcessing = Enumerable.Range(0, options.UploadingCount).Select(async _ => { while (!tasks.Reader.Completion.IsCompleted) { var count = Interlocked.Increment(ref c); var task = await tasks.Reader.ReadAsync(); try { await task(count, total); } catch (Exception e) { _logger.LogError(e, "Unexpected error running task"); } } }).ToList(); await Task.WhenAll(fileProcessing); tasks.Writer.Complete(); await Task.WhenAll(uploadProcessing); }