static async Task Run(UploadOptions uploadOptions)
{
var services = new ServiceCollection();
services.AddLogging(builder =>
{
builder.SetMinimumLevel(uploadOptions.LogLevel);
builder.AddConsole();
});
if (uploadOptions.UsePlate2Format)
{
services.AddTransient <IWorkItemGenerator, PlateFile2WorkItemGenerator>();
}
else
{
services.AddTransient <IWorkItemGenerator, PlateFileWorkItemGenerator>();
}
services.AddSingleton <AzurePlateTilePyramid>();
services.AddSingleton <Processor>();
services.AddSingleton(new AzurePlateTilePyramidOptions
{
CreateContainer = true,
SkipIfExists = uploadOptions.SkipExisting,
});
services.AddAzureClients(builder =>
{
builder.AddBlobServiceClient(uploadOptions.Storage);
if (uploadOptions.Interactive)
{
builder.UseCredential(new InteractiveBrowserCredential());
}
else
{
builder.UseCredential(new DefaultAzureCredential());
}
});
var options = new ProcessorOptions
{
BaseUrl = uploadOptions.BaseUrl,
Files = uploadOptions.File.Select(p => p.FullName)
};
using var container = services.BuildServiceProvider();
await container.GetRequiredService <Processor>().ProcessAsync(options, default);
}
public async Task ProcessAsync(ProcessorOptions options, CancellationToken token)
{
var files = Channel.CreateUnbounded <string>(new UnboundedChannelOptions
{
SingleWriter = true
});
foreach (var file in options.Files)
{
while (!files.Writer.TryWrite(file))
{
}
}
files.Writer.Complete();
var tasks = Channel.CreateBounded <Func <int, int, Task> >(new BoundedChannelOptions(10000)
{
FullMode = BoundedChannelFullMode.Wait,
SingleReader = false,
SingleWriter = false
});
int total = 0;
var fileProcessing = Enumerable.Range(0, options.FileProcessorCount).Select(async _ =>
{
while (!files.Reader.Completion.IsCompleted)
{
var file = await files.Reader.ReadAsync(token);
_logger.LogInformation("Adding {File}", file);
foreach (var generator in _generators)
{
foreach (var task in generator.GenerateWorkItems(file, options.BaseUrl, token))
{
await tasks.Writer.WriteAsync(task, token);
Interlocked.Increment(ref total);
}
}
}
}).ToList();
var c = 0;
var uploadProcessing = Enumerable.Range(0, options.UploadingCount).Select(async _ =>
{
while (!tasks.Reader.Completion.IsCompleted)
{
var count = Interlocked.Increment(ref c);
var task = await tasks.Reader.ReadAsync();
try
{
await task(count, total);
}
catch (Exception e)
{
_logger.LogError(e, "Unexpected error running task");
}
}
}).ToList();
await Task.WhenAll(fileProcessing);
tasks.Writer.Complete();
await Task.WhenAll(uploadProcessing);
}
