public async Task Parse(string[] files, bool recursive, int?max, CancellationToken token)
{
_logger.LogInformation($"Parsing files, Recursive={recursive}, Max={max}, {(files.Select(x => $"File: {x}").ToStringVector(Environment.NewLine))}");
using var monitorToken = CancellationTokenSource.CreateLinkedTokenSource(token);
Stopwatch sw = Stopwatch.StartNew();
Console.CancelKeyPress += (sender, e) =>
{
e.Cancel = false;
monitorToken.Cancel();
};
var parseOption = new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = 20, BoundedCapacity = 1000
};
var batchOption = new GroupingDataflowBlockOptions {
BoundedCapacity = 1000
};
var writeOption = new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = 1, BoundedCapacity = 1000
};
var linkOption = new DataflowLinkOptions {
PropagateCompletion = true
};
TransformBlock <string, NmeaRecord?> toParse = new TransformBlock <string, NmeaRecord?>(x => _parseNmea.Parse(x), parseOption);
BatchBlock <NmeaRecord?> toSaveBatch = new BatchBlock <NmeaRecord?>(100, batchOption);
ActionBlock <NmeaRecord?[]> toSave = new ActionBlock <NmeaRecord?[]>(x => _store.Write(x), writeOption);
toParse.LinkTo(toSaveBatch, linkOption);
_ = toParse.Completion.ContinueWith(delegate { toSaveBatch.Complete(); });
toSaveBatch.LinkTo(toSave, linkOption);
_ = toSaveBatch.Completion.ContinueWith(delegate { toSave.Complete(); });
_counters.Clear();
_tracking.Load();
_counters.Sampler = x =>
{
x.Set(Counter.ToParseIn, toParse.InputCount);
x.Set(Counter.ToParseOut, toParse.OutputCount);
x.Set(Counter.ToSaveIn, toParse.InputCount);
x.Set(Counter.ToSaveOut, toParse.OutputCount);
};
// Start the process
IReadOnlyList <string> readFiles = files
.SelectMany(x => _fileReader.GetFiles(x !, true))
.Select(x => _tracking.Check(x))
.Where(x => x != null)
.TakeWhile((x, i) => max == null || i < (int)max)
.ToList() !;
_counters.Monitor(monitorToken.Token);
_counters.Set(Counter.FileQueued, readFiles.Count);
foreach (var file in readFiles)
{
if (monitorToken.IsCancellationRequested)
{
break;
}
_tracking.Add(file);
await _fileReader.ReadFile(file, toParse);
}
toParse.Complete();
await toSave.Completion;
await _counters.Completion(monitorToken.Token);
_logger.LogInformation("Shutting down");
monitorToken.Cancel();
_store.Close();
_tracking.Save();
// Report on performance
sw.Stop();
_logger.LogInformation($"Final: {_counters.FinalScore(sw.Elapsed)}");
}
