private static async Task ScanDirectoryParallelAsync <T>(string path, ScannerOptions options, DependencyFound onDependencyFound, CancellationToken cancellationToken = default)
where T : struct, IEnabledScannersArray
{
var fileSystem = options.FileSystem;
var filesToScanChannel = Channel.CreateBounded <FileToScan <T> >(new BoundedChannelOptions(10000)
{
AllowSynchronousContinuations = true,
SingleWriter = true,
SingleReader = false,
FullMode = BoundedChannelFullMode.Wait,
});
// Start enumerating
var enumeratorTask = Task.Run(async() =>
{
using var enumerator = new ScannerFileEnumerator <T>(path, options);
while (enumerator.MoveNext())
{
await filesToScanChannel.Writer.WriteAsync(enumerator.Current, cancellationToken).ConfigureAwait(false);
}
filesToScanChannel.Writer.Complete();
}, cancellationToken);
// Parse files
var tasks = new Task[options.DegreeOfParallelism + 1];
tasks[0] = enumeratorTask;
Array.Fill(tasks, Task.Run(async() =>
{
var reader = filesToScanChannel.Reader;
while (await reader.WaitToReadAsync(cancellationToken).ConfigureAwait(false))
{
while (reader.TryRead(out var entry))
{
var scanFileContext = new ScanFileContext(entry.FullPath, onDependencyFound, fileSystem, cancellationToken);
try
{
for (var i = 0; i < options.Scanners.Count; i++)
{
if (!entry.Scanners.Get(i))
{
continue;
}
var scanner = options.Scanners[i];
scanFileContext.ResetStream();
await scanner.ScanAsync(scanFileContext).ConfigureAwait(false);
}
}
finally
{
await scanFileContext.DisposeAsync().ConfigureAwait(false);
}
}
}
}, cancellationToken), startIndex: 1, options.DegreeOfParallelism);
await Task.WhenAll(tasks).ConfigureAwait(false);
}
private static async Task ScanDirectoryAsync <T>(string path, ScannerOptions options, DependencyFound onDependencyFound, CancellationToken cancellationToken = default)
where T : struct, IEnabledScannersArray
{
var fileSystem = options.FileSystem;
using var enumerator = new ScannerFileEnumerator <T>(path, options);
while (enumerator.MoveNext())
{
var entry = enumerator.Current;
var scanFileContext = new ScanFileContext(entry.FullPath, onDependencyFound, fileSystem, cancellationToken);
try
{
for (var i = 0; i < options.Scanners.Count; i++)
{
if (!entry.Scanners.Get(i))
{
continue;
}
var scanner = options.Scanners[i];
scanFileContext.ResetStream();
await scanner.ScanAsync(scanFileContext).ConfigureAwait(false);
}
}
finally
{
await scanFileContext.DisposeAsync().ConfigureAwait(false);
}
}
}
private static async IAsyncEnumerable <Dependency> ScanDirectoryParallelAsync <T>(string path, ScannerOptions options, [EnumeratorCancellation] CancellationToken cancellationToken = default)
where T : struct, IEnabledScannersArray
{
var fileSystem = options.FileSystem;
var filesToScanChannel = Channel.CreateBounded <FileToScan <T> >(new BoundedChannelOptions(1000)
{
AllowSynchronousContinuations = true,
SingleWriter = true,
SingleReader = false,
FullMode = BoundedChannelFullMode.Wait,
});
var dependenciesChannel = Channel.CreateUnbounded <Dependency>(new UnboundedChannelOptions
{
AllowSynchronousContinuations = true,
SingleWriter = false,
SingleReader = true,
});
// Start enumerating
var enumeratorTask = Task.Run(async() =>
{
using var enumerator = new ScannerFileEnumerator <T>(path, options);
while (enumerator.MoveNext())
{
await filesToScanChannel.Writer.WriteAsync(enumerator.Current, cancellationToken).ConfigureAwait(false);
}
filesToScanChannel.Writer.Complete();
}, cancellationToken);
// Parse files
var exceptions = new ConcurrentBag <Exception>();
var tasks = new Task[options.DegreeOfParallelism];
Array.Fill(tasks, Task.Run(async() =>
{
var reader = filesToScanChannel.Reader;
while (await reader.WaitToReadAsync(cancellationToken).ConfigureAwait(false))
{
while (reader.TryRead(out var entry))
{
var scanFileContext = new ScanFileContext(entry.FullPath, d => dependenciesChannel.Writer.WriteAsync(d, cancellationToken), fileSystem, cancellationToken);
try
{
for (var i = 0; i < options.Scanners.Count; i++)
{
if (!entry.Scanners.Get(i))
{
continue;
}
var scanner = options.Scanners[i];
scanFileContext.ResetStream();
await scanner.ScanAsync(scanFileContext).ConfigureAwait(false);
}
}
finally
{
await scanFileContext.DisposeAsync().ConfigureAwait(false);
}
}
}
}, cancellationToken));
var whenAllTasks = Task.WhenAll(tasks);
var writerCompleteTask = whenAllTasks.ContinueWith(_ => dependenciesChannel.Writer.Complete(), cancellationToken);
await foreach (var value in dependenciesChannel.Reader.ReadAllAsync(cancellationToken).WithCancellation(cancellationToken).ConfigureAwait(false))
{
yield return(value);
}
await writerCompleteTask.ConfigureAwait(false);
await whenAllTasks.ConfigureAwait(false);
}
private static async IAsyncEnumerable <Dependency> ScanDirectorySingleThreadedAsync(string path, ScannerOptions options, [EnumeratorCancellation] CancellationToken cancellationToken = default)
{
var result = new List <Dependency>();
await ScanDirectoryAsync(path, options, dep => { result.Add(dep); return(default); }, cancellationToken).ConfigureAwait(false);
