public async Task Discover()
{
var options = new ExecutionDataflowBlockOptions
{
BoundedCapacity = 10000,
MaxDegreeOfParallelism = Environment.ProcessorCount,
SingleProducerConstrained = true
};
var rootBuffer = new BufferBlock <string>();
var fileBuffer = new BufferBlock <DiscoveredFile>();
var countFileBlock = new TransformBlock <DiscoveredFile, DiscoveredFile>(file =>
{
Interlocked.Increment(ref _discoveredFiles);
return(file);
});
var countRootsBlock = new TransformBlock <string, string>(root =>
{
Interlocked.Increment(ref _discoveredRoots);
return(root);
});
var discoverFilesBlock = new TransformManyBlock <string, DiscoveredFile>(root =>
{
try
{
var existingSources = _metadataStore.ExistingSources(_sourceName, root);
return(_source.Discover(root, existingSources));
}
catch (Exception ex)
{
_userInteraction.ReportError(ex.Message);
return(new DiscoveredFile[] { });
}
finally
{
Interlocked.Increment(ref _processedRoots);
}
}, options);
var injestFileBlock = new ActionBlock <DiscoveredFile>(async file =>
{
try
{
var stream = _source.GetContents(file);
string metadata = await _source.GetMetadata(file.Path);
await _metadataStore.AddDiscoveredFile(stream, file.Name, file.Extension, file.Path, metadata, _sourceName);
}
catch (Exception ex)
{
_userInteraction.ReportError(ex.Message);
}
finally
{
Interlocked.Increment(ref _processedFiles);
}
}, options);
var linkOptions = new DataflowLinkOptions {
PropagateCompletion = true
};
countRootsBlock.LinkTo(rootBuffer, linkOptions);
rootBuffer.LinkTo(discoverFilesBlock, linkOptions);
discoverFilesBlock.LinkTo(countFileBlock, linkOptions);
countFileBlock.LinkTo(fileBuffer, linkOptions);
fileBuffer.LinkTo(injestFileBlock, linkOptions);
Task getRoots = new Task(async() =>
{
var roots = _source.GetRoots();
foreach (var root in roots)
{
await countRootsBlock.SendAsync(root);
}
countRootsBlock.Complete();
}, TaskCreationOptions.LongRunning);
getRoots.Start();
await injestFileBlock.Completion;
}