Example #1
0
        public async Task Discover()
        {
            var options = new ExecutionDataflowBlockOptions
            {
                BoundedCapacity           = 10000,
                MaxDegreeOfParallelism    = Environment.ProcessorCount,
                SingleProducerConstrained = true
            };

            var rootBuffer = new BufferBlock <string>();
            var fileBuffer = new BufferBlock <DiscoveredFile>();

            var countFileBlock = new TransformBlock <DiscoveredFile, DiscoveredFile>(file =>
            {
                Interlocked.Increment(ref _discoveredFiles);
                return(file);
            });

            var countRootsBlock = new TransformBlock <string, string>(root =>
            {
                Interlocked.Increment(ref _discoveredRoots);
                return(root);
            });

            var discoverFilesBlock = new TransformManyBlock <string, DiscoveredFile>(root =>
            {
                try
                {
                    var existingSources = _metadataStore.ExistingSources(_sourceName, root);
                    return(_source.Discover(root, existingSources));
                }
                catch (Exception ex)
                {
                    _userInteraction.ReportError(ex.Message);
                    return(new DiscoveredFile[] { });
                }
                finally
                {
                    Interlocked.Increment(ref _processedRoots);
                }
            }, options);

            var injestFileBlock = new ActionBlock <DiscoveredFile>(async file =>
            {
                try
                {
                    var stream      = _source.GetContents(file);
                    string metadata = await _source.GetMetadata(file.Path);
                    await _metadataStore.AddDiscoveredFile(stream, file.Name, file.Extension, file.Path, metadata, _sourceName);
                }
                catch (Exception ex)
                {
                    _userInteraction.ReportError(ex.Message);
                }
                finally
                {
                    Interlocked.Increment(ref _processedFiles);
                }
            }, options);

            var linkOptions = new DataflowLinkOptions {
                PropagateCompletion = true
            };

            countRootsBlock.LinkTo(rootBuffer, linkOptions);
            rootBuffer.LinkTo(discoverFilesBlock, linkOptions);
            discoverFilesBlock.LinkTo(countFileBlock, linkOptions);
            countFileBlock.LinkTo(fileBuffer, linkOptions);
            fileBuffer.LinkTo(injestFileBlock, linkOptions);


            Task getRoots = new Task(async() =>
            {
                var roots = _source.GetRoots();
                foreach (var root in roots)
                {
                    await countRootsBlock.SendAsync(root);
                }
                countRootsBlock.Complete();
            }, TaskCreationOptions.LongRunning);

            getRoots.Start();

            await injestFileBlock.Completion;
        }