/// <summary> /// Transform the items into dictionary entries and add them to the given dictionary. /// </summary> public static Task ParallelAddToConcurrentDictionaryAsync <T, TKey, TValue>( this IEnumerable <T> items, ConcurrentDictionary <TKey, TValue> dictionary, Func <T, TKey> getKey, Func <T, TValue> getValue) { var block = new ActionBlock <T>(item => { dictionary[getKey(item)] = getValue(item); }, AllProcessors); return(block.PostAllAndComplete(items)); }
/// <summary> /// Perform the function on all items in parallel. /// </summary> public static Task ParallelForEachAsync <T>(this IEnumerable <T> items, int maxDegreeOfParallelism, Func <T, Task> loopBody) { var block = new ActionBlock <T>(loopBody, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism }); return(block.PostAllAndComplete(items)); }
public async Task PostAllAndCompleteTest() { const int size = 100; var flags = new bool[size]; var actionBlock = new ActionBlock <int>( i => { flags[i] = true; }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = size / 7 }); await actionBlock.PostAllAndComplete(Enumerable.Range(0, size)); flags.All(flag => flag).Should().BeTrue(); }
public async Task SameObjectAcquiresMultipleTimes(string componentName) { var context = new Context(Logger); using (var testDirectory = new DisposableDirectory(FileSystem)) using (var lock1 = new DirectoryLock(testDirectory.Path / "dir1", FileSystem, TimeSpan.FromSeconds(1), componentName)) { var failedToAcquire = false; var actionBlock = new ActionBlock <ValueUnit>(async _ => { if (!(await lock1.AcquireAsync(context)).LockAcquired) { failedToAcquire = true; } }); await actionBlock.PostAllAndComplete(Enumerable.Repeat(ValueUnit.Void, 5)); failedToAcquire.Should().BeFalse(); } }
/// <summary> /// Perform the action on all items in parallel. /// </summary> public static Task ParallelForEachAsync <T>(this IEnumerable <T> items, Action <T> loopBody) { var block = new ActionBlock <T>(loopBody, AllProcessors); return(block.PostAllAndComplete(items)); }
private async Task RunStats(AbsolutePath rootPath) { const int dividerWidth = 20; try { var allContentFiles = default(FileStats); var sharedFiles = default(FileStats); var privateFiles = default(FileStats); var cacheDeduplicatedFiles = default(FileStats); var outputDeduplicatedFiles = default(FileStats); var processFileInfoActionBlock = new ActionBlock <FileInfo>( fileInfo => { var hardLinkCount = 0; try { hardLinkCount = _fileSystem.GetHardLinkCount(fileInfo.FullPath); } catch (Exception ex) { _logger.Warning("Failed to open content file at {0}. {1}", fileInfo.FullPath, ex); } _logger.Diagnostic( "Processing file=[{0}] size=[{1}] hardLinkCount=[{2}]", fileInfo.FullPath, fileInfo.Length, hardLinkCount); allContentFiles.Increase(1, fileInfo.Length); if (hardLinkCount == 1) { privateFiles.Increase(1, fileInfo.Length); } else { if (hardLinkCount > 1) { sharedFiles.Increase(1, fileInfo.Length); cacheDeduplicatedFiles.Increase(hardLinkCount - 1, fileInfo.Length); if (hardLinkCount > 2) { outputDeduplicatedFiles.Increase(hardLinkCount - 2, fileInfo.Length); } } } }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount } ); var processFileInfoTask = processFileInfoActionBlock.PostAllAndComplete(EnumerateBlobPathsFromDisk(rootPath)); var metadataFileInfos = _fileSystem.EnumerateFiles(rootPath, EnumerateOptions.None).ToList(); var metadataFiles = new FileStats( metadataFileInfos.Count, metadataFileInfos.Select(fileInfo => fileInfo.Length).Aggregate((s1, s2) => s1 + s2)); await processFileInfoTask.ConfigureAwait(false); _logger.Always("Statistics for cache at {0}", rootPath); _logger.Always(new string('=', dividerWidth)); _logger.Always( "{0:N0} total content files ({1}) in cache", allContentFiles.FileCount, allContentFiles.ByteCount.ToSizeExpression()); _logger.Always( "{0:N0} private content files ({1}) in cache", privateFiles.FileCount, privateFiles.ByteCount.ToSizeExpression()); _logger.Always( "{0:N0} non-content files ({1})", metadataFiles.FileCount, metadataFiles.ByteCount.ToSizeExpression()); _logger.Always(string.Empty); _logger.Always("Shared Content"); _logger.Always(new string('-', dividerWidth)); _logger.Always( "{0:N0} shared files ({1}) in cache", sharedFiles.FileCount, sharedFiles.ByteCount.ToSizeExpression()); _logger.Always( "Use of hard links avoided duplication on disk of {0} for {1} savings", cacheDeduplicatedFiles.ByteCount.ToSizeExpression(), Percent(cacheDeduplicatedFiles.ByteCount, sharedFiles.ByteCount + cacheDeduplicatedFiles.ByteCount)); _logger.Always( "Use of hard links avoided duplication in output directories of {0} for {1} savings", outputDeduplicatedFiles.ByteCount.ToSizeExpression(), Percent(outputDeduplicatedFiles.ByteCount, sharedFiles.ByteCount + outputDeduplicatedFiles.ByteCount)); } catch (Exception exception) { _logger.Error(exception.Message); _logger.Debug(exception); } }