/// <summary>
/// Transform the items into dictionary entries and add them to the given dictionary.
/// </summary>
public static Task ParallelAddToConcurrentDictionaryAsync <T, TKey, TValue>(
this IEnumerable <T> items, ConcurrentDictionary <TKey, TValue> dictionary, Func <T, TKey> getKey, Func <T, TValue> getValue)
{
var block = new ActionBlock <T>(item => { dictionary[getKey(item)] = getValue(item); }, AllProcessors);
return(block.PostAllAndComplete(items));
}
/// <summary>
/// Perform the function on all items in parallel.
/// </summary>
public static Task ParallelForEachAsync <T>(this IEnumerable <T> items, int maxDegreeOfParallelism, Func <T, Task> loopBody)
{
var block = new ActionBlock <T>(loopBody, new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = maxDegreeOfParallelism
});
return(block.PostAllAndComplete(items));
}
public async Task PostAllAndCompleteTest()
{
const int size = 100;
var flags = new bool[size];
var actionBlock = new ActionBlock <int>(
i => { flags[i] = true; },
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = size / 7
});
await actionBlock.PostAllAndComplete(Enumerable.Range(0, size));
flags.All(flag => flag).Should().BeTrue();
}
public async Task SameObjectAcquiresMultipleTimes(string componentName)
{
var context = new Context(Logger);
using (var testDirectory = new DisposableDirectory(FileSystem))
using (var lock1 = new DirectoryLock(testDirectory.Path / "dir1", FileSystem, TimeSpan.FromSeconds(1), componentName))
{
var failedToAcquire = false;
var actionBlock = new ActionBlock <ValueUnit>(async _ =>
{
if (!(await lock1.AcquireAsync(context)).LockAcquired)
{
failedToAcquire = true;
}
});
await actionBlock.PostAllAndComplete(Enumerable.Repeat(ValueUnit.Void, 5));
failedToAcquire.Should().BeFalse();
}
}
/// <summary>
/// Perform the action on all items in parallel.
/// </summary>
public static Task ParallelForEachAsync <T>(this IEnumerable <T> items, Action <T> loopBody)
{
var block = new ActionBlock <T>(loopBody, AllProcessors);
return(block.PostAllAndComplete(items));
}
private async Task RunStats(AbsolutePath rootPath)
{
const int dividerWidth = 20;
try
{
var allContentFiles = default(FileStats);
var sharedFiles = default(FileStats);
var privateFiles = default(FileStats);
var cacheDeduplicatedFiles = default(FileStats);
var outputDeduplicatedFiles = default(FileStats);
var processFileInfoActionBlock = new ActionBlock <FileInfo>(
fileInfo =>
{
var hardLinkCount = 0;
try
{
hardLinkCount = _fileSystem.GetHardLinkCount(fileInfo.FullPath);
}
catch (Exception ex)
{
_logger.Warning("Failed to open content file at {0}. {1}", fileInfo.FullPath, ex);
}
_logger.Diagnostic(
"Processing file=[{0}] size=[{1}] hardLinkCount=[{2}]", fileInfo.FullPath, fileInfo.Length, hardLinkCount);
allContentFiles.Increase(1, fileInfo.Length);
if (hardLinkCount == 1)
{
privateFiles.Increase(1, fileInfo.Length);
}
else
{
if (hardLinkCount > 1)
{
sharedFiles.Increase(1, fileInfo.Length);
cacheDeduplicatedFiles.Increase(hardLinkCount - 1, fileInfo.Length);
if (hardLinkCount > 2)
{
outputDeduplicatedFiles.Increase(hardLinkCount - 2, fileInfo.Length);
}
}
}
},
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = Environment.ProcessorCount
}
);
var processFileInfoTask =
processFileInfoActionBlock.PostAllAndComplete(EnumerateBlobPathsFromDisk(rootPath));
var metadataFileInfos = _fileSystem.EnumerateFiles(rootPath, EnumerateOptions.None).ToList();
var metadataFiles = new FileStats(
metadataFileInfos.Count, metadataFileInfos.Select(fileInfo => fileInfo.Length).Aggregate((s1, s2) => s1 + s2));
await processFileInfoTask.ConfigureAwait(false);
_logger.Always("Statistics for cache at {0}", rootPath);
_logger.Always(new string('=', dividerWidth));
_logger.Always(
"{0:N0} total content files ({1}) in cache",
allContentFiles.FileCount,
allContentFiles.ByteCount.ToSizeExpression());
_logger.Always(
"{0:N0} private content files ({1}) in cache",
privateFiles.FileCount,
privateFiles.ByteCount.ToSizeExpression());
_logger.Always(
"{0:N0} non-content files ({1})",
metadataFiles.FileCount,
metadataFiles.ByteCount.ToSizeExpression());
_logger.Always(string.Empty);
_logger.Always("Shared Content");
_logger.Always(new string('-', dividerWidth));
_logger.Always(
"{0:N0} shared files ({1}) in cache",
sharedFiles.FileCount,
sharedFiles.ByteCount.ToSizeExpression());
_logger.Always(
"Use of hard links avoided duplication on disk of {0} for {1} savings",
cacheDeduplicatedFiles.ByteCount.ToSizeExpression(),
Percent(cacheDeduplicatedFiles.ByteCount, sharedFiles.ByteCount + cacheDeduplicatedFiles.ByteCount));
_logger.Always(
"Use of hard links avoided duplication in output directories of {0} for {1} savings",
outputDeduplicatedFiles.ByteCount.ToSizeExpression(),
Percent(outputDeduplicatedFiles.ByteCount, sharedFiles.ByteCount + outputDeduplicatedFiles.ByteCount));
}
catch (Exception exception)
{
_logger.Error(exception.Message);
_logger.Debug(exception);
}
}