public Task PutSameContentManyTimesTest(bool useRedundantPutFileShortcut)
{
var context = new Context(Logger);
ContentStoreSettings = new ContentStoreSettings()
{
UseRedundantPutFileShortcut = useRedundantPutFileShortcut
};
return(TestStore(context, Clock, async store =>
{
byte[] bytes = ThreadSafeRandom.GetBytes(ValueSize);
ContentHash contentHash = bytes.CalculateHash(ContentHashType);
// Verify content doesn't exist yet in store
Assert.False(await store.ContainsAsync(context, contentHash, null));
using (var tempDirectory = new DisposableDirectory(FileSystem))
{
ContentHash hashFromPut;
using (var pinContext = store.CreatePinContext())
{
var concurrency = 24;
var iterations = 100;
var items = Enumerable.Range(0, concurrency).Select(i =>
{
AbsolutePath pathToContent = tempDirectory.Path / $"tempContent{i}.txt";
FileSystem.WriteAllBytes(pathToContent, bytes);
return (pathToContent, iterations);
}).ToArray();
await ParallelAlgorithms.WhenDoneAsync(24, CancellationToken.None, async(scheduleItem, item) =>
{
// Put the content into the store w/ hard link
var r = await store.PutFileAsync(
context, item.pathToContent, FileRealizationMode.Any, ContentHashType, new PinRequest(pinContext));
hashFromPut = r.ContentHash;
Clock.Increment();
Assert.True(pinContext.Contains(hashFromPut));
if (item.iterations != 0)
{
scheduleItem((item.pathToContent, item.iterations - 1));
}
},
items);
}
}
}));
}
public async Task TestParallelAlgorithmsCancellationTokenAsync()
{
// cancel after 2 seconds
var cts = new CancellationTokenSource();
cts.CancelAfter(TimeSpan.FromSeconds(2));
// run something that never ends in parallel
await ParallelAlgorithms.WhenDoneAsync(
degreeOfParallelism : 20,
cts.Token,
action : (scheduleItem, item) =>
{
// keep rescheduling the same item forever
scheduleItem(item);
return(Task.Delay(TimeSpan.FromMilliseconds(10)));
},
items : Enumerable.Range(0, 1000));
XAssert.IsTrue(cts.IsCancellationRequested);
}
/// <summary>
/// Starts by parsing <paramref name="configPath"/>, recursively continuing to parse any files imported via an 'importFile' call.
///
/// Any errors are logged to <see cref="Logger"/>.
///
/// Returns a map of parsed files; the result is never null, but in case of an error the content may be unspecified.
/// </summary>
private async Task <IReadOnlyDictionary <AbsolutePath, ISourceFile> > ParseConfigFileAsync(AbsolutePath configPath)
{
// Set of specs being processed or queued for processing
var queuedSpecs = new ConcurrentDictionary <AbsolutePath, Unit>()
{
{ configPath, Unit.Void }
};
// Set of parsed files
var result = new ConcurrentDictionary <AbsolutePath, ISourceFile>();
await ParallelAlgorithms.WhenDoneAsync(
DegreeOfParallelism,
Context.CancellationToken,
async (addItem, path) =>
{
// TODO: File bug to ensure we fail on errors.
var parseResult = await ParseFileAndDiscoverImportsAsync(path);
var numberOfProcessedConfigs = FrontEndStatistics.ConfigurationProcessing.Increment();
NotifyProgress(numberOfProcessedConfigs);
result[path] = parseResult.Source;
if (parseResult.Imports?.Count > 0)
{
foreach (var dependency in parseResult.Imports)
{
// Add the dependency for parsing only if the dependency was not processed or scheduled for processing.
if (queuedSpecs.TryAdd(dependency, Unit.Void))
{
addItem(dependency);
}
}
}
},
configPath);
return(result.ToDictionary(kvp => kvp.Key, kvp => kvp.Value));
}
private async Task UploadFilesAsync(OperationContext context, List <AbsolutePath> files, ConcurrentDictionary <string, string> newCheckpointInfo, string incrementalCheckpointsPrefix)
{
if (_configuration.IncrementalCheckpointDegreeOfParallelism <= 1)
{
foreach (var file in files)
{
await UploadOrTouchFileAsync(context, file, newCheckpointInfo, incrementalCheckpointsPrefix);
}
}
else
{
await ParallelAlgorithms.WhenDoneAsync(
_configuration.IncrementalCheckpointDegreeOfParallelism,
context.Token,
action : async(addItem, file) =>
{
// Intentionally using async/await to generate a state machine that will have the current method name in it (to simplify postmortem).
await UploadOrTouchFileAsync(context, file, newCheckpointInfo, incrementalCheckpointsPrefix);
},
items : files.ToArray());
}
}
private async Task RestoreFilesAsync(OperationContext context, AbsolutePath checkpointTargetDirectory, Dictionary <string, string> newCheckpointInfo)
{
if (_configuration.IncrementalCheckpointDegreeOfParallelism <= 1)
{
foreach (var(key, value) in newCheckpointInfo)
{
await RestoreFileAsync(context, checkpointTargetDirectory, key, value).ThrowIfFailure();
}
}
else
{
await ParallelAlgorithms.WhenDoneAsync(
_configuration.IncrementalCheckpointDegreeOfParallelism,
context.Token,
action : async(addItem, kvp) =>
{
var key = kvp.Key;
var value = kvp.Value;
await RestoreFileAsync(context, checkpointTargetDirectory, key, value).ThrowIfFailure();
},
items : newCheckpointInfo.ToArray());
}
}
