public async Task RoundtripSimpleStructure()
{
var context = BuildXLContext.CreateInstanceForTesting();
var cache = new InMemoryArtifactContentCache(context);
var originalEntry = new PipCacheDescriptorV2Metadata()
{
Id = 123,
NumberOfWarnings = 456,
TraceInfo = "Good job."
};
Possible <ContentHash> maybeStored =
await cache.TrySerializeAndStoreContent(originalEntry);
XAssert.IsTrue(maybeStored.Succeeded);
var maybeDeserialized =
await cache.TryLoadAndDeserializeContent <PipCacheDescriptorV2Metadata>(maybeStored.Result);
XAssert.IsTrue(maybeDeserialized.Succeeded);
PipCacheDescriptorV2Metadata roundtripped = maybeDeserialized.Result;
XAssert.IsNotNull(roundtripped, "Expected content available");
XAssert.AreEqual(originalEntry.Id, roundtripped.Id);
XAssert.AreEqual(originalEntry.NumberOfWarnings, roundtripped.NumberOfWarnings);
XAssert.AreEqual(originalEntry.TraceInfo, roundtripped.TraceInfo);
}
/// <summary>
/// Report metadata and pathset from workers
/// </summary>
public virtual void ReportRemoteMetadataAndPathSet(
PipCacheDescriptorV2Metadata metadata,
ContentHash?metadataHash,
ObservedPathSet?pathSet,
ContentHash?pathSetHash,
WeakContentFingerprint?weakFingerprint,
StrongContentFingerprint?strongFingerprint,
bool isExecution)
{
}
/// <summary>
/// Stores cache descriptor metadata for pip
/// </summary>
public virtual async Task <Possible <ContentHash> > TryStoreMetadataAsync(PipCacheDescriptorV2Metadata metadata)
{
BoxRef <long> metadataSize = new BoxRef <long>();
var result = await ArtifactContentCache.TrySerializeAndStoreContent(metadata.ToEntry(), metadataSize);
if (result.Succeeded)
{
Counters.IncrementCounter(PipCachingCounter.StoredMetadataCount);
Counters.AddToCounter(PipCachingCounter.StoredMetadataSize, metadataSize.Value);
}
return(result);
}
/// <summary>
/// Serialize metadata to writer
/// </summary>
public static void WritePipCacheDescriptor(BuildXLWriter writer, PipCacheDescriptorV2Metadata metadata)
{
if (metadata != null)
{
writer.Write(true);
var blob = BondExtensions.Serialize(metadata);
writer.WriteCompact(blob.Count);
writer.Write(blob.Array, blob.Offset, blob.Count);
}
else
{
writer.Write(false);
}
}
private static void AssertMetadataEquals(PipCacheDescriptorV2Metadata metadata1, PipCacheDescriptorV2Metadata metadata2)
{
Assert.Equal(metadata1.StaticOutputHashes.Count, metadata2.StaticOutputHashes.Count);
for (int i = 0; i < metadata1.StaticOutputHashes.Count; i++)
{
Assert.Equal(metadata1.StaticOutputHashes[i].Info.FileName, metadata2.StaticOutputHashes[i].Info.FileName);
}
Assert.Equal(metadata1.DynamicOutputs.Count, metadata2.DynamicOutputs.Count);
for (int i = 0; i < metadata1.DynamicOutputs.Count; i++)
{
Assert.Equal(metadata1.DynamicOutputs[i].Count, metadata2.DynamicOutputs[i].Count);
for (int j = 0; j < metadata1.DynamicOutputs[i].Count; j++)
{
Assert.Equal(metadata1.DynamicOutputs[i][j].RelativePath, metadata2.DynamicOutputs[i][j].RelativePath);
Assert.Equal(metadata1.DynamicOutputs[i][j].Info.FileName, metadata2.DynamicOutputs[i][j].Info.FileName);
}
}
}
public async Task TestHistoricMetadataPathStringRoundtrip()
{
LoggingContext loggingContext = CreateLoggingContextForTest();
PipExecutionContext context;
HistoricMetadataCache cache = null;
var hmcFolderName = "hmc";
for (int i = 0; i < 3; i++)
{
CreateHistoricCache(loggingContext, hmcFolderName, out context, out cache, out var memoryArtifactCache);
var process1 = SchedulerTest.CreateDummyProcess(context, new PipId(1));
var process2 = SchedulerTest.CreateDummyProcess(context, new PipId(2));
var pathTable = context.PathTable;
// Add some random paths to ensure path table indices are different after loading
AbsolutePath.Create(pathTable, X("/H/aslj/sfas/832.stxt"));
AbsolutePath.Create(pathTable, X("/R/f/s/Historic"));
AbsolutePath.Create(pathTable, X("/M/hgf/sf4as/83afsd"));
AbsolutePath.Create(pathTable, X("/Z/bd/sfas/Cache"));
var abPath1 = AbsolutePath.Create(pathTable, X("/H/aslj/sfas/p1OUT.bin"));
var abPath2 = AbsolutePath.Create(pathTable, X("/H/aslj/sfas/P2.txt"));
var pathSet1 = ObservedPathSetTestUtilities.CreatePathSet(
pathTable,
X("/X/a/b/c"),
X("/X/d/e"),
X("/X/a/b/c/d"));
PipCacheDescriptorV2Metadata metadata1 =
new PipCacheDescriptorV2Metadata
{
StaticOutputHashes = new List <AbsolutePathFileMaterializationInfo>
{
new AbsolutePathFileMaterializationInfo
{
AbsolutePath = abPath1.GetName(pathTable).ToString(context.StringTable),
Info = new BondFileMaterializationInfo {
FileName = "p1OUT.bin"
}
}
}
};
var storedPathSet1 = await cache.TryStorePathSetAsync(pathSet1, preservePathCasing : false);
var storedMetadata1 = await cache.TryStoreMetadataAsync(metadata1);
var weakFingerprint1 = new WeakContentFingerprint(FingerprintUtilities.CreateRandom());
var strongFingerprint1 = new StrongContentFingerprint(FingerprintUtilities.CreateRandom());
var cacheEntry = new CacheEntry(storedMetadata1.Result, nameof(HistoricMetadataCacheTests), ArrayView <ContentHash> .Empty);
var publishedCacheEntry = await cache.TryPublishCacheEntryAsync(process1, weakFingerprint1, storedPathSet1.Result, strongFingerprint1, cacheEntry);
var pathSet2 = ObservedPathSetTestUtilities.CreatePathSet(
pathTable,
X("/F/a/y/c"),
X("/B/d/e"),
X("/G/a/z/c/d"),
X("/B/a/b/c"));
PipCacheDescriptorV2Metadata metadata2 =
new PipCacheDescriptorV2Metadata
{
StaticOutputHashes = new List <AbsolutePathFileMaterializationInfo>
{
new AbsolutePathFileMaterializationInfo
{
AbsolutePath = abPath2.ToString(pathTable),
Info = new BondFileMaterializationInfo {
FileName = abPath2.GetName(pathTable).ToString(context.StringTable)
}
}
},
DynamicOutputs = new List <List <RelativePathFileMaterializationInfo> >
{
new List <RelativePathFileMaterializationInfo>
{
new RelativePathFileMaterializationInfo
{
RelativePath = @"dir\P2Dynamic.txt",
Info = new BondFileMaterializationInfo {
FileName = "p2dynamic.txt"
}
},
new RelativePathFileMaterializationInfo
{
RelativePath = @"dir\P2dynout2.txt",
Info = new BondFileMaterializationInfo {
FileName = null
}
}
}
}
};
var storedPathSet2 = await cache.TryStorePathSetAsync(pathSet2, preservePathCasing : false);
var storedMetadata2 = await cache.TryStoreMetadataAsync(metadata2);
var cacheEntry2 = new CacheEntry(storedMetadata2.Result, nameof(HistoricMetadataCacheTests), ArrayView <ContentHash> .Empty);
var strongFingerprint2 = new StrongContentFingerprint(FingerprintUtilities.CreateRandom());
var publishedCacheEntry2 = await cache.TryPublishCacheEntryAsync(process1, weakFingerprint1, storedPathSet2.Result, strongFingerprint2, cacheEntry2);
await cache.CloseAsync();
memoryArtifactCache.Clear();
PipExecutionContext loadedContext;
HistoricMetadataCache loadedCache;
TaskSourceSlim <bool> loadCompletionSource = TaskSourceSlim.Create <bool>();
TaskSourceSlim <bool> loadCalled = TaskSourceSlim.Create <bool>();
BoxRef <bool> calledLoad = new BoxRef <bool>();
CreateHistoricCache(loggingContext, "hmc", out loadedContext, out loadedCache, out memoryArtifactCache, loadTask: async hmc =>
{
loadCalled.SetResult(true);
await loadCompletionSource.Task;
});
var operationContext = OperationContext.CreateUntracked(loggingContext);
var retrievePathSet1Task = loadedCache.TryRetrievePathSetAsync(operationContext, WeakContentFingerprint.Zero, storedPathSet1.Result);
var retrievdMetadata1Task = loadedCache.TryRetrieveMetadataAsync(
process1,
WeakContentFingerprint.Zero,
StrongContentFingerprint.Zero,
storedMetadata1.Result,
storedPathSet1.Result);
var getCacheEntry1Task = loadedCache.TryGetCacheEntryAsync(
process1,
weakFingerprint1,
storedPathSet1.Result,
strongFingerprint1);
Assert.False(retrievePathSet1Task.IsCompleted, "Before load task completes. TryRetrievePathSetAsync operations should block");
Assert.False(retrievdMetadata1Task.IsCompleted, "Before load task completes. TryRetrieveMetadataAsync operations should block");
Assert.False(getCacheEntry1Task.IsCompleted, "Before load task completes. TryGetCacheEntryAsync operations should block");
Assert.True(loadCalled.Task.Wait(TimeSpan.FromSeconds(10)) && loadCalled.Task.Result, "Load should have been called in as a result of querying");
loadCompletionSource.SetResult(true);
var maybeLoadedPathSet1 = await retrievePathSet1Task;
var maybeLoadedMetadata1 = await retrievdMetadata1Task;
var maybeLoadedCacheEntry1 = await getCacheEntry1Task;
Assert.Equal(storedMetadata1.Result, maybeLoadedCacheEntry1.Result.Value.MetadataHash);
var maybeLoadedPathSet2 = await loadedCache.TryRetrievePathSetAsync(operationContext, WeakContentFingerprint.Zero, storedPathSet2.Result);
var maybeLoadedMetadata2 = await loadedCache.TryRetrieveMetadataAsync(
process2,
WeakContentFingerprint.Zero,
StrongContentFingerprint.Zero,
storedMetadata2.Result,
storedPathSet2.Result);
AssertPathSetEquals(pathTable, pathSet1, loadedContext.PathTable, maybeLoadedPathSet1.Result);
AssertPathSetEquals(pathTable, pathSet2, loadedContext.PathTable, maybeLoadedPathSet2.Result);
AssertMetadataEquals(metadata1, maybeLoadedMetadata1.Result);
AssertMetadataEquals(metadata2, maybeLoadedMetadata2.Result);
await loadedCache.CloseAsync();
}
}