public void TestProcessExecutionResultSerialization()
{
var reportedAccess = CreateRandomReportedFileAccess();
Fingerprint fingerprint = FingerprintUtilities.CreateRandom();
var processExecutionResult = ExecutionResult.CreateSealed(
result: PipResultStatus.Succeeded,
numberOfWarnings: 12,
outputContent: ReadOnlyArray <(FileArtifact, FileMaterializationInfo, PipOutputOrigin)> .FromWithoutCopy(CreateRandomOutputContent(), CreateRandomOutputContent()),
directoryOutputs: ReadOnlyArray <(DirectoryArtifact, ReadOnlyArray <FileArtifact>)> .FromWithoutCopy(CreateRandomOutputDirectory(), CreateRandomOutputDirectory()),
performanceInformation: new ProcessPipExecutionPerformance(
PipExecutionLevel.Executed,
DateTime.UtcNow,
DateTime.UtcNow + TimeSpan.FromMinutes(2),
FingerprintUtilities.ZeroFingerprint,
TimeSpan.FromMinutes(2),
new FileMonitoringViolationCounters(2, 3, 4),
default(IOCounters),
TimeSpan.FromMinutes(3),
TimeSpan.FromMinutes(3),
ProcessMemoryCounters.CreateFromBytes(12324, 12325, 12326, 12326),
33,
7,
0),
fingerprint: new WeakContentFingerprint(fingerprint),
fileAccessViolationsNotAllowlisted: new[]
{
reportedAccess,
CreateRandomReportedFileAccess(),
// Create reported file access that uses the same process to test deduplication during deserialization
CreateRandomReportedFileAccess(reportedAccess.Process),
},
allowlistedFileAccessViolations: new ReportedFileAccess[0],
mustBeConsideredPerpetuallyDirty: true,
dynamicallyObservedFiles: ReadOnlyArray <AbsolutePath> .FromWithoutCopy(
CreateSourceFile().Path,
CreateSourceFile().Path
),
dynamicallyProbedFiles: ReadOnlyArray <AbsolutePath> .FromWithoutCopy(
CreateSourceFile().Path,
CreateSourceFile().Path,
CreateSourceFile().Path
),
dynamicallyObservedEnumerations: ReadOnlyArray <AbsolutePath> .FromWithoutCopy(
CreateSourceFile().Path
),
allowedUndeclaredSourceReads: new ReadOnlyHashSet <AbsolutePath> {
CreateSourceFile().Path,
CreateSourceFile().Path
},
absentPathProbesUnderOutputDirectories: new ReadOnlyHashSet <AbsolutePath> {
CreateSourceFile().Path,
CreateSourceFile().Path
},
twoPhaseCachingInfo: new TwoPhaseCachingInfo(
new WeakContentFingerprint(Fingerprint.Random(FingerprintUtilities.FingerprintLength)),
ContentHashingUtilities.CreateRandom(),
new StrongContentFingerprint(Fingerprint.Random(FingerprintUtilities.FingerprintLength)),
new CacheEntry(ContentHashingUtilities.CreateRandom(), null, CreateRandomContentHashArray())),
pipCacheDescriptorV2Metadata: null,
converged: true,
pathSet: null,
cacheLookupStepDurations: null,
pipProperties: new Dictionary <string, int> {
{ "Foo", 1 }, { "Bar", 9 }
},
hasUserRetries: true);
ExecutionResultSerializer serializer = new ExecutionResultSerializer(0, Context);
ExecutionResult deserializedProcessExecutionResult;
using (var stream = new MemoryStream())
using (var writer = new BuildXLWriter(false, stream, true, false))
using (var reader = new BuildXLReader(false, stream, true))
{
serializer.Serialize(writer, processExecutionResult, preservePathCasing: false);
stream.Position = 0;
deserializedProcessExecutionResult = serializer.Deserialize(reader,
processExecutionResult.PerformanceInformation.WorkerId);
}
// Ensure successful pip result is changed to not materialized.
XAssert.AreEqual(PipResultStatus.NotMaterialized, deserializedProcessExecutionResult.Result);
AssertEqual(processExecutionResult, deserializedProcessExecutionResult,
r => r.NumberOfWarnings,
r => r.Converged,
r => r.OutputContent.Length,
r => r.DirectoryOutputs.Length,
r => r.PerformanceInformation.ExecutionLevel,
r => r.PerformanceInformation.ExecutionStop,
r => r.PerformanceInformation.ExecutionStart,
r => r.PerformanceInformation.ProcessExecutionTime,
r => r.PerformanceInformation.FileMonitoringViolations.NumFileAccessViolationsNotAllowlisted,
r => r.PerformanceInformation.FileMonitoringViolations.NumFileAccessesAllowlistedAndCacheable,
r => r.PerformanceInformation.FileMonitoringViolations.NumFileAccessesAllowlistedButNotCacheable,
r => r.PerformanceInformation.UserTime,
r => r.PerformanceInformation.KernelTime,
r => r.PerformanceInformation.MemoryCounters.PeakWorkingSetMb,
r => r.PerformanceInformation.MemoryCounters.AverageWorkingSetMb,
r => r.PerformanceInformation.MemoryCounters.PeakCommitSizeMb,
r => r.PerformanceInformation.MemoryCounters.AverageCommitSizeMb,
r => r.PerformanceInformation.NumberOfProcesses,
r => r.FileAccessViolationsNotAllowlisted.Count,
r => r.MustBeConsideredPerpetuallyDirty,
r => r.DynamicallyObservedFiles.Length,
r => r.DynamicallyProbedFiles.Length,
r => r.DynamicallyObservedEnumerations.Length,
r => r.AllowedUndeclaredReads.Count,
r => r.TwoPhaseCachingInfo.WeakFingerprint,
r => r.TwoPhaseCachingInfo.StrongFingerprint,
r => r.TwoPhaseCachingInfo.PathSetHash,
r => r.TwoPhaseCachingInfo.CacheEntry.MetadataHash,
r => r.TwoPhaseCachingInfo.CacheEntry.OriginatingCache,
r => r.TwoPhaseCachingInfo.CacheEntry.ReferencedContent.Length,
r => r.PipProperties.Count,
r => r.HasUserRetries,
r => r.RetryInfo
);
for (int i = 0; i < processExecutionResult.OutputContent.Length; i++)
{
int j = i;
AssertEqual(processExecutionResult, deserializedProcessExecutionResult,
r => r.OutputContent[j].Item1,
r => r.OutputContent[j].Item2
);
// Ensure that output origin from deserialzed output content is changed to not materialized.
XAssert.AreEqual(PipOutputOrigin.NotMaterialized, deserializedProcessExecutionResult.OutputContent[i].Item3);
}
for (int i = 0; i < processExecutionResult.DirectoryOutputs.Length; i++)
{
var expected = processExecutionResult.DirectoryOutputs[i];
var result = deserializedProcessExecutionResult.DirectoryOutputs[i];
XAssert.AreEqual(expected.Item1, result.Item1);
XAssert.AreEqual(expected.Item2.Length, result.Item2.Length);
for (int j = 0; j < expected.Item2.Length; j++)
{
XAssert.AreEqual(expected.Item2[j], result.Item2[j]);
}
}
for (int i = 0; i < processExecutionResult.FileAccessViolationsNotAllowlisted.Count; i++)
{
// Compare individual fields for ReportedFileAccess since it uses reference
// equality for reported process which would not work for serialization/deserialization
AssertEqual(processExecutionResult.FileAccessViolationsNotAllowlisted[i], deserializedProcessExecutionResult.FileAccessViolationsNotAllowlisted[i]);
}
// Ensure that reported process instances are deduplicated.
XAssert.AreSame(deserializedProcessExecutionResult.FileAccessViolationsNotAllowlisted[0].Process,
deserializedProcessExecutionResult.FileAccessViolationsNotAllowlisted[2].Process);
for (int i = 0; i < processExecutionResult.DynamicallyObservedFiles.Length; i++)
{
AssertEqual(processExecutionResult.DynamicallyObservedFiles[i], deserializedProcessExecutionResult.DynamicallyObservedFiles[i]);
}
for (int i = 0; i < processExecutionResult.DynamicallyProbedFiles.Length; i++)
{
AssertEqual(processExecutionResult.DynamicallyProbedFiles[i], deserializedProcessExecutionResult.DynamicallyProbedFiles[i]);
}
for (int i = 0; i < processExecutionResult.DynamicallyObservedEnumerations.Length; i++)
{
AssertEqual(processExecutionResult.DynamicallyObservedEnumerations[i], deserializedProcessExecutionResult.DynamicallyObservedEnumerations[i]);
}
XAssert.AreSetsEqual(processExecutionResult.AllowedUndeclaredReads, deserializedProcessExecutionResult.AllowedUndeclaredReads, expectedResult: true);
var referencedContentLength = processExecutionResult.TwoPhaseCachingInfo.CacheEntry.ReferencedContent.Length;
for (int i = 0; i < referencedContentLength; i++)
{
XAssert.AreEqual(
processExecutionResult.TwoPhaseCachingInfo.CacheEntry.ReferencedContent[i],
deserializedProcessExecutionResult.TwoPhaseCachingInfo.CacheEntry.ReferencedContent[i]);
}
XAssert.AreEqual(9, deserializedProcessExecutionResult.PipProperties["Bar"]);
}
public async Task TestHistoricMetadataPathStringRoundtrip()
{
LoggingContext loggingContext = CreateLoggingContextForTest();
PipExecutionContext context;
HistoricMetadataCache cache = null;
var hmcFolderName = "hmc";
for (int i = 0; i < 3; i++)
{
CreateHistoricCache(loggingContext, hmcFolderName, out context, out cache, out var memoryArtifactCache);
var process1 = SchedulerTest.CreateDummyProcess(context, new PipId(1));
var process2 = SchedulerTest.CreateDummyProcess(context, new PipId(2));
var pathTable = context.PathTable;
// Add some random paths to ensure path table indices are different after loading
AbsolutePath.Create(pathTable, X("/H/aslj/sfas/832.stxt"));
AbsolutePath.Create(pathTable, X("/R/f/s/Historic"));
AbsolutePath.Create(pathTable, X("/M/hgf/sf4as/83afsd"));
AbsolutePath.Create(pathTable, X("/Z/bd/sfas/Cache"));
var abPath1 = AbsolutePath.Create(pathTable, X("/H/aslj/sfas/p1OUT.bin"));
var abPath2 = AbsolutePath.Create(pathTable, X("/H/aslj/sfas/P2.txt"));
var pathSet1 = ObservedPathSetTestUtilities.CreatePathSet(
pathTable,
X("/X/a/b/c"),
X("/X/d/e"),
X("/X/a/b/c/d"));
PipCacheDescriptorV2Metadata metadata1 =
new PipCacheDescriptorV2Metadata
{
StaticOutputHashes = new List <AbsolutePathFileMaterializationInfo>
{
new AbsolutePathFileMaterializationInfo
{
AbsolutePath = abPath1.GetName(pathTable).ToString(context.StringTable),
Info = new BondFileMaterializationInfo {
FileName = "p1OUT.bin"
}
}
}
};
var storedPathSet1 = await cache.TryStorePathSetAsync(pathSet1, preservePathCasing : false);
var storedMetadata1 = await cache.TryStoreMetadataAsync(metadata1);
var weakFingerprint1 = new WeakContentFingerprint(FingerprintUtilities.CreateRandom());
var strongFingerprint1 = new StrongContentFingerprint(FingerprintUtilities.CreateRandom());
var cacheEntry = new CacheEntry(storedMetadata1.Result, nameof(HistoricMetadataCacheTests), ArrayView <ContentHash> .Empty);
var publishedCacheEntry = await cache.TryPublishCacheEntryAsync(process1, weakFingerprint1, storedPathSet1.Result, strongFingerprint1, cacheEntry);
var pathSet2 = ObservedPathSetTestUtilities.CreatePathSet(
pathTable,
X("/F/a/y/c"),
X("/B/d/e"),
X("/G/a/z/c/d"),
X("/B/a/b/c"));
PipCacheDescriptorV2Metadata metadata2 =
new PipCacheDescriptorV2Metadata
{
StaticOutputHashes = new List <AbsolutePathFileMaterializationInfo>
{
new AbsolutePathFileMaterializationInfo
{
AbsolutePath = abPath2.ToString(pathTable),
Info = new BondFileMaterializationInfo {
FileName = abPath2.GetName(pathTable).ToString(context.StringTable)
}
}
},
DynamicOutputs = new List <List <RelativePathFileMaterializationInfo> >
{
new List <RelativePathFileMaterializationInfo>
{
new RelativePathFileMaterializationInfo
{
RelativePath = @"dir\P2Dynamic.txt",
Info = new BondFileMaterializationInfo {
FileName = "p2dynamic.txt"
}
},
new RelativePathFileMaterializationInfo
{
RelativePath = @"dir\P2dynout2.txt",
Info = new BondFileMaterializationInfo {
FileName = null
}
}
}
}
};
var storedPathSet2 = await cache.TryStorePathSetAsync(pathSet2, preservePathCasing : false);
var storedMetadata2 = await cache.TryStoreMetadataAsync(metadata2);
var cacheEntry2 = new CacheEntry(storedMetadata2.Result, nameof(HistoricMetadataCacheTests), ArrayView <ContentHash> .Empty);
var strongFingerprint2 = new StrongContentFingerprint(FingerprintUtilities.CreateRandom());
var publishedCacheEntry2 = await cache.TryPublishCacheEntryAsync(process1, weakFingerprint1, storedPathSet2.Result, strongFingerprint2, cacheEntry2);
await cache.CloseAsync();
memoryArtifactCache.Clear();
PipExecutionContext loadedContext;
HistoricMetadataCache loadedCache;
TaskSourceSlim <bool> loadCompletionSource = TaskSourceSlim.Create <bool>();
TaskSourceSlim <bool> loadCalled = TaskSourceSlim.Create <bool>();
BoxRef <bool> calledLoad = new BoxRef <bool>();
CreateHistoricCache(loggingContext, "hmc", out loadedContext, out loadedCache, out memoryArtifactCache, loadTask: async hmc =>
{
loadCalled.SetResult(true);
await loadCompletionSource.Task;
});
var operationContext = OperationContext.CreateUntracked(loggingContext);
var retrievePathSet1Task = loadedCache.TryRetrievePathSetAsync(operationContext, WeakContentFingerprint.Zero, storedPathSet1.Result);
var retrievdMetadata1Task = loadedCache.TryRetrieveMetadataAsync(
process1,
WeakContentFingerprint.Zero,
StrongContentFingerprint.Zero,
storedMetadata1.Result,
storedPathSet1.Result);
var getCacheEntry1Task = loadedCache.TryGetCacheEntryAsync(
process1,
weakFingerprint1,
storedPathSet1.Result,
strongFingerprint1);
Assert.False(retrievePathSet1Task.IsCompleted, "Before load task completes. TryRetrievePathSetAsync operations should block");
Assert.False(retrievdMetadata1Task.IsCompleted, "Before load task completes. TryRetrieveMetadataAsync operations should block");
Assert.False(getCacheEntry1Task.IsCompleted, "Before load task completes. TryGetCacheEntryAsync operations should block");
Assert.True(loadCalled.Task.Wait(TimeSpan.FromSeconds(10)) && loadCalled.Task.Result, "Load should have been called in as a result of querying");
loadCompletionSource.SetResult(true);
var maybeLoadedPathSet1 = await retrievePathSet1Task;
var maybeLoadedMetadata1 = await retrievdMetadata1Task;
var maybeLoadedCacheEntry1 = await getCacheEntry1Task;
Assert.Equal(storedMetadata1.Result, maybeLoadedCacheEntry1.Result.Value.MetadataHash);
var maybeLoadedPathSet2 = await loadedCache.TryRetrievePathSetAsync(operationContext, WeakContentFingerprint.Zero, storedPathSet2.Result);
var maybeLoadedMetadata2 = await loadedCache.TryRetrieveMetadataAsync(
process2,
WeakContentFingerprint.Zero,
StrongContentFingerprint.Zero,
storedMetadata2.Result,
storedPathSet2.Result);
AssertPathSetEquals(pathTable, pathSet1, loadedContext.PathTable, maybeLoadedPathSet1.Result);
AssertPathSetEquals(pathTable, pathSet2, loadedContext.PathTable, maybeLoadedPathSet2.Result);
AssertMetadataEquals(metadata1, maybeLoadedMetadata1.Result);
AssertMetadataEquals(metadata2, maybeLoadedMetadata2.Result);
await loadedCache.CloseAsync();
}
}
/// <summary>
/// Create a random StrongFingerprint
/// </summary>
public static StrongFingerprint CreateRandomStrongFingerprint(string cacheId)
{
return(new StrongFingerprint(CreateRandomWeakFingerprintHash(), CreateRandomCasHash(), new Hash(FingerprintUtilities.CreateRandom()), cacheId));
}