public void TestProcessExecutionResultSerialization() { var reportedAccess = CreateRandomReportedFileAccess(); Fingerprint fingerprint = FingerprintUtilities.CreateRandom(); var processExecutionResult = ExecutionResult.CreateSealed( result: PipResultStatus.Succeeded, numberOfWarnings: 12, outputContent: ReadOnlyArray <(FileArtifact, FileMaterializationInfo, PipOutputOrigin)> .FromWithoutCopy(CreateRandomOutputContent(), CreateRandomOutputContent()), directoryOutputs: ReadOnlyArray <(DirectoryArtifact, ReadOnlyArray <FileArtifact>)> .FromWithoutCopy(CreateRandomOutputDirectory(), CreateRandomOutputDirectory()), performanceInformation: new ProcessPipExecutionPerformance( PipExecutionLevel.Executed, DateTime.UtcNow, DateTime.UtcNow + TimeSpan.FromMinutes(2), FingerprintUtilities.ZeroFingerprint, TimeSpan.FromMinutes(2), new FileMonitoringViolationCounters(2, 3, 4), default(IOCounters), TimeSpan.FromMinutes(3), TimeSpan.FromMinutes(3), ProcessMemoryCounters.CreateFromBytes(12324, 12325, 12326, 12326), 33, 7, 0), fingerprint: new WeakContentFingerprint(fingerprint), fileAccessViolationsNotAllowlisted: new[] { reportedAccess, CreateRandomReportedFileAccess(), // Create reported file access that uses the same process to test deduplication during deserialization CreateRandomReportedFileAccess(reportedAccess.Process), }, allowlistedFileAccessViolations: new ReportedFileAccess[0], mustBeConsideredPerpetuallyDirty: true, dynamicallyObservedFiles: ReadOnlyArray <AbsolutePath> .FromWithoutCopy( CreateSourceFile().Path, CreateSourceFile().Path ), dynamicallyProbedFiles: ReadOnlyArray <AbsolutePath> .FromWithoutCopy( CreateSourceFile().Path, CreateSourceFile().Path, CreateSourceFile().Path ), dynamicallyObservedEnumerations: ReadOnlyArray <AbsolutePath> .FromWithoutCopy( CreateSourceFile().Path ), allowedUndeclaredSourceReads: new ReadOnlyHashSet <AbsolutePath> { CreateSourceFile().Path, CreateSourceFile().Path }, absentPathProbesUnderOutputDirectories: new ReadOnlyHashSet <AbsolutePath> { CreateSourceFile().Path, CreateSourceFile().Path }, twoPhaseCachingInfo: new TwoPhaseCachingInfo( new WeakContentFingerprint(Fingerprint.Random(FingerprintUtilities.FingerprintLength)), ContentHashingUtilities.CreateRandom(), new StrongContentFingerprint(Fingerprint.Random(FingerprintUtilities.FingerprintLength)), new CacheEntry(ContentHashingUtilities.CreateRandom(), null, CreateRandomContentHashArray())), pipCacheDescriptorV2Metadata: null, converged: true, pathSet: null, cacheLookupStepDurations: null, pipProperties: new Dictionary <string, int> { { "Foo", 1 }, { "Bar", 9 } }, hasUserRetries: true); ExecutionResultSerializer serializer = new ExecutionResultSerializer(0, Context); ExecutionResult deserializedProcessExecutionResult; using (var stream = new MemoryStream()) using (var writer = new BuildXLWriter(false, stream, true, false)) using (var reader = new BuildXLReader(false, stream, true)) { serializer.Serialize(writer, processExecutionResult, preservePathCasing: false); stream.Position = 0; deserializedProcessExecutionResult = serializer.Deserialize(reader, processExecutionResult.PerformanceInformation.WorkerId); } // Ensure successful pip result is changed to not materialized. XAssert.AreEqual(PipResultStatus.NotMaterialized, deserializedProcessExecutionResult.Result); AssertEqual(processExecutionResult, deserializedProcessExecutionResult, r => r.NumberOfWarnings, r => r.Converged, r => r.OutputContent.Length, r => r.DirectoryOutputs.Length, r => r.PerformanceInformation.ExecutionLevel, r => r.PerformanceInformation.ExecutionStop, r => r.PerformanceInformation.ExecutionStart, r => r.PerformanceInformation.ProcessExecutionTime, r => r.PerformanceInformation.FileMonitoringViolations.NumFileAccessViolationsNotAllowlisted, r => r.PerformanceInformation.FileMonitoringViolations.NumFileAccessesAllowlistedAndCacheable, r => r.PerformanceInformation.FileMonitoringViolations.NumFileAccessesAllowlistedButNotCacheable, r => r.PerformanceInformation.UserTime, r => r.PerformanceInformation.KernelTime, r => r.PerformanceInformation.MemoryCounters.PeakWorkingSetMb, r => r.PerformanceInformation.MemoryCounters.AverageWorkingSetMb, r => r.PerformanceInformation.MemoryCounters.PeakCommitSizeMb, r => r.PerformanceInformation.MemoryCounters.AverageCommitSizeMb, r => r.PerformanceInformation.NumberOfProcesses, r => r.FileAccessViolationsNotAllowlisted.Count, r => r.MustBeConsideredPerpetuallyDirty, r => r.DynamicallyObservedFiles.Length, r => r.DynamicallyProbedFiles.Length, r => r.DynamicallyObservedEnumerations.Length, r => r.AllowedUndeclaredReads.Count, r => r.TwoPhaseCachingInfo.WeakFingerprint, r => r.TwoPhaseCachingInfo.StrongFingerprint, r => r.TwoPhaseCachingInfo.PathSetHash, r => r.TwoPhaseCachingInfo.CacheEntry.MetadataHash, r => r.TwoPhaseCachingInfo.CacheEntry.OriginatingCache, r => r.TwoPhaseCachingInfo.CacheEntry.ReferencedContent.Length, r => r.PipProperties.Count, r => r.HasUserRetries, r => r.RetryInfo ); for (int i = 0; i < processExecutionResult.OutputContent.Length; i++) { int j = i; AssertEqual(processExecutionResult, deserializedProcessExecutionResult, r => r.OutputContent[j].Item1, r => r.OutputContent[j].Item2 ); // Ensure that output origin from deserialzed output content is changed to not materialized. XAssert.AreEqual(PipOutputOrigin.NotMaterialized, deserializedProcessExecutionResult.OutputContent[i].Item3); } for (int i = 0; i < processExecutionResult.DirectoryOutputs.Length; i++) { var expected = processExecutionResult.DirectoryOutputs[i]; var result = deserializedProcessExecutionResult.DirectoryOutputs[i]; XAssert.AreEqual(expected.Item1, result.Item1); XAssert.AreEqual(expected.Item2.Length, result.Item2.Length); for (int j = 0; j < expected.Item2.Length; j++) { XAssert.AreEqual(expected.Item2[j], result.Item2[j]); } } for (int i = 0; i < processExecutionResult.FileAccessViolationsNotAllowlisted.Count; i++) { // Compare individual fields for ReportedFileAccess since it uses reference // equality for reported process which would not work for serialization/deserialization AssertEqual(processExecutionResult.FileAccessViolationsNotAllowlisted[i], deserializedProcessExecutionResult.FileAccessViolationsNotAllowlisted[i]); } // Ensure that reported process instances are deduplicated. XAssert.AreSame(deserializedProcessExecutionResult.FileAccessViolationsNotAllowlisted[0].Process, deserializedProcessExecutionResult.FileAccessViolationsNotAllowlisted[2].Process); for (int i = 0; i < processExecutionResult.DynamicallyObservedFiles.Length; i++) { AssertEqual(processExecutionResult.DynamicallyObservedFiles[i], deserializedProcessExecutionResult.DynamicallyObservedFiles[i]); } for (int i = 0; i < processExecutionResult.DynamicallyProbedFiles.Length; i++) { AssertEqual(processExecutionResult.DynamicallyProbedFiles[i], deserializedProcessExecutionResult.DynamicallyProbedFiles[i]); } for (int i = 0; i < processExecutionResult.DynamicallyObservedEnumerations.Length; i++) { AssertEqual(processExecutionResult.DynamicallyObservedEnumerations[i], deserializedProcessExecutionResult.DynamicallyObservedEnumerations[i]); } XAssert.AreSetsEqual(processExecutionResult.AllowedUndeclaredReads, deserializedProcessExecutionResult.AllowedUndeclaredReads, expectedResult: true); var referencedContentLength = processExecutionResult.TwoPhaseCachingInfo.CacheEntry.ReferencedContent.Length; for (int i = 0; i < referencedContentLength; i++) { XAssert.AreEqual( processExecutionResult.TwoPhaseCachingInfo.CacheEntry.ReferencedContent[i], deserializedProcessExecutionResult.TwoPhaseCachingInfo.CacheEntry.ReferencedContent[i]); } XAssert.AreEqual(9, deserializedProcessExecutionResult.PipProperties["Bar"]); }
public async Task TestHistoricMetadataPathStringRoundtrip() { LoggingContext loggingContext = CreateLoggingContextForTest(); PipExecutionContext context; HistoricMetadataCache cache = null; var hmcFolderName = "hmc"; for (int i = 0; i < 3; i++) { CreateHistoricCache(loggingContext, hmcFolderName, out context, out cache, out var memoryArtifactCache); var process1 = SchedulerTest.CreateDummyProcess(context, new PipId(1)); var process2 = SchedulerTest.CreateDummyProcess(context, new PipId(2)); var pathTable = context.PathTable; // Add some random paths to ensure path table indices are different after loading AbsolutePath.Create(pathTable, X("/H/aslj/sfas/832.stxt")); AbsolutePath.Create(pathTable, X("/R/f/s/Historic")); AbsolutePath.Create(pathTable, X("/M/hgf/sf4as/83afsd")); AbsolutePath.Create(pathTable, X("/Z/bd/sfas/Cache")); var abPath1 = AbsolutePath.Create(pathTable, X("/H/aslj/sfas/p1OUT.bin")); var abPath2 = AbsolutePath.Create(pathTable, X("/H/aslj/sfas/P2.txt")); var pathSet1 = ObservedPathSetTestUtilities.CreatePathSet( pathTable, X("/X/a/b/c"), X("/X/d/e"), X("/X/a/b/c/d")); PipCacheDescriptorV2Metadata metadata1 = new PipCacheDescriptorV2Metadata { StaticOutputHashes = new List <AbsolutePathFileMaterializationInfo> { new AbsolutePathFileMaterializationInfo { AbsolutePath = abPath1.GetName(pathTable).ToString(context.StringTable), Info = new BondFileMaterializationInfo { FileName = "p1OUT.bin" } } } }; var storedPathSet1 = await cache.TryStorePathSetAsync(pathSet1, preservePathCasing : false); var storedMetadata1 = await cache.TryStoreMetadataAsync(metadata1); var weakFingerprint1 = new WeakContentFingerprint(FingerprintUtilities.CreateRandom()); var strongFingerprint1 = new StrongContentFingerprint(FingerprintUtilities.CreateRandom()); var cacheEntry = new CacheEntry(storedMetadata1.Result, nameof(HistoricMetadataCacheTests), ArrayView <ContentHash> .Empty); var publishedCacheEntry = await cache.TryPublishCacheEntryAsync(process1, weakFingerprint1, storedPathSet1.Result, strongFingerprint1, cacheEntry); var pathSet2 = ObservedPathSetTestUtilities.CreatePathSet( pathTable, X("/F/a/y/c"), X("/B/d/e"), X("/G/a/z/c/d"), X("/B/a/b/c")); PipCacheDescriptorV2Metadata metadata2 = new PipCacheDescriptorV2Metadata { StaticOutputHashes = new List <AbsolutePathFileMaterializationInfo> { new AbsolutePathFileMaterializationInfo { AbsolutePath = abPath2.ToString(pathTable), Info = new BondFileMaterializationInfo { FileName = abPath2.GetName(pathTable).ToString(context.StringTable) } } }, DynamicOutputs = new List <List <RelativePathFileMaterializationInfo> > { new List <RelativePathFileMaterializationInfo> { new RelativePathFileMaterializationInfo { RelativePath = @"dir\P2Dynamic.txt", Info = new BondFileMaterializationInfo { FileName = "p2dynamic.txt" } }, new RelativePathFileMaterializationInfo { RelativePath = @"dir\P2dynout2.txt", Info = new BondFileMaterializationInfo { FileName = null } } } } }; var storedPathSet2 = await cache.TryStorePathSetAsync(pathSet2, preservePathCasing : false); var storedMetadata2 = await cache.TryStoreMetadataAsync(metadata2); var cacheEntry2 = new CacheEntry(storedMetadata2.Result, nameof(HistoricMetadataCacheTests), ArrayView <ContentHash> .Empty); var strongFingerprint2 = new StrongContentFingerprint(FingerprintUtilities.CreateRandom()); var publishedCacheEntry2 = await cache.TryPublishCacheEntryAsync(process1, weakFingerprint1, storedPathSet2.Result, strongFingerprint2, cacheEntry2); await cache.CloseAsync(); memoryArtifactCache.Clear(); PipExecutionContext loadedContext; HistoricMetadataCache loadedCache; TaskSourceSlim <bool> loadCompletionSource = TaskSourceSlim.Create <bool>(); TaskSourceSlim <bool> loadCalled = TaskSourceSlim.Create <bool>(); BoxRef <bool> calledLoad = new BoxRef <bool>(); CreateHistoricCache(loggingContext, "hmc", out loadedContext, out loadedCache, out memoryArtifactCache, loadTask: async hmc => { loadCalled.SetResult(true); await loadCompletionSource.Task; }); var operationContext = OperationContext.CreateUntracked(loggingContext); var retrievePathSet1Task = loadedCache.TryRetrievePathSetAsync(operationContext, WeakContentFingerprint.Zero, storedPathSet1.Result); var retrievdMetadata1Task = loadedCache.TryRetrieveMetadataAsync( process1, WeakContentFingerprint.Zero, StrongContentFingerprint.Zero, storedMetadata1.Result, storedPathSet1.Result); var getCacheEntry1Task = loadedCache.TryGetCacheEntryAsync( process1, weakFingerprint1, storedPathSet1.Result, strongFingerprint1); Assert.False(retrievePathSet1Task.IsCompleted, "Before load task completes. TryRetrievePathSetAsync operations should block"); Assert.False(retrievdMetadata1Task.IsCompleted, "Before load task completes. TryRetrieveMetadataAsync operations should block"); Assert.False(getCacheEntry1Task.IsCompleted, "Before load task completes. TryGetCacheEntryAsync operations should block"); Assert.True(loadCalled.Task.Wait(TimeSpan.FromSeconds(10)) && loadCalled.Task.Result, "Load should have been called in as a result of querying"); loadCompletionSource.SetResult(true); var maybeLoadedPathSet1 = await retrievePathSet1Task; var maybeLoadedMetadata1 = await retrievdMetadata1Task; var maybeLoadedCacheEntry1 = await getCacheEntry1Task; Assert.Equal(storedMetadata1.Result, maybeLoadedCacheEntry1.Result.Value.MetadataHash); var maybeLoadedPathSet2 = await loadedCache.TryRetrievePathSetAsync(operationContext, WeakContentFingerprint.Zero, storedPathSet2.Result); var maybeLoadedMetadata2 = await loadedCache.TryRetrieveMetadataAsync( process2, WeakContentFingerprint.Zero, StrongContentFingerprint.Zero, storedMetadata2.Result, storedPathSet2.Result); AssertPathSetEquals(pathTable, pathSet1, loadedContext.PathTable, maybeLoadedPathSet1.Result); AssertPathSetEquals(pathTable, pathSet2, loadedContext.PathTable, maybeLoadedPathSet2.Result); AssertMetadataEquals(metadata1, maybeLoadedMetadata1.Result); AssertMetadataEquals(metadata2, maybeLoadedMetadata2.Result); await loadedCache.CloseAsync(); } }
/// <summary> /// Create a random StrongFingerprint /// </summary> public static StrongFingerprint CreateRandomStrongFingerprint(string cacheId) { return(new StrongFingerprint(CreateRandomWeakFingerprintHash(), CreateRandomCasHash(), new Hash(FingerprintUtilities.CreateRandom()), cacheId)); }