public void ObservedFileNamesNormalizedTheSameWayInPathsetAndJsonFingerprinter() { // This test is guarding codesync between JsonFingerprinter.cs and ObservedPathSet.cs var fileNames = new string[] { "a", "b", "C" }; var pathTable = new PathTable(); var pathSet = ObservedPathSetTestUtilities.CreatePathSet( pathTable, observedAccessedFileNames: fileNames, X("/X/a/b/c")); var roundtrip = SerializeRoundTripAndAssertEquivalent(pathTable, pathSet); var sb = new StringBuilder(); using (var writer = new global::BuildXL.Engine.Cache.JsonFingerprinter(sb, pathTable: pathTable)) { writer.AddCollection <StringId, StringId[]>( "fileNames", fileNames.Select(fn => StringId.Create(pathTable.StringTable, fn)).ToArray(), (w, e) => w.AddFileName(e)); } var fpOutput = sb.ToString(); XAssert.IsTrue(roundtrip.ObservedAccessedFileNames.All(fileName => fpOutput.Contains($"\"{fileName.ToString(pathTable.StringTable)}\""))); }
public void RoundTripSerializationEmpty() { var pathTable = new PathTable(); var pathSet = ObservedPathSetTestUtilities.CreatePathSet(pathTable, new string[0]); var roundtrip = SerializeRoundTripAndAssertEquivalent(pathTable, pathSet); XAssert.AreEqual(0, roundtrip.Paths.Length); }
[Trait("Category", "WindowsOSOnly")] // Paths are case-sensitive on Linux-based systems. public void RoundTripSerializationNormalizesCasingAndRemovesDuplicatesInObservedAccessedFileNames() { var pathTable = new PathTable(); var pathSet = ObservedPathSetTestUtilities.CreatePathSet( pathTable, observedAccessedFileNames: new string[] { "d", "D", "f" }, X("/X/a/b/c")); var roundtrip = SerializeRoundTripAndAssertEquivalent(pathTable, pathSet); ObservedPathSetTestUtilities.AssertPathSetContainsDuplicates(pathSet); ObservedPathSetTestUtilities.AssertPathSetDoesNotContainDuplicates(roundtrip); }
public void RoundTripSerializationRemovesDuplicatesWithUnrelatedPaths() { var pathTable = new PathTable(); var pathSet = ObservedPathSetTestUtilities.CreatePathSet( pathTable, X("/X/a/b/c"), X("/X/a/b/c"), X("/Y/d/e/f"), X("/Y/d/e/f")); var roundtrip = SerializeRoundTripAndAssertEquivalent(pathTable, pathSet); ObservedPathSetTestUtilities.AssertPathSetContainsDuplicates(pathSet); ObservedPathSetTestUtilities.AssertPathSetDoesNotContainDuplicates(roundtrip); }
public void NoCompression() { var pathTable = new PathTable(); // No compression since no prefix is shared. var pathSet = ObservedPathSetTestUtilities.CreatePathSet( pathTable, X("/X/a/b"), X("/Y/a/b")); AssertCompressedSizeExpected( pathTable, pathSet, X("/X/a/b"), X("/Y/a/b")); }
public void PrefixCompressionWithinComponents() { var pathTable = new PathTable(); // No compression since no prefix is shared. var pathSet = ObservedPathSetTestUtilities.CreatePathSet( pathTable, X("/X/abcdef"), X("/X/abcxyz/123"), X("/X/abcxyz/123456")); AssertCompressedSizeExpected( pathTable, pathSet, X("/X/abcdef"), $"xyz{Path.DirectorySeparatorChar}123", "456"); }
public void ProjectPathSet() { var pathTable = new PathTable(); AbsolutePath firstPath = AbsolutePath.Create(pathTable, X("/X/bar")); AbsolutePath secondPath = AbsolutePath.Create(pathTable, X("/X/foo")); var p = CreateResult( pathTable, ObservedInput.CreateFileContentRead(firstPath, ContentHashingUtilities.EmptyHash), ObservedInput.CreateFileContentRead(secondPath, ContentHashingUtilities.EmptyHash)); ObservedPathSet projected = p.GetPathSet(unsafeOptions: null); ObservedPathSetTestUtilities.AssertPathSetsEquivalent( projected, ObservedPathSetTestUtilities.CreatePathSet(pathTable, firstPath, secondPath)); }
public async Task TokenizedPathSet() { var pathTable = new PathTable(); var pathExpanderA = new MountPathExpander(pathTable); AddMount(pathExpanderA, pathTable, AbsolutePath.Create(pathTable, X("/x/users/AUser")), "UserProfile", isSystem: true); AddMount(pathExpanderA, pathTable, AbsolutePath.Create(pathTable, X("/x/windows")), "Windows", isSystem: true); AddMount(pathExpanderA, pathTable, AbsolutePath.Create(pathTable, X("/x/test")), "TestRoot", isSystem: false); var pathSetA = ObservedPathSetTestUtilities.CreatePathSet( pathTable, X("/x/abc"), X("/x/users/AUser/def"), X("/x/windows"), X("/x/test/abc")); ObservedPathSet roundtripA = SerializeRoundTripAndAssertEquivalent(pathTable, pathSetA); XAssert.AreEqual(4, roundtripA.Paths.Length); ContentHash pathSetHashA = await pathSetA.ToContentHash(pathTable, pathExpanderA); var pathExpanderB = new MountPathExpander(pathTable); AddMount(pathExpanderB, pathTable, AbsolutePath.Create(pathTable, X("/y/users/BUser")), "UserProfile", isSystem: true); AddMount(pathExpanderB, pathTable, AbsolutePath.Create(pathTable, X("/y/windows")), "Windows", isSystem: true); AddMount(pathExpanderB, pathTable, AbsolutePath.Create(pathTable, X("/y/abc/test")), "TestRoot", isSystem: false); var pathSetB = ObservedPathSetTestUtilities.CreatePathSet( pathTable, X("/x/abc"), X("/y/users/BUser/def"), X("/y/windows"), X("/y/abc/test/abc")); ObservedPathSet roundtripB = SerializeRoundTripAndAssertEquivalent(pathTable, pathSetB); XAssert.AreEqual(4, roundtripB.Paths.Length); ContentHash pathSetHashB = await pathSetB.ToContentHash(pathTable, pathExpanderB); AssertTrue(pathSetHashA == pathSetHashB); }
private static void AssertCompressedSizeExpected(PathTable pathTable, ObservedPathSet pathSet, params string[] uncompressedStrings) { long compressedSize = GetSizeOfSerializedContent(writer => pathSet.Serialize(pathTable, writer)); int numberOfUniquePaths = ObservedPathSetTestUtilities.RemoveDuplicates(pathSet).Count; // This is correct assuming the following: // - Each string can be represented with a one byte length prefix, and a one byte reuse-count. // - The number of strings can be represented in one byte. // - Each character takes one byte when UTF8 encoded. long expectedCompressedSize = GetSizeOfSerializedContent(writer => pathSet.UnsafeOptions.Serialize(writer)) + 1 + // The number of observed accesses file names (0) 1 + // String count (3 * numberOfUniquePaths) + // Length isSearchPath, isDirectoryPath, and reuse uncompressedStrings.Sum(s => s.Length); XAssert.AreEqual(expectedCompressedSize, compressedSize, "Wrong size for compressed path-set"); }
public void RoundTripSerializationOfPathsWithUnicodeChars() { var pathTable = new PathTable(); var mpe = new global::BuildXL.Engine.MountPathExpander(pathTable); mpe.Add( pathTable, new global::BuildXL.Pips.SemanticPathInfo( PathAtom.Create(pathTable.StringTable, "xcode"), AbsolutePath.Create(pathTable, X("/c/Applications")), false, true, true, false, false, false)); var pathSet = ObservedPathSetTestUtilities.CreatePathSet( pathTable, X("/c/Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/Library/CoreSimulator/Profiles/DeviceTypes/iPhone Xʀ.simdevicetype"), X("/c/Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/Library/CoreSimulator/Profiles/DeviceTypes/iPhone Xʀ.simdevicetype/Contents")); SerializeRoundTripAndAssertEquivalent(pathTable, pathSet, mpe); }
public void PrefixCompressionWithReset() { var pathTable = new PathTable(); // No compression since no prefix is shared. var pathSet = ObservedPathSetTestUtilities.CreatePathSet( pathTable, X("/X/abc"), X("/X/abc/def"), X("/Y/abc"), X("/Y/abc/def")); AssertCompressedSizeExpected( pathTable, pathSet, X("/X/abc"), $"{Path.DirectorySeparatorChar}def", X("/Y/abc"), $"{Path.DirectorySeparatorChar}def"); }
public void PrefixCompressionTrivial() { var pathTable = new PathTable(); // No compression since no prefix is shared. var pathSet = ObservedPathSetTestUtilities.CreatePathSet( pathTable, X("/X/a"), X("/X/a/b"), X("/X/a/b/c"), X("/X/a/b/c/d")); AssertCompressedSizeExpected( pathTable, pathSet, X("/X/a"), $"{Path.DirectorySeparatorChar}b", $"{Path.DirectorySeparatorChar}c", $"{Path.DirectorySeparatorChar}d"); }
private static ObservedPathSet SerializeRoundTripAndAssertEquivalent(PathTable pathTable, ObservedPathSet original, PathExpander pathExpander = null) { using (var mem = new MemoryStream()) { using (var writer = new BuildXLWriter(stream: mem, debug: true, leaveOpen: true, logStats: true)) { original.Serialize(pathTable, writer, pathExpander); } mem.Position = 0; ObservedPathSet roundtrip; using (var reader = new BuildXLReader(stream: mem, debug: true, leaveOpen: true)) { var maybeRoundtrip = ObservedPathSet.TryDeserialize(pathTable, reader, pathExpander); XAssert.IsTrue(maybeRoundtrip.Succeeded, "Failed to deserialize a path set unexpectedly"); roundtrip = maybeRoundtrip.Result; } ObservedPathSetTestUtilities.AssertPathSetsEquivalent(original, roundtrip); return(roundtrip); } }
public async Task TestHistoricMetadataPathStringRoundtrip() { LoggingContext loggingContext = CreateLoggingContextForTest(); PipExecutionContext context; HistoricMetadataCache cache = null; var hmcFolderName = "hmc"; for (int i = 0; i < 3; i++) { CreateHistoricCache(loggingContext, hmcFolderName, out context, out cache, out var memoryArtifactCache); var process1 = SchedulerTest.CreateDummyProcess(context, new PipId(1)); var process2 = SchedulerTest.CreateDummyProcess(context, new PipId(2)); var pathTable = context.PathTable; // Add some random paths to ensure path table indices are different after loading AbsolutePath.Create(pathTable, X("/H/aslj/sfas/832.stxt")); AbsolutePath.Create(pathTable, X("/R/f/s/Historic")); AbsolutePath.Create(pathTable, X("/M/hgf/sf4as/83afsd")); AbsolutePath.Create(pathTable, X("/Z/bd/sfas/Cache")); var abPath1 = AbsolutePath.Create(pathTable, X("/H/aslj/sfas/p1OUT.bin")); var abPath2 = AbsolutePath.Create(pathTable, X("/H/aslj/sfas/P2.txt")); var pathSet1 = ObservedPathSetTestUtilities.CreatePathSet( pathTable, X("/X/a/b/c"), X("/X/d/e"), X("/X/a/b/c/d")); PipCacheDescriptorV2Metadata metadata1 = new PipCacheDescriptorV2Metadata { StaticOutputHashes = new List <AbsolutePathFileMaterializationInfo> { new AbsolutePathFileMaterializationInfo { AbsolutePath = abPath1.GetName(pathTable).ToString(context.StringTable), Info = new BondFileMaterializationInfo { FileName = "p1OUT.bin" } } } }; var storedPathSet1 = await cache.TryStorePathSetAsync(pathSet1, preservePathCasing : false); var storedMetadata1 = await cache.TryStoreMetadataAsync(metadata1); var weakFingerprint1 = new WeakContentFingerprint(FingerprintUtilities.CreateRandom()); var strongFingerprint1 = new StrongContentFingerprint(FingerprintUtilities.CreateRandom()); var cacheEntry = new CacheEntry(storedMetadata1.Result, nameof(HistoricMetadataCacheTests), ArrayView <ContentHash> .Empty); var publishedCacheEntry = await cache.TryPublishCacheEntryAsync(process1, weakFingerprint1, storedPathSet1.Result, strongFingerprint1, cacheEntry); var pathSet2 = ObservedPathSetTestUtilities.CreatePathSet( pathTable, X("/F/a/y/c"), X("/B/d/e"), X("/G/a/z/c/d"), X("/B/a/b/c")); PipCacheDescriptorV2Metadata metadata2 = new PipCacheDescriptorV2Metadata { StaticOutputHashes = new List <AbsolutePathFileMaterializationInfo> { new AbsolutePathFileMaterializationInfo { AbsolutePath = abPath2.ToString(pathTable), Info = new BondFileMaterializationInfo { FileName = abPath2.GetName(pathTable).ToString(context.StringTable) } } }, DynamicOutputs = new List <List <RelativePathFileMaterializationInfo> > { new List <RelativePathFileMaterializationInfo> { new RelativePathFileMaterializationInfo { RelativePath = @"dir\P2Dynamic.txt", Info = new BondFileMaterializationInfo { FileName = "p2dynamic.txt" } }, new RelativePathFileMaterializationInfo { RelativePath = @"dir\P2dynout2.txt", Info = new BondFileMaterializationInfo { FileName = null } } } } }; var storedPathSet2 = await cache.TryStorePathSetAsync(pathSet2, preservePathCasing : false); var storedMetadata2 = await cache.TryStoreMetadataAsync(metadata2); var cacheEntry2 = new CacheEntry(storedMetadata2.Result, nameof(HistoricMetadataCacheTests), ArrayView <ContentHash> .Empty); var strongFingerprint2 = new StrongContentFingerprint(FingerprintUtilities.CreateRandom()); var publishedCacheEntry2 = await cache.TryPublishCacheEntryAsync(process1, weakFingerprint1, storedPathSet2.Result, strongFingerprint2, cacheEntry2); await cache.CloseAsync(); memoryArtifactCache.Clear(); PipExecutionContext loadedContext; HistoricMetadataCache loadedCache; TaskSourceSlim <bool> loadCompletionSource = TaskSourceSlim.Create <bool>(); TaskSourceSlim <bool> loadCalled = TaskSourceSlim.Create <bool>(); BoxRef <bool> calledLoad = new BoxRef <bool>(); CreateHistoricCache(loggingContext, "hmc", out loadedContext, out loadedCache, out memoryArtifactCache, loadTask: async hmc => { loadCalled.SetResult(true); await loadCompletionSource.Task; }); var operationContext = OperationContext.CreateUntracked(loggingContext); var retrievePathSet1Task = loadedCache.TryRetrievePathSetAsync(operationContext, WeakContentFingerprint.Zero, storedPathSet1.Result); var retrievdMetadata1Task = loadedCache.TryRetrieveMetadataAsync( process1, WeakContentFingerprint.Zero, StrongContentFingerprint.Zero, storedMetadata1.Result, storedPathSet1.Result); var getCacheEntry1Task = loadedCache.TryGetCacheEntryAsync( process1, weakFingerprint1, storedPathSet1.Result, strongFingerprint1); Assert.False(retrievePathSet1Task.IsCompleted, "Before load task completes. TryRetrievePathSetAsync operations should block"); Assert.False(retrievdMetadata1Task.IsCompleted, "Before load task completes. TryRetrieveMetadataAsync operations should block"); Assert.False(getCacheEntry1Task.IsCompleted, "Before load task completes. TryGetCacheEntryAsync operations should block"); Assert.True(loadCalled.Task.Wait(TimeSpan.FromSeconds(10)) && loadCalled.Task.Result, "Load should have been called in as a result of querying"); loadCompletionSource.SetResult(true); var maybeLoadedPathSet1 = await retrievePathSet1Task; var maybeLoadedMetadata1 = await retrievdMetadata1Task; var maybeLoadedCacheEntry1 = await getCacheEntry1Task; Assert.Equal(storedMetadata1.Result, maybeLoadedCacheEntry1.Result.Value.MetadataHash); var maybeLoadedPathSet2 = await loadedCache.TryRetrievePathSetAsync(operationContext, WeakContentFingerprint.Zero, storedPathSet2.Result); var maybeLoadedMetadata2 = await loadedCache.TryRetrieveMetadataAsync( process2, WeakContentFingerprint.Zero, StrongContentFingerprint.Zero, storedMetadata2.Result, storedPathSet2.Result); AssertPathSetEquals(pathTable, pathSet1, loadedContext.PathTable, maybeLoadedPathSet1.Result); AssertPathSetEquals(pathTable, pathSet2, loadedContext.PathTable, maybeLoadedPathSet2.Result); AssertMetadataEquals(metadata1, maybeLoadedMetadata1.Result); AssertMetadataEquals(metadata2, maybeLoadedMetadata2.Result); await loadedCache.CloseAsync(); } }