public void ObservedFileNamesNormalizedTheSameWayInPathsetAndJsonFingerprinter()
{
// This test is guarding codesync between JsonFingerprinter.cs and ObservedPathSet.cs
var fileNames = new string[] { "a", "b", "C" };
var pathTable = new PathTable();
var pathSet = ObservedPathSetTestUtilities.CreatePathSet(
pathTable,
observedAccessedFileNames: fileNames,
X("/X/a/b/c"));
var roundtrip = SerializeRoundTripAndAssertEquivalent(pathTable, pathSet);
var sb = new StringBuilder();
using (var writer = new global::BuildXL.Engine.Cache.JsonFingerprinter(sb, pathTable: pathTable))
{
writer.AddCollection <StringId, StringId[]>(
"fileNames",
fileNames.Select(fn => StringId.Create(pathTable.StringTable, fn)).ToArray(),
(w, e) => w.AddFileName(e));
}
var fpOutput = sb.ToString();
XAssert.IsTrue(roundtrip.ObservedAccessedFileNames.All(fileName => fpOutput.Contains($"\"{fileName.ToString(pathTable.StringTable)}\"")));
}
public void RoundTripSerializationEmpty()
{
var pathTable = new PathTable();
var pathSet = ObservedPathSetTestUtilities.CreatePathSet(pathTable, new string[0]);
var roundtrip = SerializeRoundTripAndAssertEquivalent(pathTable, pathSet);
XAssert.AreEqual(0, roundtrip.Paths.Length);
}
[Trait("Category", "WindowsOSOnly")] // Paths are case-sensitive on Linux-based systems.
public void RoundTripSerializationNormalizesCasingAndRemovesDuplicatesInObservedAccessedFileNames()
{
var pathTable = new PathTable();
var pathSet = ObservedPathSetTestUtilities.CreatePathSet(
pathTable,
observedAccessedFileNames: new string[] { "d", "D", "f" },
X("/X/a/b/c"));
var roundtrip = SerializeRoundTripAndAssertEquivalent(pathTable, pathSet);
ObservedPathSetTestUtilities.AssertPathSetContainsDuplicates(pathSet);
ObservedPathSetTestUtilities.AssertPathSetDoesNotContainDuplicates(roundtrip);
}
public void RoundTripSerializationRemovesDuplicatesWithUnrelatedPaths()
{
var pathTable = new PathTable();
var pathSet = ObservedPathSetTestUtilities.CreatePathSet(
pathTable,
X("/X/a/b/c"),
X("/X/a/b/c"),
X("/Y/d/e/f"),
X("/Y/d/e/f"));
var roundtrip = SerializeRoundTripAndAssertEquivalent(pathTable, pathSet);
ObservedPathSetTestUtilities.AssertPathSetContainsDuplicates(pathSet);
ObservedPathSetTestUtilities.AssertPathSetDoesNotContainDuplicates(roundtrip);
}
public void NoCompression()
{
var pathTable = new PathTable();
// No compression since no prefix is shared.
var pathSet = ObservedPathSetTestUtilities.CreatePathSet(
pathTable,
X("/X/a/b"),
X("/Y/a/b"));
AssertCompressedSizeExpected(
pathTable,
pathSet,
X("/X/a/b"),
X("/Y/a/b"));
}
public void PrefixCompressionWithinComponents()
{
var pathTable = new PathTable();
// No compression since no prefix is shared.
var pathSet = ObservedPathSetTestUtilities.CreatePathSet(
pathTable,
X("/X/abcdef"),
X("/X/abcxyz/123"),
X("/X/abcxyz/123456"));
AssertCompressedSizeExpected(
pathTable,
pathSet,
X("/X/abcdef"),
$"xyz{Path.DirectorySeparatorChar}123",
"456");
}
public void ProjectPathSet()
{
var pathTable = new PathTable();
AbsolutePath firstPath = AbsolutePath.Create(pathTable, X("/X/bar"));
AbsolutePath secondPath = AbsolutePath.Create(pathTable, X("/X/foo"));
var p = CreateResult(
pathTable,
ObservedInput.CreateFileContentRead(firstPath, ContentHashingUtilities.EmptyHash),
ObservedInput.CreateFileContentRead(secondPath, ContentHashingUtilities.EmptyHash));
ObservedPathSet projected = p.GetPathSet(unsafeOptions: null);
ObservedPathSetTestUtilities.AssertPathSetsEquivalent(
projected,
ObservedPathSetTestUtilities.CreatePathSet(pathTable, firstPath, secondPath));
}
public async Task TokenizedPathSet()
{
var pathTable = new PathTable();
var pathExpanderA = new MountPathExpander(pathTable);
AddMount(pathExpanderA, pathTable, AbsolutePath.Create(pathTable, X("/x/users/AUser")), "UserProfile", isSystem: true);
AddMount(pathExpanderA, pathTable, AbsolutePath.Create(pathTable, X("/x/windows")), "Windows", isSystem: true);
AddMount(pathExpanderA, pathTable, AbsolutePath.Create(pathTable, X("/x/test")), "TestRoot", isSystem: false);
var pathSetA = ObservedPathSetTestUtilities.CreatePathSet(
pathTable,
X("/x/abc"),
X("/x/users/AUser/def"),
X("/x/windows"),
X("/x/test/abc"));
ObservedPathSet roundtripA = SerializeRoundTripAndAssertEquivalent(pathTable, pathSetA);
XAssert.AreEqual(4, roundtripA.Paths.Length);
ContentHash pathSetHashA = await pathSetA.ToContentHash(pathTable, pathExpanderA);
var pathExpanderB = new MountPathExpander(pathTable);
AddMount(pathExpanderB, pathTable, AbsolutePath.Create(pathTable, X("/y/users/BUser")), "UserProfile", isSystem: true);
AddMount(pathExpanderB, pathTable, AbsolutePath.Create(pathTable, X("/y/windows")), "Windows", isSystem: true);
AddMount(pathExpanderB, pathTable, AbsolutePath.Create(pathTable, X("/y/abc/test")), "TestRoot", isSystem: false);
var pathSetB = ObservedPathSetTestUtilities.CreatePathSet(
pathTable,
X("/x/abc"),
X("/y/users/BUser/def"),
X("/y/windows"),
X("/y/abc/test/abc"));
ObservedPathSet roundtripB = SerializeRoundTripAndAssertEquivalent(pathTable, pathSetB);
XAssert.AreEqual(4, roundtripB.Paths.Length);
ContentHash pathSetHashB = await pathSetB.ToContentHash(pathTable, pathExpanderB);
AssertTrue(pathSetHashA == pathSetHashB);
}
private static void AssertCompressedSizeExpected(PathTable pathTable, ObservedPathSet pathSet, params string[] uncompressedStrings)
{
long compressedSize = GetSizeOfSerializedContent(writer => pathSet.Serialize(pathTable, writer));
int numberOfUniquePaths = ObservedPathSetTestUtilities.RemoveDuplicates(pathSet).Count;
// This is correct assuming the following:
// - Each string can be represented with a one byte length prefix, and a one byte reuse-count.
// - The number of strings can be represented in one byte.
// - Each character takes one byte when UTF8 encoded.
long expectedCompressedSize =
GetSizeOfSerializedContent(writer => pathSet.UnsafeOptions.Serialize(writer)) +
1 + // The number of observed accesses file names (0)
1 + // String count
(3 * numberOfUniquePaths) + // Length isSearchPath, isDirectoryPath, and reuse
uncompressedStrings.Sum(s => s.Length);
XAssert.AreEqual(expectedCompressedSize, compressedSize, "Wrong size for compressed path-set");
}
public void RoundTripSerializationOfPathsWithUnicodeChars()
{
var pathTable = new PathTable();
var mpe = new global::BuildXL.Engine.MountPathExpander(pathTable);
mpe.Add(
pathTable,
new global::BuildXL.Pips.SemanticPathInfo(
PathAtom.Create(pathTable.StringTable, "xcode"),
AbsolutePath.Create(pathTable, X("/c/Applications")),
false, true, true, false, false, false));
var pathSet = ObservedPathSetTestUtilities.CreatePathSet(
pathTable,
X("/c/Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/Library/CoreSimulator/Profiles/DeviceTypes/iPhone Xʀ.simdevicetype"),
X("/c/Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/Library/CoreSimulator/Profiles/DeviceTypes/iPhone Xʀ.simdevicetype/Contents"));
SerializeRoundTripAndAssertEquivalent(pathTable, pathSet, mpe);
}
public void PrefixCompressionWithReset()
{
var pathTable = new PathTable();
// No compression since no prefix is shared.
var pathSet = ObservedPathSetTestUtilities.CreatePathSet(
pathTable,
X("/X/abc"),
X("/X/abc/def"),
X("/Y/abc"),
X("/Y/abc/def"));
AssertCompressedSizeExpected(
pathTable,
pathSet,
X("/X/abc"),
$"{Path.DirectorySeparatorChar}def",
X("/Y/abc"),
$"{Path.DirectorySeparatorChar}def");
}
public void PrefixCompressionTrivial()
{
var pathTable = new PathTable();
// No compression since no prefix is shared.
var pathSet = ObservedPathSetTestUtilities.CreatePathSet(
pathTable,
X("/X/a"),
X("/X/a/b"),
X("/X/a/b/c"),
X("/X/a/b/c/d"));
AssertCompressedSizeExpected(
pathTable,
pathSet,
X("/X/a"),
$"{Path.DirectorySeparatorChar}b",
$"{Path.DirectorySeparatorChar}c",
$"{Path.DirectorySeparatorChar}d");
}
private static ObservedPathSet SerializeRoundTripAndAssertEquivalent(PathTable pathTable, ObservedPathSet original, PathExpander pathExpander = null)
{
using (var mem = new MemoryStream())
{
using (var writer = new BuildXLWriter(stream: mem, debug: true, leaveOpen: true, logStats: true))
{
original.Serialize(pathTable, writer, pathExpander);
}
mem.Position = 0;
ObservedPathSet roundtrip;
using (var reader = new BuildXLReader(stream: mem, debug: true, leaveOpen: true))
{
var maybeRoundtrip = ObservedPathSet.TryDeserialize(pathTable, reader, pathExpander);
XAssert.IsTrue(maybeRoundtrip.Succeeded, "Failed to deserialize a path set unexpectedly");
roundtrip = maybeRoundtrip.Result;
}
ObservedPathSetTestUtilities.AssertPathSetsEquivalent(original, roundtrip);
return(roundtrip);
}
}
public async Task TestHistoricMetadataPathStringRoundtrip()
{
LoggingContext loggingContext = CreateLoggingContextForTest();
PipExecutionContext context;
HistoricMetadataCache cache = null;
var hmcFolderName = "hmc";
for (int i = 0; i < 3; i++)
{
CreateHistoricCache(loggingContext, hmcFolderName, out context, out cache, out var memoryArtifactCache);
var process1 = SchedulerTest.CreateDummyProcess(context, new PipId(1));
var process2 = SchedulerTest.CreateDummyProcess(context, new PipId(2));
var pathTable = context.PathTable;
// Add some random paths to ensure path table indices are different after loading
AbsolutePath.Create(pathTable, X("/H/aslj/sfas/832.stxt"));
AbsolutePath.Create(pathTable, X("/R/f/s/Historic"));
AbsolutePath.Create(pathTable, X("/M/hgf/sf4as/83afsd"));
AbsolutePath.Create(pathTable, X("/Z/bd/sfas/Cache"));
var abPath1 = AbsolutePath.Create(pathTable, X("/H/aslj/sfas/p1OUT.bin"));
var abPath2 = AbsolutePath.Create(pathTable, X("/H/aslj/sfas/P2.txt"));
var pathSet1 = ObservedPathSetTestUtilities.CreatePathSet(
pathTable,
X("/X/a/b/c"),
X("/X/d/e"),
X("/X/a/b/c/d"));
PipCacheDescriptorV2Metadata metadata1 =
new PipCacheDescriptorV2Metadata
{
StaticOutputHashes = new List <AbsolutePathFileMaterializationInfo>
{
new AbsolutePathFileMaterializationInfo
{
AbsolutePath = abPath1.GetName(pathTable).ToString(context.StringTable),
Info = new BondFileMaterializationInfo {
FileName = "p1OUT.bin"
}
}
}
};
var storedPathSet1 = await cache.TryStorePathSetAsync(pathSet1, preservePathCasing : false);
var storedMetadata1 = await cache.TryStoreMetadataAsync(metadata1);
var weakFingerprint1 = new WeakContentFingerprint(FingerprintUtilities.CreateRandom());
var strongFingerprint1 = new StrongContentFingerprint(FingerprintUtilities.CreateRandom());
var cacheEntry = new CacheEntry(storedMetadata1.Result, nameof(HistoricMetadataCacheTests), ArrayView <ContentHash> .Empty);
var publishedCacheEntry = await cache.TryPublishCacheEntryAsync(process1, weakFingerprint1, storedPathSet1.Result, strongFingerprint1, cacheEntry);
var pathSet2 = ObservedPathSetTestUtilities.CreatePathSet(
pathTable,
X("/F/a/y/c"),
X("/B/d/e"),
X("/G/a/z/c/d"),
X("/B/a/b/c"));
PipCacheDescriptorV2Metadata metadata2 =
new PipCacheDescriptorV2Metadata
{
StaticOutputHashes = new List <AbsolutePathFileMaterializationInfo>
{
new AbsolutePathFileMaterializationInfo
{
AbsolutePath = abPath2.ToString(pathTable),
Info = new BondFileMaterializationInfo {
FileName = abPath2.GetName(pathTable).ToString(context.StringTable)
}
}
},
DynamicOutputs = new List <List <RelativePathFileMaterializationInfo> >
{
new List <RelativePathFileMaterializationInfo>
{
new RelativePathFileMaterializationInfo
{
RelativePath = @"dir\P2Dynamic.txt",
Info = new BondFileMaterializationInfo {
FileName = "p2dynamic.txt"
}
},
new RelativePathFileMaterializationInfo
{
RelativePath = @"dir\P2dynout2.txt",
Info = new BondFileMaterializationInfo {
FileName = null
}
}
}
}
};
var storedPathSet2 = await cache.TryStorePathSetAsync(pathSet2, preservePathCasing : false);
var storedMetadata2 = await cache.TryStoreMetadataAsync(metadata2);
var cacheEntry2 = new CacheEntry(storedMetadata2.Result, nameof(HistoricMetadataCacheTests), ArrayView <ContentHash> .Empty);
var strongFingerprint2 = new StrongContentFingerprint(FingerprintUtilities.CreateRandom());
var publishedCacheEntry2 = await cache.TryPublishCacheEntryAsync(process1, weakFingerprint1, storedPathSet2.Result, strongFingerprint2, cacheEntry2);
await cache.CloseAsync();
memoryArtifactCache.Clear();
PipExecutionContext loadedContext;
HistoricMetadataCache loadedCache;
TaskSourceSlim <bool> loadCompletionSource = TaskSourceSlim.Create <bool>();
TaskSourceSlim <bool> loadCalled = TaskSourceSlim.Create <bool>();
BoxRef <bool> calledLoad = new BoxRef <bool>();
CreateHistoricCache(loggingContext, "hmc", out loadedContext, out loadedCache, out memoryArtifactCache, loadTask: async hmc =>
{
loadCalled.SetResult(true);
await loadCompletionSource.Task;
});
var operationContext = OperationContext.CreateUntracked(loggingContext);
var retrievePathSet1Task = loadedCache.TryRetrievePathSetAsync(operationContext, WeakContentFingerprint.Zero, storedPathSet1.Result);
var retrievdMetadata1Task = loadedCache.TryRetrieveMetadataAsync(
process1,
WeakContentFingerprint.Zero,
StrongContentFingerprint.Zero,
storedMetadata1.Result,
storedPathSet1.Result);
var getCacheEntry1Task = loadedCache.TryGetCacheEntryAsync(
process1,
weakFingerprint1,
storedPathSet1.Result,
strongFingerprint1);
Assert.False(retrievePathSet1Task.IsCompleted, "Before load task completes. TryRetrievePathSetAsync operations should block");
Assert.False(retrievdMetadata1Task.IsCompleted, "Before load task completes. TryRetrieveMetadataAsync operations should block");
Assert.False(getCacheEntry1Task.IsCompleted, "Before load task completes. TryGetCacheEntryAsync operations should block");
Assert.True(loadCalled.Task.Wait(TimeSpan.FromSeconds(10)) && loadCalled.Task.Result, "Load should have been called in as a result of querying");
loadCompletionSource.SetResult(true);
var maybeLoadedPathSet1 = await retrievePathSet1Task;
var maybeLoadedMetadata1 = await retrievdMetadata1Task;
var maybeLoadedCacheEntry1 = await getCacheEntry1Task;
Assert.Equal(storedMetadata1.Result, maybeLoadedCacheEntry1.Result.Value.MetadataHash);
var maybeLoadedPathSet2 = await loadedCache.TryRetrievePathSetAsync(operationContext, WeakContentFingerprint.Zero, storedPathSet2.Result);
var maybeLoadedMetadata2 = await loadedCache.TryRetrieveMetadataAsync(
process2,
WeakContentFingerprint.Zero,
StrongContentFingerprint.Zero,
storedMetadata2.Result,
storedPathSet2.Result);
AssertPathSetEquals(pathTable, pathSet1, loadedContext.PathTable, maybeLoadedPathSet1.Result);
AssertPathSetEquals(pathTable, pathSet2, loadedContext.PathTable, maybeLoadedPathSet2.Result);
AssertMetadataEquals(metadata1, maybeLoadedMetadata1.Result);
AssertMetadataEquals(metadata2, maybeLoadedMetadata2.Result);
await loadedCache.CloseAsync();
}
}
