public void AddContentHash()
{
AbsolutePath.TryCreate(PathTable, X("/d/nonsensePath"), out AbsolutePath path);
var hash = ContentHashingUtilities.CreateRandom();
using (var writer = new JsonFingerprinter(StringBuilder, pathTable: PathTable))
{
writer.Add(path, hash);
}
VerifyJsonContent(PathToString(path), JsonFingerprinter.ContentHashToString(hash));
StringBuilder.Clear();
using (var writer = new JsonFingerprinter(StringBuilder, pathTable: PathTable))
{
writer.Add("PathContentHash", path, hash);
}
VerifyJsonContent("PathContentHash", PathToString(path), JsonFingerprinter.ContentHashToString(hash));
StringBuilder.Clear();
using (var writer = new JsonFingerprinter(StringBuilder, pathTable: PathTable))
{
writer.Add("ContentHash", hash);
}
VerifyJsonContent("ContentHash", JsonFingerprinter.ContentHashToString(hash));
}
public void AddCollection()
{
var intCollection = new int[] { 1, 2, 3 };
var stringCollection = new string[] { X("/d/1"), X("/d/2"), X("/d/3") };
using (var writer = new JsonFingerprinter(StringBuilder, pathTable: PathTable))
{
writer.AddCollection <int, int[]>("ints", intCollection, (w, e) => w.Add(e));
}
VerifyJsonContent(intCollection.Select(i => i.ToString()).ToArray());
// Test multiple operations per element
using (var writer = new JsonFingerprinter(StringBuilder, pathTable: PathTable))
{
writer.AddCollection <string, string[]>(
"strings",
stringCollection,
(w, e) =>
{
AbsolutePath.TryCreate(PathTable, e, out AbsolutePath path);
w.Add(e, path);
w.Add(path);
});
}
VerifyJsonContent(stringCollection);
}
public void ComboTest()
{
AbsolutePath.TryCreate(PathTable, X("/d/nonsensePath"), out AbsolutePath path);
var stringId = StringId.Create(PathTable.StringTable, "testString");
var fingerprint = FingerprintUtilities.ZeroFingerprint;
var hash = ContentHashingUtilities.CreateRandom();
var longValue = (long)23;
var stringValue = "asdf";
var intCollection = new int[] { 1, 2, 3 };
// Do some random operations and make sure there's no exceptions
using (var writer = new JsonFingerprinter(StringBuilder, pathTable: PathTable))
{
writer.Add(stringValue, longValue);
writer.AddCollection <int, int[]>("ints", intCollection, (w, e) => w.Add(e));
writer.Add(stringValue, hash);
writer.Add(stringValue, stringId);
writer.Add(stringValue, stringId);
writer.Add(stringValue, path, hash);
}
VerifyJsonContent("testString");
VerifyJsonContent(longValue.ToString());
VerifyJsonContent(intCollection.Select(i => i.ToString()).ToArray());
VerifyJsonContent(JsonFingerprinter.ContentHashToString(hash));
}
public void AddAbsolutePath()
{
AbsolutePath.TryCreate(PathTable, X("/d/nonsense"), out AbsolutePath path);
using (var writer = new JsonFingerprinter(StringBuilder, pathTable: PathTable))
{
writer.Add("AbsolutePath", path);
}
VerifyJsonContent("AbsolutePath", PathToString(path));
}
public void AddStringId()
{
var stringId = StringId.Create(PathTable.StringTable, "testString");
using (var writer = new JsonFingerprinter(StringBuilder, pathTable: PathTable))
{
writer.Add("StringId", stringId);
}
VerifyJsonContent("StringId", stringId.ToString(PathTable.StringTable));
}
public void AddFingerprint()
{
var fingerprint = FingerprintUtilities.ZeroFingerprint;
using (var writer = new JsonFingerprinter(StringBuilder, pathTable: PathTable))
{
writer.Add("fingerprint", fingerprint);
}
VerifyJsonContent("fingerprint", fingerprint.ToString());
}
public void AddComboCollection()
{
using (var writer = new JsonFingerprinter(StringBuilder, pathTable: PathTable))
{
writer.AddCollection <int, int[]>("testCollection", new int[] { 1 }, (f, v) =>
{
f.Add("string", "text");
f.Add(v);
f.Add("int", 2323);
f.Add("long", (long)2323);
f.Add(v);
});
}
VerifyJsonContent("string", "text");
VerifyJsonContent("int", 2323.ToString());
VerifyJsonContent(1.ToString());
}
/// <summary>
/// Given the root <see cref="JsonNode"/> a <see cref="JsonTree"/>, converts the tree into a valid JSON string.
/// </summary>
/// <remarks>
/// This function is recursive to allow re-using <see cref="JsonFingerprinter"/> helper functions for nested objects instead of having to re-build the JSON string from scratch.
/// The max nested depth for JSON representation of fingerprints is relatively low (~5 stacks), so stack memory should be trivial.
/// </remarks>
public static string Serialize(JsonNode root)
{
return(JsonFingerprinter.CreateJsonString(wr =>
{
// If the root is being used to just point to a bunch of child nodes, skip printing it
if (string.IsNullOrEmpty(root.Name))
{
for (var it = root.Children.First; it != null; it = it.Next)
{
BuildStringHelper(it.Value, wr);
}
}
else
{
BuildStringHelper(root, wr);
}
},
Formatting.Indented));
}
public void NestedTest()
{
var intCollection = new int[] { 1, 2, 3 };
var stringCollection = new string[] { X("/d/1"), X("/d/2"), X("/d/3") };
string hello = "hello", world = "world", exclamation = "!";
using (var writer = new JsonFingerprinter(StringBuilder, pathTable: PathTable))
{
writer.AddCollection <string, string[]>("strings", stringCollection, (fCollection, ele) =>
{
fCollection.Add(ele);
fCollection.AddNested("nestedObject", (fNested) =>
{
fNested.AddCollection <int, int[]>("nestedCollectionInNestedObject", intCollection, (fNestedCollectionNestedObject, intEle) =>
{
fNestedCollectionNestedObject.Add(intEle);
fNestedCollectionNestedObject.Add(hello);
});
});
fCollection.AddCollection <int, int[]>("nestedCollection", intCollection, (fNestedCollection, intEle) =>
{
fNestedCollection.Add(intEle);
fNestedCollection.Add(exclamation);
});
});
writer.AddNested("object", (fNested) =>
{
fNested.AddCollection <string, string[]>("strings", stringCollection, (fCollection, ele) =>
{
fCollection.AddNested("string", (fNestedInCollection) =>
{
fNestedInCollection.Add(ele, ele);
fNestedInCollection.Add(world, world);
});
});
});
}
VerifyJsonContent(hello, world, exclamation);
}
public void AddPrimitives()
{
using (var writer = new JsonFingerprinter(StringBuilder, pathTable: PathTable))
{
writer.Add("string", "text");
}
VerifyJsonContent("string", "text");
StringBuilder.Clear();
using (var writer = new JsonFingerprinter(StringBuilder, pathTable: PathTable))
{
writer.Add("int", 2323);
}
VerifyJsonContent("int", 2323.ToString());
StringBuilder.Clear();
using (var writer = new JsonFingerprinter(StringBuilder, pathTable: PathTable))
{
writer.Add("long", (long)235);
}
VerifyJsonContent("long", ((long)235).ToString());
}
public void EmptyTest()
{
using (var writer = new JsonFingerprinter(StringBuilder, pathTable: PathTable))
{
}
}
public void StrongFingerprintMissWithAugmentedWeakFingerprintPostCacheLookUp()
{
Configuration.Cache.AugmentWeakFingerprintPathSetThreshold = 2;
var directory = CreateUniqueDirectoryArtifact();
var sourceFile = CreateSourceFile(directory.Path);
var readFileA = CreateSourceFile(directory.Path);
var readFileB = CreateSourceFile(directory.Path);
var readFileC = CreateSourceFile(directory.Path);
var sealedDirectory = CreateAndScheduleSealDirectoryArtifact(
directory.Path,
global::BuildXL.Pips.Operations.SealDirectoryKind.SourceAllDirectories);
var builder = CreatePipBuilder(new[]
{
Operation.ReadFileFromOtherFile(sourceFile, doNotInfer: true),
Operation.WriteFile(CreateOutputFileArtifact())
});
builder.AddInputDirectory(sealedDirectory);
var process = SchedulePipBuilder(builder).Process;
// Pip will read file source and file A.
// Two-phase cache will contain mapping
// wp => (#{source, A}, sp1)
File.WriteAllText(ArtifactToString(sourceFile), ArtifactToString(readFileA));
RunScheduler().AssertCacheMiss(process.PipId);
// Pip will read file source and file B.
// Two-phase cache will contain mapping
// wp => (#{source, B}, sp2), (#{source, A}, sp1)
File.WriteAllText(ArtifactToString(sourceFile), ArtifactToString(readFileB));
var result = RunScheduler().AssertCacheMiss(process.PipId);
var cacheInfo2 = result.RunData.ExecutionCachingInfos[process.PipId];
// Pip will read file source and file C.
// Two-phase cache will contain mappings
// wp => (#{source}, aug_marker), (#{source, B}, sp2), (#{source, A}, sp1)
// aug_wp1 => (#{source, C}, sp3)
//
// Fingerprint store contains mapping
// pip => (wp, #{source, C}, sp3)
File.WriteAllText(ArtifactToString(sourceFile), ArtifactToString(readFileC));
result = RunScheduler().AssertCacheMiss(process.PipId);
var cacheInfo3 = result.RunData.ExecutionCachingInfos[process.PipId];;
FingerprintStoreSession(
result,
store =>
{
store.TryGetFingerprintStoreEntryBySemiStableHash(process.FormattedSemiStableHash, out var entry);
// Weak fingerprint stored in the fingerprint store is the original one; not the augmented one.
// So use the weak fingerprint from previous build, i.e., cacheInfo2.
XAssert.AreEqual(cacheInfo2.WeakFingerprint.ToString(), entry.PipToFingerprintKeys.Value.WeakFingerprint);
XAssert.AreEqual(
JsonFingerprinter.ContentHashToString(cacheInfo3.PathSetHash),
entry.PipToFingerprintKeys.Value.FormattedPathSetHash);
XAssert.AreEqual(cacheInfo3.StrongFingerprint.ToString(), entry.PipToFingerprintKeys.Value.StrongFingerprint);
});
// Modify file C.
File.WriteAllText(ArtifactToString(readFileC), "modified");
// Runtime cache miss analysis will be done post cache look-up.
// Although perhaps the last evaluated cache entry is (#{source, A}, sp1), the fingerprint store
// will try to find the most relevant entry for cache miss analysis, which in this case (#{source, C}, sp3).
RunScheduler(m_testHooks).AssertCacheMiss(process.PipId);
XAssert.IsTrue(m_testHooks.FingerprintStoreTestHooks.TryGetCacheMiss(process.PipId, out var cacheMiss));
XAssert.AreEqual(CacheMissAnalysisResult.StrongFingerprintMismatch, cacheMiss.DetailAndResult.Result);
XAssert.IsTrue(cacheMiss.IsFromCacheLookUp);
// Ensure that the diff contains file C because the fingerprint store uses the most relevant cache entry for
// cache miss analysis, i.e., (#{source, C}, sp3)
XAssert.Contains(JsonConvert.SerializeObject(cacheMiss.DetailAndResult.Detail.Info).ToUpperInvariant(), ArtifactToString(readFileC).Replace("\\", "\\\\").ToUpperInvariant());
}
