public void AddContentHash() { AbsolutePath.TryCreate(PathTable, X("/d/nonsensePath"), out AbsolutePath path); var hash = ContentHashingUtilities.CreateRandom(); using (var writer = new JsonFingerprinter(StringBuilder, pathTable: PathTable)) { writer.Add(path, hash); } VerifyJsonContent(PathToString(path), JsonFingerprinter.ContentHashToString(hash)); StringBuilder.Clear(); using (var writer = new JsonFingerprinter(StringBuilder, pathTable: PathTable)) { writer.Add("PathContentHash", path, hash); } VerifyJsonContent("PathContentHash", PathToString(path), JsonFingerprinter.ContentHashToString(hash)); StringBuilder.Clear(); using (var writer = new JsonFingerprinter(StringBuilder, pathTable: PathTable)) { writer.Add("ContentHash", hash); } VerifyJsonContent("ContentHash", JsonFingerprinter.ContentHashToString(hash)); }
public void AddCollection() { var intCollection = new int[] { 1, 2, 3 }; var stringCollection = new string[] { X("/d/1"), X("/d/2"), X("/d/3") }; using (var writer = new JsonFingerprinter(StringBuilder, pathTable: PathTable)) { writer.AddCollection <int, int[]>("ints", intCollection, (w, e) => w.Add(e)); } VerifyJsonContent(intCollection.Select(i => i.ToString()).ToArray()); // Test multiple operations per element using (var writer = new JsonFingerprinter(StringBuilder, pathTable: PathTable)) { writer.AddCollection <string, string[]>( "strings", stringCollection, (w, e) => { AbsolutePath.TryCreate(PathTable, e, out AbsolutePath path); w.Add(e, path); w.Add(path); }); } VerifyJsonContent(stringCollection); }
public void ComboTest() { AbsolutePath.TryCreate(PathTable, X("/d/nonsensePath"), out AbsolutePath path); var stringId = StringId.Create(PathTable.StringTable, "testString"); var fingerprint = FingerprintUtilities.ZeroFingerprint; var hash = ContentHashingUtilities.CreateRandom(); var longValue = (long)23; var stringValue = "asdf"; var intCollection = new int[] { 1, 2, 3 }; // Do some random operations and make sure there's no exceptions using (var writer = new JsonFingerprinter(StringBuilder, pathTable: PathTable)) { writer.Add(stringValue, longValue); writer.AddCollection <int, int[]>("ints", intCollection, (w, e) => w.Add(e)); writer.Add(stringValue, hash); writer.Add(stringValue, stringId); writer.Add(stringValue, stringId); writer.Add(stringValue, path, hash); } VerifyJsonContent("testString"); VerifyJsonContent(longValue.ToString()); VerifyJsonContent(intCollection.Select(i => i.ToString()).ToArray()); VerifyJsonContent(JsonFingerprinter.ContentHashToString(hash)); }
public void AddAbsolutePath() { AbsolutePath.TryCreate(PathTable, X("/d/nonsense"), out AbsolutePath path); using (var writer = new JsonFingerprinter(StringBuilder, pathTable: PathTable)) { writer.Add("AbsolutePath", path); } VerifyJsonContent("AbsolutePath", PathToString(path)); }
public void AddStringId() { var stringId = StringId.Create(PathTable.StringTable, "testString"); using (var writer = new JsonFingerprinter(StringBuilder, pathTable: PathTable)) { writer.Add("StringId", stringId); } VerifyJsonContent("StringId", stringId.ToString(PathTable.StringTable)); }
public void AddFingerprint() { var fingerprint = FingerprintUtilities.ZeroFingerprint; using (var writer = new JsonFingerprinter(StringBuilder, pathTable: PathTable)) { writer.Add("fingerprint", fingerprint); } VerifyJsonContent("fingerprint", fingerprint.ToString()); }
public void AddComboCollection() { using (var writer = new JsonFingerprinter(StringBuilder, pathTable: PathTable)) { writer.AddCollection <int, int[]>("testCollection", new int[] { 1 }, (f, v) => { f.Add("string", "text"); f.Add(v); f.Add("int", 2323); f.Add("long", (long)2323); f.Add(v); }); } VerifyJsonContent("string", "text"); VerifyJsonContent("int", 2323.ToString()); VerifyJsonContent(1.ToString()); }
/// <summary> /// Given the root <see cref="JsonNode"/> a <see cref="JsonTree"/>, converts the tree into a valid JSON string. /// </summary> /// <remarks> /// This function is recursive to allow re-using <see cref="JsonFingerprinter"/> helper functions for nested objects instead of having to re-build the JSON string from scratch. /// The max nested depth for JSON representation of fingerprints is relatively low (~5 stacks), so stack memory should be trivial. /// </remarks> public static string Serialize(JsonNode root) { return(JsonFingerprinter.CreateJsonString(wr => { // If the root is being used to just point to a bunch of child nodes, skip printing it if (string.IsNullOrEmpty(root.Name)) { for (var it = root.Children.First; it != null; it = it.Next) { BuildStringHelper(it.Value, wr); } } else { BuildStringHelper(root, wr); } }, Formatting.Indented)); }
public void NestedTest() { var intCollection = new int[] { 1, 2, 3 }; var stringCollection = new string[] { X("/d/1"), X("/d/2"), X("/d/3") }; string hello = "hello", world = "world", exclamation = "!"; using (var writer = new JsonFingerprinter(StringBuilder, pathTable: PathTable)) { writer.AddCollection <string, string[]>("strings", stringCollection, (fCollection, ele) => { fCollection.Add(ele); fCollection.AddNested("nestedObject", (fNested) => { fNested.AddCollection <int, int[]>("nestedCollectionInNestedObject", intCollection, (fNestedCollectionNestedObject, intEle) => { fNestedCollectionNestedObject.Add(intEle); fNestedCollectionNestedObject.Add(hello); }); }); fCollection.AddCollection <int, int[]>("nestedCollection", intCollection, (fNestedCollection, intEle) => { fNestedCollection.Add(intEle); fNestedCollection.Add(exclamation); }); }); writer.AddNested("object", (fNested) => { fNested.AddCollection <string, string[]>("strings", stringCollection, (fCollection, ele) => { fCollection.AddNested("string", (fNestedInCollection) => { fNestedInCollection.Add(ele, ele); fNestedInCollection.Add(world, world); }); }); }); } VerifyJsonContent(hello, world, exclamation); }
public void AddPrimitives() { using (var writer = new JsonFingerprinter(StringBuilder, pathTable: PathTable)) { writer.Add("string", "text"); } VerifyJsonContent("string", "text"); StringBuilder.Clear(); using (var writer = new JsonFingerprinter(StringBuilder, pathTable: PathTable)) { writer.Add("int", 2323); } VerifyJsonContent("int", 2323.ToString()); StringBuilder.Clear(); using (var writer = new JsonFingerprinter(StringBuilder, pathTable: PathTable)) { writer.Add("long", (long)235); } VerifyJsonContent("long", ((long)235).ToString()); }
public void EmptyTest() { using (var writer = new JsonFingerprinter(StringBuilder, pathTable: PathTable)) { } }
public void StrongFingerprintMissWithAugmentedWeakFingerprintPostCacheLookUp() { Configuration.Cache.AugmentWeakFingerprintPathSetThreshold = 2; var directory = CreateUniqueDirectoryArtifact(); var sourceFile = CreateSourceFile(directory.Path); var readFileA = CreateSourceFile(directory.Path); var readFileB = CreateSourceFile(directory.Path); var readFileC = CreateSourceFile(directory.Path); var sealedDirectory = CreateAndScheduleSealDirectoryArtifact( directory.Path, global::BuildXL.Pips.Operations.SealDirectoryKind.SourceAllDirectories); var builder = CreatePipBuilder(new[] { Operation.ReadFileFromOtherFile(sourceFile, doNotInfer: true), Operation.WriteFile(CreateOutputFileArtifact()) }); builder.AddInputDirectory(sealedDirectory); var process = SchedulePipBuilder(builder).Process; // Pip will read file source and file A. // Two-phase cache will contain mapping // wp => (#{source, A}, sp1) File.WriteAllText(ArtifactToString(sourceFile), ArtifactToString(readFileA)); RunScheduler().AssertCacheMiss(process.PipId); // Pip will read file source and file B. // Two-phase cache will contain mapping // wp => (#{source, B}, sp2), (#{source, A}, sp1) File.WriteAllText(ArtifactToString(sourceFile), ArtifactToString(readFileB)); var result = RunScheduler().AssertCacheMiss(process.PipId); var cacheInfo2 = result.RunData.ExecutionCachingInfos[process.PipId]; // Pip will read file source and file C. // Two-phase cache will contain mappings // wp => (#{source}, aug_marker), (#{source, B}, sp2), (#{source, A}, sp1) // aug_wp1 => (#{source, C}, sp3) // // Fingerprint store contains mapping // pip => (wp, #{source, C}, sp3) File.WriteAllText(ArtifactToString(sourceFile), ArtifactToString(readFileC)); result = RunScheduler().AssertCacheMiss(process.PipId); var cacheInfo3 = result.RunData.ExecutionCachingInfos[process.PipId];; FingerprintStoreSession( result, store => { store.TryGetFingerprintStoreEntryBySemiStableHash(process.FormattedSemiStableHash, out var entry); // Weak fingerprint stored in the fingerprint store is the original one; not the augmented one. // So use the weak fingerprint from previous build, i.e., cacheInfo2. XAssert.AreEqual(cacheInfo2.WeakFingerprint.ToString(), entry.PipToFingerprintKeys.Value.WeakFingerprint); XAssert.AreEqual( JsonFingerprinter.ContentHashToString(cacheInfo3.PathSetHash), entry.PipToFingerprintKeys.Value.FormattedPathSetHash); XAssert.AreEqual(cacheInfo3.StrongFingerprint.ToString(), entry.PipToFingerprintKeys.Value.StrongFingerprint); }); // Modify file C. File.WriteAllText(ArtifactToString(readFileC), "modified"); // Runtime cache miss analysis will be done post cache look-up. // Although perhaps the last evaluated cache entry is (#{source, A}, sp1), the fingerprint store // will try to find the most relevant entry for cache miss analysis, which in this case (#{source, C}, sp3). RunScheduler(m_testHooks).AssertCacheMiss(process.PipId); XAssert.IsTrue(m_testHooks.FingerprintStoreTestHooks.TryGetCacheMiss(process.PipId, out var cacheMiss)); XAssert.AreEqual(CacheMissAnalysisResult.StrongFingerprintMismatch, cacheMiss.DetailAndResult.Result); XAssert.IsTrue(cacheMiss.IsFromCacheLookUp); // Ensure that the diff contains file C because the fingerprint store uses the most relevant cache entry for // cache miss analysis, i.e., (#{source, C}, sp3) XAssert.Contains(JsonConvert.SerializeObject(cacheMiss.DetailAndResult.Detail.Info).ToUpperInvariant(), ArtifactToString(readFileC).Replace("\\", "\\\\").ToUpperInvariant()); }