private static JProperty DiffUnsafeOptions(JsonNode unsafeOptionsNode, JsonNode otherUnsafeOptionsNode) { Contract.Requires(unsafeOptionsNode != null); Contract.Requires(otherUnsafeOptionsNode != null); Contract.Requires(unsafeOptionsNode.Name == otherUnsafeOptionsNode.Name); // Get the diff result of the single values in unsafeUnsafeOptions JObject result = DiffNameValuePairs(unsafeOptionsNode, otherUnsafeOptionsNode); // Deal with the preserveOutputInfo struct in unsafeUnsafeOptions JsonNode preserveOutputInfoNode = JsonTree.FindNodeByName(unsafeOptionsNode, nameof(PreserveOutputsInfo)); JsonNode otherpreserveOutputInfoNode = JsonTree.FindNodeByName(otherUnsafeOptionsNode, nameof(PreserveOutputsInfo)); if (preserveOutputInfoNode != null || otherpreserveOutputInfoNode != null) { JObject preserveOutputInfoResult = DiffNameValuePairs( preserveOutputInfoNode ?? new JsonNode() { Name = nameof(PreserveOutputsInfo) }, otherpreserveOutputInfoNode ?? new JsonNode() { Name = nameof(PreserveOutputsInfo) }); if (preserveOutputInfoResult != null) { result = result ?? new JObject(); AddPropertyIfNotNull(result, new JProperty(nameof(PreserveOutputsInfo), preserveOutputInfoResult)); } } return(result != null ? new JProperty(unsafeOptionsNode.Name, result) : null); }
/// <summary> /// Diffs strong fingerprints. /// </summary> /// <param name="pathSetHash">Pathset hash.</param> /// <param name="pathSetTree">Pathset tree.</param> /// <param name="strongFingerprintInputTree">Strong fingerprint input tree.</param> /// <param name="otherPathSetHash">Other pathset hash.</param> /// <param name="otherPathSetTree">Other pathset tree.</param> /// <param name="otherStrongFingerprintInputTree">Other strong fingerprint input tree.</param> /// <param name="getDirectoryMembership">Delegate for getting directory membership.</param> /// <param name="getOtherDirectoryMembership">Delegate for getting other directory membership.</param> /// <returns></returns> public static JObject DiffPathSets( string pathSetHash, JsonNode pathSetTree, JsonNode strongFingerprintInputTree, string otherPathSetHash, JsonNode otherPathSetTree, JsonNode otherStrongFingerprintInputTree, Func <string, IReadOnlyList <string> > getDirectoryMembership, Func <string, IReadOnlyList <string> > getOtherDirectoryMembership) { JObject result = new JObject(); if (pathSetHash == otherPathSetHash) { return(result); } // { // PathSetHash: { Old: old_path_set_hash, New: new_path_set_hash } // } AddPropertyIfNotNull(result, RenderSingleValueDiff("PathSetHash", pathSetHash, otherPathSetHash)); JsonNode unsafeOptionsNode = JsonTree.FindNodeByName(pathSetTree, ObservedPathSet.Labels.UnsafeOptions); JsonNode otherUnsafeOptionsNode = JsonTree.FindNodeByName(otherPathSetTree, ObservedPathSet.Labels.UnsafeOptions); // This is less ideal because we can't see the difference. // TODO: dump unsafe option data to the fingerprint store so that we can analyze the content. // { // UnsafeOptions: { Old: old_bits, New: new_bits: } // } AddPropertyIfNotNull(result, RenderSingleValueDiff(ObservedPathSet.Labels.UnsafeOptions, unsafeOptionsNode.Values[0], otherUnsafeOptionsNode.Values[0])); AddPropertyIfNotNull( result, DiffObservedPaths( pathSetTree, strongFingerprintInputTree, otherPathSetTree, otherStrongFingerprintInputTree, getDirectoryMembership, getOtherDirectoryMembership)); JsonNode obsFileNameNode = JsonTree.FindNodeByName(pathSetTree, ObservedPathSet.Labels.ObservedAccessedFileNames); JsonNode otherObsFileNameNode = JsonTree.FindNodeByName(otherPathSetTree, ObservedPathSet.Labels.ObservedAccessedFileNames); bool hasDiff = ExtractUnorderedListDiff(obsFileNameNode.Values, otherObsFileNameNode.Values, out var addedFileNames, out var removedFileName); if (hasDiff) { result.Add(new JProperty( ObservedPathSet.Labels.ObservedAccessedFileNames, RenderUnorderedListDiff(addedFileNames, removedFileName, RenderPath))); } Contract.Assert(result.Count > 0); return(result); }
public void NestedNamelessArraysTest() { var jsonA = "{\"key\":[\"value\", [\"nestedValue\"]]}"; var treeA = JsonTree.Deserialize(jsonA); var keyNode = JsonTree.FindNodeByName(treeA, "key"); XAssert.IsTrue(keyNode.Values.Contains("value") && keyNode.Values.Contains("nestedValue")); string deserialized = JsonTree.Serialize(treeA); XAssert.IsTrue(deserialized.Contains("value") && deserialized.Contains("nestedValue")); XAssert.AreNotEqual(JsonTree.PrettyPrintJson(jsonA), deserialized); }
/// <summary> /// Path set hash inputs are stored separately from the strong fingerprint inputs. /// This merges the path set hash inputs tree into the strong fingerprint inputs tree /// while maintaining the 1:1 relationship between the path set and observed inputs. /// /// Node notation: /// [id] "{name}":"{value}" /// Tree notation: /// {parentNode} /// {childNode} /// /// Start with the following subtrees: /// /// From strong fingerprint /// /// [1] "PathSet":"VSO0:7E2E49845EC0AE7413519E3EE605272078AF0B1C2911C021681D1D9197CC134A00" /// [2] "ObservedInputs":"" /// [3] "E":"VSO0:E0C5007DC8CF2D331236F156F136C50CACE2A5D549CD132D9B44ABD1F13D50CC00", /// /// From path set hash /// /// [4] "PathSet":"" /// [5] "Path":"B:/out/objects/n/x/qbkexxlc8je93wycw7yrlw0a305n7k/xunit-out/CacheMissAnaAD836B23/3/obj/readonly/src_0" /// [6] "Flags":"IsDirectoryPath, DirectoryEnumeration, DirectoryEnumerationWithAllPattern" /// [7] "EnumeratePatternRegex":"^.*$" /// /// And end with: /// /// [1] "PathSet":"VSO0:7E2E49845EC0AE7413519E3EE605272078AF0B1C2911C021681D1D9197CC134A00" /// [5'] "B:/out/objects/n/x/qbkexxlc8je93wycw7yrlw0a305n7k/xunit-out/CacheMissAnaAD836B23/3/obj/readonly/src_0":"" /// [6'] "Flags":"IsDirectoryPath, DirectoryEnumeration, DirectoryEnumerationWithAllPattern" /// [7'] "EnumeratePatternRegex":"^.*$" /// [3'] "ObservedInput":"E:VSO0:E0C5007DC8CF2D331236F156F136C50CACE2A5D549CD132D9B44ABD1F13D50CC00" /// [8] "Members":"[src_1, src_2]" /// </summary> /// <returns> /// The root node of the merged tree (which will be the strong fingerprint tree's root). /// </returns> private JsonNode MergeStrongFingerprintAndPathSetTrees(JsonNode strongFingerprintTree, JsonNode pathSetTree) { // [1] "PathSet":"VSO0:7E2E49845EC0AE7413519E3EE605272078AF0B1C2911C021681D1D9197CC134A00") var parentPathNode = JsonTree.FindNodeByName(strongFingerprintTree, ObservedPathEntryConstants.PathSet); // [2] "ObservedInputs":"" var observedInputsNode = JsonTree.FindNodeByName(strongFingerprintTree, ObservedInputConstants.ObservedInputs); JsonTree.EmancipateBranch(observedInputsNode); // In preparation for merging with observed inputs nodes, // remove the path set node's branch from the path set tree // [4] "PathSet":"" var pathSetNode = JsonTree.FindNodeByName(pathSetTree, ObservedPathEntryConstants.PathSet); JsonTree.EmancipateBranch(pathSetNode); JsonNode currPathNode = null; JsonNode currFlagNode = null; JsonNode currRegexNode = null; var observedInputIt = observedInputsNode.Children.First; for (var it = pathSetNode.Children.First; it != null; it = pathSetNode.Children.First) { var child = it.Value; switch (child.Name) { case ObservedPathEntryConstants.Path: currPathNode = child; // Switch from literal string "path" to actual file system path // [5'] "B:/out/objects/n/x/qbkexxlc8je93wycw7yrlw0a305n7k/xunit-out/CacheMissAnaAD836B23/3/obj/readonly/src_0":"" currPathNode.Name = currPathNode.Values[0]; // The name captures the node's value, so clear the values to avoid extraneous value comparison when diffing currPathNode.Values.Clear(); JsonTree.ReparentBranch(currPathNode, parentPathNode); // [6'] "Flags":"IsDirectoryPath, DirectoryEnumeration, DirectoryEnumerationWithAllPattern" JsonTree.ReparentBranch(currFlagNode, currPathNode); // [7'] "EnumeratePatternRegex":"^.*$" JsonTree.ReparentBranch(currRegexNode, currPathNode); // [3'] "ObservedInput":"E:VSO0:E0C5007DC8CF2D331236F156F136C50CACE2A5D549CD132D9B44ABD1F13D50CC00" // [8] "Members":"[src_1, src_2]" ReparentObservedInput(observedInputIt.Value, currPathNode); observedInputIt = observedInputsNode.Children.First; break; case ObservedPathEntryConstants.Flags: // [6] "Flags":"IsDirectoryPath, DirectoryEnumeration, DirectoryEnumerationWithAllPattern" currFlagNode = child; JsonTree.EmancipateBranch(currFlagNode); break; case ObservedPathEntryConstants.EnumeratePatternRegex: // [7] "EnumeratePatternRegex":"^.*$" currRegexNode = child; JsonTree.EmancipateBranch(currRegexNode); break; default: break; } } // Re-parent any other branches of the path set tree to the strong fingerprint tree // so they are still in a full strong fingerprint tree comparison. // We re-parent under parentPathNode because branches of pathSetTree are elements of PathSet var node = pathSetTree.Children.First; while (node != null) { JsonTree.ReparentBranch(node.Value, parentPathNode); node = pathSetTree.Children.First; } return(strongFingerprintTree); }
private static JProperty DiffObservedPaths( JsonNode pathSetTree, JsonNode strongFingerprintInputTree, JsonNode otherPathSetTree, JsonNode otherStrongFingerprintInputTree, Func <string, IReadOnlyList <string> > getDirectoryMembership, Func <string, IReadOnlyList <string> > getOtherDirectoryMembership) { JsonNode pathsNode = JsonTree.FindNodeByName(pathSetTree, ObservedPathSet.Labels.Paths); JsonNode otherPathsNode = JsonTree.FindNodeByName(otherPathSetTree, ObservedPathSet.Labels.Paths); JsonNode observedInputsTree = JsonTree.FindNodeByName(strongFingerprintInputTree, ObservedInputConstants.ObservedInputs); JsonNode otherObservedInputsTree = JsonTree.FindNodeByName(otherStrongFingerprintInputTree, ObservedInputConstants.ObservedInputs); using (var pathSetDataPool = ObservedInputDataMapPool.GetInstance()) using (var otherPathSetDataPool = ObservedInputDataMapPool.GetInstance()) { var pathSetData = pathSetDataPool.Instance; var otherPathSetData = otherPathSetDataPool.Instance; traversePathSetPaths(pathsNode, observedInputsTree, pathSetData); traversePathSetPaths(otherPathsNode, otherObservedInputsTree, otherPathSetData); bool hasDiff = ExtractUnorderedMapDiff( pathSetData, otherPathSetData, (data, otherData) => data.Equals(otherData), out var added, out var removed, out var changed); if (hasDiff) { // { // Paths: { // Added : [..paths..], // Removed: [..paths..], // Changed: { // path: { Old: ..., New: ... } // }: // } // } return(new JProperty( ObservedPathSet.Labels.Paths, RenderUnorderedMapDiff( pathSetData, otherPathSetData, added, removed, changed, RenderPath, (dataA, dataB) => dataA.DescribeDiffWithoutPath(dataB), c => diffDirectoryIfApplicable(pathSetData, otherPathSetData, c)))); } return(null); } JProperty diffDirectoryIfApplicable(Dictionary <string, ObservedInputData> obsInputData, Dictionary <string, ObservedInputData> otherObsInputData, string possiblyChangeDirectory) { // { // Members: { // Added : [..file..], // Removed : [..file..] // } // } const string MembersLabel = "Members"; var change = obsInputData[possiblyChangeDirectory]; var otherChange = otherObsInputData[possiblyChangeDirectory]; if (change.AccessType == ObservedInputConstants.DirectoryEnumeration && otherChange.AccessType == ObservedInputConstants.DirectoryEnumeration && change.Pattern == otherChange.Pattern) { var members = getDirectoryMembership(change.Hash); if (members == null) { return(new JProperty(MembersLabel, $"{CacheMissAnalysisUtilities.RepeatedStrings.MissingDirectoryMembershipFingerprint} ({nameof(ObservedInputData.Hash)}: {change.Hash})")); } var otherMembers = getOtherDirectoryMembership(otherChange.Hash); if (otherMembers == null) { return(new JProperty(MembersLabel, $"{CacheMissAnalysisUtilities.RepeatedStrings.MissingDirectoryMembershipFingerprint} ({nameof(ObservedInputData.Hash)}: {otherChange.Hash})")); } bool hasDiff = ExtractUnorderedListDiff(members, otherMembers, out var addedMembers, out var removedMembers); if (hasDiff) { return(new JProperty(MembersLabel, RenderUnorderedListDiff(addedMembers, removedMembers, RenderPath))); } } return(null); } void traversePathSetPaths( JsonNode pathSetTree, JsonNode strongFingerprintInputTree, Dictionary <string, ObservedInputData> populatedData) { TraversePathSetPaths(pathSetTree, strongFingerprintInputTree, data => populatedData[data.Path] = data); } }
/// <summary> /// Diffs strong fingerprints. /// </summary> /// <param name="pathSetHash">Pathset hash.</param> /// <param name="pathSetTree">Pathset tree.</param> /// <param name="strongFingerprintInputTree">Strong fingerprint input tree.</param> /// <param name="otherPathSetHash">Other pathset hash.</param> /// <param name="otherPathSetTree">Other pathset tree.</param> /// <param name="otherStrongFingerprintInputTree">Other strong fingerprint input tree.</param> /// <param name="getDirectoryMembership">Delegate for getting directory membership.</param> /// <param name="getOtherDirectoryMembership">Delegate for getting other directory membership.</param> /// <returns></returns> public static JObject DiffPathSets( string pathSetHash, JsonNode pathSetTree, JsonNode strongFingerprintInputTree, string otherPathSetHash, JsonNode otherPathSetTree, JsonNode otherStrongFingerprintInputTree, Func <string, IReadOnlyList <string> > getDirectoryMembership, Func <string, IReadOnlyList <string> > getOtherDirectoryMembership) { JObject result = new JObject(); if (pathSetHash == otherPathSetHash) { return(result); } // { // PathSetHash: { Old: old_path_set_hash, New: new_path_set_hash } // } AddPropertyIfNotNull(result, RenderSingleValueDiff("PathSetHash", pathSetHash, otherPathSetHash)); JsonNode unsafeOptionsNode = JsonTree.FindNodeByName(pathSetTree, ObservedPathSet.Labels.UnsafeOptions); JsonNode otherUnsafeOptionsNode = JsonTree.FindNodeByName(otherPathSetTree, ObservedPathSet.Labels.UnsafeOptions); // { // UnsafeOptions: // { // <property_Name>: // { // Old: old_value, // New: new_value // } // PreserveOutputInfo: // { // <property_Name>: // { // Old: old_value, // New: new_value // } // } // } AddPropertyIfNotNull(result, DiffUnsafeOptions(unsafeOptionsNode, otherUnsafeOptionsNode)); AddPropertyIfNotNull( result, DiffObservedPaths( pathSetTree, strongFingerprintInputTree, otherPathSetTree, otherStrongFingerprintInputTree, getDirectoryMembership, getOtherDirectoryMembership)); JsonNode obsFileNameNode = JsonTree.FindNodeByName(pathSetTree, ObservedPathSet.Labels.ObservedAccessedFileNames); JsonNode otherObsFileNameNode = JsonTree.FindNodeByName(otherPathSetTree, ObservedPathSet.Labels.ObservedAccessedFileNames); bool hasDiff = ExtractUnorderedListDiff(obsFileNameNode.Values, otherObsFileNameNode.Values, out var addedFileNames, out var removedFileName); if (hasDiff) { result.Add(new JProperty( ObservedPathSet.Labels.ObservedAccessedFileNames, RenderUnorderedListDiff(addedFileNames, removedFileName, RenderPath))); } Contract.Assert(result.Count > 0); return(result); }