private static JProperty DiffUnsafeOptions(JsonNode unsafeOptionsNode, JsonNode otherUnsafeOptionsNode)
{
Contract.Requires(unsafeOptionsNode != null);
Contract.Requires(otherUnsafeOptionsNode != null);
Contract.Requires(unsafeOptionsNode.Name == otherUnsafeOptionsNode.Name);
// Get the diff result of the single values in unsafeUnsafeOptions
JObject result = DiffNameValuePairs(unsafeOptionsNode, otherUnsafeOptionsNode);
// Deal with the preserveOutputInfo struct in unsafeUnsafeOptions
JsonNode preserveOutputInfoNode = JsonTree.FindNodeByName(unsafeOptionsNode, nameof(PreserveOutputsInfo));
JsonNode otherpreserveOutputInfoNode = JsonTree.FindNodeByName(otherUnsafeOptionsNode, nameof(PreserveOutputsInfo));
if (preserveOutputInfoNode != null || otherpreserveOutputInfoNode != null)
{
JObject preserveOutputInfoResult = DiffNameValuePairs(
preserveOutputInfoNode ?? new JsonNode()
{
Name = nameof(PreserveOutputsInfo)
},
otherpreserveOutputInfoNode ?? new JsonNode()
{
Name = nameof(PreserveOutputsInfo)
});
if (preserveOutputInfoResult != null)
{
result = result ?? new JObject();
AddPropertyIfNotNull(result, new JProperty(nameof(PreserveOutputsInfo), preserveOutputInfoResult));
}
}
return(result != null ? new JProperty(unsafeOptionsNode.Name, result) : null);
}
/// <summary>
/// Diffs strong fingerprints.
/// </summary>
/// <param name="pathSetHash">Pathset hash.</param>
/// <param name="pathSetTree">Pathset tree.</param>
/// <param name="strongFingerprintInputTree">Strong fingerprint input tree.</param>
/// <param name="otherPathSetHash">Other pathset hash.</param>
/// <param name="otherPathSetTree">Other pathset tree.</param>
/// <param name="otherStrongFingerprintInputTree">Other strong fingerprint input tree.</param>
/// <param name="getDirectoryMembership">Delegate for getting directory membership.</param>
/// <param name="getOtherDirectoryMembership">Delegate for getting other directory membership.</param>
/// <returns></returns>
public static JObject DiffPathSets(
string pathSetHash,
JsonNode pathSetTree,
JsonNode strongFingerprintInputTree,
string otherPathSetHash,
JsonNode otherPathSetTree,
JsonNode otherStrongFingerprintInputTree,
Func <string, IReadOnlyList <string> > getDirectoryMembership,
Func <string, IReadOnlyList <string> > getOtherDirectoryMembership)
{
JObject result = new JObject();
if (pathSetHash == otherPathSetHash)
{
return(result);
}
// {
// PathSetHash: { Old: old_path_set_hash, New: new_path_set_hash }
// }
AddPropertyIfNotNull(result, RenderSingleValueDiff("PathSetHash", pathSetHash, otherPathSetHash));
JsonNode unsafeOptionsNode = JsonTree.FindNodeByName(pathSetTree, ObservedPathSet.Labels.UnsafeOptions);
JsonNode otherUnsafeOptionsNode = JsonTree.FindNodeByName(otherPathSetTree, ObservedPathSet.Labels.UnsafeOptions);
// This is less ideal because we can't see the difference.
// TODO: dump unsafe option data to the fingerprint store so that we can analyze the content.
// {
// UnsafeOptions: { Old: old_bits, New: new_bits: }
// }
AddPropertyIfNotNull(result, RenderSingleValueDiff(ObservedPathSet.Labels.UnsafeOptions, unsafeOptionsNode.Values[0], otherUnsafeOptionsNode.Values[0]));
AddPropertyIfNotNull(
result,
DiffObservedPaths(
pathSetTree,
strongFingerprintInputTree,
otherPathSetTree,
otherStrongFingerprintInputTree,
getDirectoryMembership,
getOtherDirectoryMembership));
JsonNode obsFileNameNode = JsonTree.FindNodeByName(pathSetTree, ObservedPathSet.Labels.ObservedAccessedFileNames);
JsonNode otherObsFileNameNode = JsonTree.FindNodeByName(otherPathSetTree, ObservedPathSet.Labels.ObservedAccessedFileNames);
bool hasDiff = ExtractUnorderedListDiff(obsFileNameNode.Values, otherObsFileNameNode.Values, out var addedFileNames, out var removedFileName);
if (hasDiff)
{
result.Add(new JProperty(
ObservedPathSet.Labels.ObservedAccessedFileNames,
RenderUnorderedListDiff(addedFileNames, removedFileName, RenderPath)));
}
Contract.Assert(result.Count > 0);
return(result);
}
public void NestedNamelessArraysTest()
{
var jsonA = "{\"key\":[\"value\", [\"nestedValue\"]]}";
var treeA = JsonTree.Deserialize(jsonA);
var keyNode = JsonTree.FindNodeByName(treeA, "key");
XAssert.IsTrue(keyNode.Values.Contains("value") && keyNode.Values.Contains("nestedValue"));
string deserialized = JsonTree.Serialize(treeA);
XAssert.IsTrue(deserialized.Contains("value") && deserialized.Contains("nestedValue"));
XAssert.AreNotEqual(JsonTree.PrettyPrintJson(jsonA), deserialized);
}
/// <summary>
/// Path set hash inputs are stored separately from the strong fingerprint inputs.
/// This merges the path set hash inputs tree into the strong fingerprint inputs tree
/// while maintaining the 1:1 relationship between the path set and observed inputs.
///
/// Node notation:
/// [id] "{name}":"{value}"
/// Tree notation:
/// {parentNode}
/// {childNode}
///
/// Start with the following subtrees:
///
/// From strong fingerprint
///
/// [1] "PathSet":"VSO0:7E2E49845EC0AE7413519E3EE605272078AF0B1C2911C021681D1D9197CC134A00"
/// [2] "ObservedInputs":""
/// [3] "E":"VSO0:E0C5007DC8CF2D331236F156F136C50CACE2A5D549CD132D9B44ABD1F13D50CC00",
///
/// From path set hash
///
/// [4] "PathSet":""
/// [5] "Path":"B:/out/objects/n/x/qbkexxlc8je93wycw7yrlw0a305n7k/xunit-out/CacheMissAnaAD836B23/3/obj/readonly/src_0"
/// [6] "Flags":"IsDirectoryPath, DirectoryEnumeration, DirectoryEnumerationWithAllPattern"
/// [7] "EnumeratePatternRegex":"^.*$"
///
/// And end with:
///
/// [1] "PathSet":"VSO0:7E2E49845EC0AE7413519E3EE605272078AF0B1C2911C021681D1D9197CC134A00"
/// [5'] "B:/out/objects/n/x/qbkexxlc8je93wycw7yrlw0a305n7k/xunit-out/CacheMissAnaAD836B23/3/obj/readonly/src_0":""
/// [6'] "Flags":"IsDirectoryPath, DirectoryEnumeration, DirectoryEnumerationWithAllPattern"
/// [7'] "EnumeratePatternRegex":"^.*$"
/// [3'] "ObservedInput":"E:VSO0:E0C5007DC8CF2D331236F156F136C50CACE2A5D549CD132D9B44ABD1F13D50CC00"
/// [8] "Members":"[src_1, src_2]"
/// </summary>
/// <returns>
/// The root node of the merged tree (which will be the strong fingerprint tree's root).
/// </returns>
private JsonNode MergeStrongFingerprintAndPathSetTrees(JsonNode strongFingerprintTree, JsonNode pathSetTree)
{
// [1] "PathSet":"VSO0:7E2E49845EC0AE7413519E3EE605272078AF0B1C2911C021681D1D9197CC134A00")
var parentPathNode = JsonTree.FindNodeByName(strongFingerprintTree, ObservedPathEntryConstants.PathSet);
// [2] "ObservedInputs":""
var observedInputsNode = JsonTree.FindNodeByName(strongFingerprintTree, ObservedInputConstants.ObservedInputs);
JsonTree.EmancipateBranch(observedInputsNode);
// In preparation for merging with observed inputs nodes,
// remove the path set node's branch from the path set tree
// [4] "PathSet":""
var pathSetNode = JsonTree.FindNodeByName(pathSetTree, ObservedPathEntryConstants.PathSet);
JsonTree.EmancipateBranch(pathSetNode);
JsonNode currPathNode = null;
JsonNode currFlagNode = null;
JsonNode currRegexNode = null;
var observedInputIt = observedInputsNode.Children.First;
for (var it = pathSetNode.Children.First; it != null; it = pathSetNode.Children.First)
{
var child = it.Value;
switch (child.Name)
{
case ObservedPathEntryConstants.Path:
currPathNode = child;
// Switch from literal string "path" to actual file system path
// [5'] "B:/out/objects/n/x/qbkexxlc8je93wycw7yrlw0a305n7k/xunit-out/CacheMissAnaAD836B23/3/obj/readonly/src_0":""
currPathNode.Name = currPathNode.Values[0];
// The name captures the node's value, so clear the values to avoid extraneous value comparison when diffing
currPathNode.Values.Clear();
JsonTree.ReparentBranch(currPathNode, parentPathNode);
// [6'] "Flags":"IsDirectoryPath, DirectoryEnumeration, DirectoryEnumerationWithAllPattern"
JsonTree.ReparentBranch(currFlagNode, currPathNode);
// [7'] "EnumeratePatternRegex":"^.*$"
JsonTree.ReparentBranch(currRegexNode, currPathNode);
// [3'] "ObservedInput":"E:VSO0:E0C5007DC8CF2D331236F156F136C50CACE2A5D549CD132D9B44ABD1F13D50CC00"
// [8] "Members":"[src_1, src_2]"
ReparentObservedInput(observedInputIt.Value, currPathNode);
observedInputIt = observedInputsNode.Children.First;
break;
case ObservedPathEntryConstants.Flags:
// [6] "Flags":"IsDirectoryPath, DirectoryEnumeration, DirectoryEnumerationWithAllPattern"
currFlagNode = child;
JsonTree.EmancipateBranch(currFlagNode);
break;
case ObservedPathEntryConstants.EnumeratePatternRegex:
// [7] "EnumeratePatternRegex":"^.*$"
currRegexNode = child;
JsonTree.EmancipateBranch(currRegexNode);
break;
default:
break;
}
}
// Re-parent any other branches of the path set tree to the strong fingerprint tree
// so they are still in a full strong fingerprint tree comparison.
// We re-parent under parentPathNode because branches of pathSetTree are elements of PathSet
var node = pathSetTree.Children.First;
while (node != null)
{
JsonTree.ReparentBranch(node.Value, parentPathNode);
node = pathSetTree.Children.First;
}
return(strongFingerprintTree);
}
private static JProperty DiffObservedPaths(
JsonNode pathSetTree,
JsonNode strongFingerprintInputTree,
JsonNode otherPathSetTree,
JsonNode otherStrongFingerprintInputTree,
Func <string, IReadOnlyList <string> > getDirectoryMembership,
Func <string, IReadOnlyList <string> > getOtherDirectoryMembership)
{
JsonNode pathsNode = JsonTree.FindNodeByName(pathSetTree, ObservedPathSet.Labels.Paths);
JsonNode otherPathsNode = JsonTree.FindNodeByName(otherPathSetTree, ObservedPathSet.Labels.Paths);
JsonNode observedInputsTree = JsonTree.FindNodeByName(strongFingerprintInputTree, ObservedInputConstants.ObservedInputs);
JsonNode otherObservedInputsTree = JsonTree.FindNodeByName(otherStrongFingerprintInputTree, ObservedInputConstants.ObservedInputs);
using (var pathSetDataPool = ObservedInputDataMapPool.GetInstance())
using (var otherPathSetDataPool = ObservedInputDataMapPool.GetInstance())
{
var pathSetData = pathSetDataPool.Instance;
var otherPathSetData = otherPathSetDataPool.Instance;
traversePathSetPaths(pathsNode, observedInputsTree, pathSetData);
traversePathSetPaths(otherPathsNode, otherObservedInputsTree, otherPathSetData);
bool hasDiff = ExtractUnorderedMapDiff(
pathSetData,
otherPathSetData,
(data, otherData) => data.Equals(otherData),
out var added,
out var removed,
out var changed);
if (hasDiff)
{
// {
// Paths: {
// Added : [..paths..],
// Removed: [..paths..],
// Changed: {
// path: { Old: ..., New: ... }
// }:
// }
// }
return(new JProperty(
ObservedPathSet.Labels.Paths,
RenderUnorderedMapDiff(
pathSetData,
otherPathSetData,
added,
removed,
changed,
RenderPath,
(dataA, dataB) => dataA.DescribeDiffWithoutPath(dataB),
c => diffDirectoryIfApplicable(pathSetData, otherPathSetData, c))));
}
return(null);
}
JProperty diffDirectoryIfApplicable(Dictionary <string, ObservedInputData> obsInputData, Dictionary <string, ObservedInputData> otherObsInputData, string possiblyChangeDirectory)
{
// {
// Members: {
// Added : [..file..],
// Removed : [..file..]
// }
// }
const string MembersLabel = "Members";
var change = obsInputData[possiblyChangeDirectory];
var otherChange = otherObsInputData[possiblyChangeDirectory];
if (change.AccessType == ObservedInputConstants.DirectoryEnumeration &&
otherChange.AccessType == ObservedInputConstants.DirectoryEnumeration &&
change.Pattern == otherChange.Pattern)
{
var members = getDirectoryMembership(change.Hash);
if (members == null)
{
return(new JProperty(MembersLabel, $"{CacheMissAnalysisUtilities.RepeatedStrings.MissingDirectoryMembershipFingerprint} ({nameof(ObservedInputData.Hash)}: {change.Hash})"));
}
var otherMembers = getOtherDirectoryMembership(otherChange.Hash);
if (otherMembers == null)
{
return(new JProperty(MembersLabel, $"{CacheMissAnalysisUtilities.RepeatedStrings.MissingDirectoryMembershipFingerprint} ({nameof(ObservedInputData.Hash)}: {otherChange.Hash})"));
}
bool hasDiff = ExtractUnorderedListDiff(members, otherMembers, out var addedMembers, out var removedMembers);
if (hasDiff)
{
return(new JProperty(MembersLabel, RenderUnorderedListDiff(addedMembers, removedMembers, RenderPath)));
}
}
return(null);
}
void traversePathSetPaths(
JsonNode pathSetTree,
JsonNode strongFingerprintInputTree,
Dictionary <string, ObservedInputData> populatedData)
{
TraversePathSetPaths(pathSetTree, strongFingerprintInputTree, data => populatedData[data.Path] = data);
}
}
/// <summary>
/// Diffs strong fingerprints.
/// </summary>
/// <param name="pathSetHash">Pathset hash.</param>
/// <param name="pathSetTree">Pathset tree.</param>
/// <param name="strongFingerprintInputTree">Strong fingerprint input tree.</param>
/// <param name="otherPathSetHash">Other pathset hash.</param>
/// <param name="otherPathSetTree">Other pathset tree.</param>
/// <param name="otherStrongFingerprintInputTree">Other strong fingerprint input tree.</param>
/// <param name="getDirectoryMembership">Delegate for getting directory membership.</param>
/// <param name="getOtherDirectoryMembership">Delegate for getting other directory membership.</param>
/// <returns></returns>
public static JObject DiffPathSets(
string pathSetHash,
JsonNode pathSetTree,
JsonNode strongFingerprintInputTree,
string otherPathSetHash,
JsonNode otherPathSetTree,
JsonNode otherStrongFingerprintInputTree,
Func <string, IReadOnlyList <string> > getDirectoryMembership,
Func <string, IReadOnlyList <string> > getOtherDirectoryMembership)
{
JObject result = new JObject();
if (pathSetHash == otherPathSetHash)
{
return(result);
}
// {
// PathSetHash: { Old: old_path_set_hash, New: new_path_set_hash }
// }
AddPropertyIfNotNull(result, RenderSingleValueDiff("PathSetHash", pathSetHash, otherPathSetHash));
JsonNode unsafeOptionsNode = JsonTree.FindNodeByName(pathSetTree, ObservedPathSet.Labels.UnsafeOptions);
JsonNode otherUnsafeOptionsNode = JsonTree.FindNodeByName(otherPathSetTree, ObservedPathSet.Labels.UnsafeOptions);
// {
// UnsafeOptions:
// {
// <property_Name>:
// {
// Old: old_value,
// New: new_value
// }
// PreserveOutputInfo:
// {
// <property_Name>:
// {
// Old: old_value,
// New: new_value
// }
// }
// }
AddPropertyIfNotNull(result, DiffUnsafeOptions(unsafeOptionsNode, otherUnsafeOptionsNode));
AddPropertyIfNotNull(
result,
DiffObservedPaths(
pathSetTree,
strongFingerprintInputTree,
otherPathSetTree,
otherStrongFingerprintInputTree,
getDirectoryMembership,
getOtherDirectoryMembership));
JsonNode obsFileNameNode = JsonTree.FindNodeByName(pathSetTree, ObservedPathSet.Labels.ObservedAccessedFileNames);
JsonNode otherObsFileNameNode = JsonTree.FindNodeByName(otherPathSetTree, ObservedPathSet.Labels.ObservedAccessedFileNames);
bool hasDiff = ExtractUnorderedListDiff(obsFileNameNode.Values, otherObsFileNameNode.Values, out var addedFileNames, out var removedFileName);
if (hasDiff)
{
result.Add(new JProperty(
ObservedPathSet.Labels.ObservedAccessedFileNames,
RenderUnorderedListDiff(addedFileNames, removedFileName, RenderPath)));
}
Contract.Assert(result.Count > 0);
return(result);
}