public static JsonMapNode FindByPath(this JsonMapNode node, IReadOnlyList <string> jsonPath, bool throwIfNotFound = true)
{
JsonMapNode current = node;
for (int i = 0; i < jsonPath.Count; ++i)
{
if (current.Nodes == null)
{
if (throwIfNotFound)
{
throw new ArgumentException($"Under \"{BuildJsonPath(jsonPath, i - 1)}\", no nodes were included in map to find \"{jsonPath[i]}\"");
}
return(null);
}
if (!current.Nodes.TryGetValue(jsonPath[i], out JsonMapNode child))
{
if (throwIfNotFound)
{
throw new ArgumentException($"Under \"{BuildJsonPath(jsonPath, i - 1)}\", node \"{jsonPath[i]}\" was not found in map. Available Nodes: \r\n{String.Join("\r\n", current.Nodes.Keys)}");
}
return(null);
}
current = child;
}
return(current);
}
static void WriteTreeToDepth(JsonMapNode current, int depthLimit, string name, TextWriter writer, int indent = 0)
{
// At root, ...
if (indent == 0)
{
// Find the desired start element if a json path was passed
current = current.FindByPath(name);
// Write the column names for subsequent output
writer.WriteLine($"{Pad("Name", 40)} {PadRight("Count", 15)} {PadRight("Bytes", 15)}");
// Call this element 'Root' in the output
if (String.IsNullOrEmpty(name))
{
name = "[Root]";
}
}
// Write basics for this element
writer.WriteLine($"{Pad(new string(' ', indent * 2) + name, 40)} {PadRight(current.Count.ToString("n0"), 15)} {PadRight(current.Length.ToString("n0"), 15)}");
// Recurse for children to depth limit
if (current.Nodes != null && (indent < depthLimit || depthLimit < 0))
{
foreach (KeyValuePair <string, JsonMapNode> child in current.Nodes.OrderByDescending(kvp => kvp.Value.Length))
{
WriteTreeToDepth(child.Value, depthLimit, child.Key, writer, indent + 1);
}
}
}
public void RunWithoutCatch(PageOptions options)
{
if (options.RunIndex < 0)
{
throw new ArgumentOutOfRangeException("runIndex");
}
if (options.Index < 0)
{
throw new ArgumentOutOfRangeException("index");
}
if (options.Count < 0)
{
throw new ArgumentOutOfRangeException("count");
}
if (!_fileSystem.FileExists(options.InputFilePath))
{
throw new FileNotFoundException($"Input file \"{options.InputFilePath}\" not found.");
}
if (options.Force == false && _fileSystem.FileExists(options.OutputFilePath))
{
Console.WriteLine($"Output file \"{options.OutputFilePath}\" already exists. Stopping.");
return;
}
// Load the JsonMap, if previously built and up-to-date, or rebuild it
JsonMapNode root = LoadOrRebuildMap(options);
// Write the desired page from the Sarif file
ExtractPage(options, root);
}
static void WriteArrayEntryLengths(JsonMapNode root, string jsonPath, int threshold, TextWriter writer)
{
JsonMapNode current = root.FindByPath(jsonPath);
int countWritten = 0;
long totalLength = 0;
if (current.ArrayStarts == null)
{
throw new ArgumentException($"No ArrayStarts in map for node at \"{jsonPath}\".");
}
if (current.Every != 1)
{
writer.WriteLine($"Warning: Array Starts only included for every {current.Every} elements; true count {current.Count:n0}");
}
writer.WriteLine($"{jsonPath} is {ToSizeString(current.Length)} and has {current.Count:n0} elements with lengths:");
// ArrayStarts are converted to absolute in JsonMapNode
for (int i = 1; i < current.ArrayStarts.Count; ++i)
{
// Difference between [i] and [i - 1] is the length of [i - 1]
long length = current.ArrayStarts[i] - current.ArrayStarts[i - 1];
if (length > threshold)
{
writer.WriteLine($"[{(i - 1) * current.Every}] => {length:n0}");
countWritten++;
totalLength += length;
}
}
// Last element length is approximately from the last start to the array end
if (current.Count > 0 && current.Every == 1)
{
long lastLength = current.End - current.ArrayStarts[current.ArrayStarts.Count - 1] - 1;
if (lastLength > threshold)
{
writer.WriteLine($"[{current.Count - 1}] => {lastLength:n0}");
countWritten++;
totalLength += lastLength;
}
}
if (threshold > 0)
{
double containerPortion = totalLength / (double)root.Length;
writer.WriteLine($"Elements with length >= {threshold:n0} bytes:");
writer.WriteLine($" {countWritten:n0} / {current.Count:n0} of elements ({(countWritten / (double)current.Count):p0}).");
writer.WriteLine($" {totalLength:n0} / {current.Length:n0} bytes ({(totalLength / (double)current.Length):p0}).");
}
}
static void Build(string jsonFilePath, double ratio = 0.01)
{
string mapPath = Path.ChangeExtension(jsonFilePath, ".map.json");
Console.WriteLine($"Building {ratio:p1} map of \"{jsonFilePath}\"...");
Stopwatch w = Stopwatch.StartNew();
JsonMapNode root = JsonMapBuilder.Build(jsonFilePath, new JsonMapSettings(ratio, (10 * JsonMapSettings.Megabyte) * (ratio / 0.01)));
File.WriteAllText(mapPath, JsonConvert.SerializeObject(root, Formatting.None));
Console.WriteLine($"Done in {w.Elapsed.TotalSeconds:n1}s; map is {ToSizeString(new FileInfo(mapPath).Length)}");
Console.WriteLine();
}
public int RunWithoutCatch(PageOptions options)
{
if (!ValidateOptions(options, _fileSystem))
{
return(1);
}
// Load the JsonMap, if previously built and up-to-date, or rebuild it
JsonMapNode root = LoadOrRebuildMap(options);
// Write the desired page from the Sarif file
ExtractPage(options, root);
return(SUCCESS);
}
static bool IsOverThreshold(JsonMapNode parent, JsonMapNode child, double threshold)
{
if (threshold <= 0)
{
return(true);
}
else if (threshold <= 1)
{
// Threshold under 1: Percentage of parent
return(child.Length >= parent.Length * threshold);
}
else
{
// Threshold over 1: Absolute size
return(child.Length >= threshold);
}
}
static void Extract(string jsonFilePath, JsonMapNode root, string jsonPath, string outputPath)
{
JsonMapNode current = root.FindByPath(jsonPath);
Console.WriteLine($"Extracting \"{jsonPath}\" from \"{jsonFilePath}\" into \"{outputPath}\"...");
byte[] buffer = new byte[64 * 1024];
long lengthToCopy = current.Length;
using (FileStream source = File.OpenRead(jsonFilePath))
using (FileStream target = File.Create(outputPath))
{
source.Seek(current.Start, SeekOrigin.Begin);
while (lengthToCopy > 0)
{
int lengthRead = source.Read(buffer, 0, (int)Math.Min(buffer.Length, lengthToCopy));
target.Write(buffer, 0, lengthRead);
lengthToCopy -= lengthRead;
}
}
Console.WriteLine($"Done. {ToSizeString(current.Length)} extracted.");
}
private void ExtractPage(PageOptions options, JsonMapNode root)
{
Stopwatch w = Stopwatch.StartNew();
Console.WriteLine($"Extracting {options.Count:n0} results from index {options.Index:n0}\r\n from \"{options.InputFilePath}\"\r\n into \"{options.OutputFilePath}\"...");
JsonMapNode runs, run, results;
// Get 'runs' node from map. If log was too small, page using the object model
if (root == null ||
root.Nodes == null ||
root.Nodes.TryGetValue("runs", out runs) == false)
{
PageViaOm(options);
return;
}
// Verify RunIndex in range
if (options.RunIndex >= runs.Count)
{
throw new ArgumentOutOfRangeException($"Page requested for RunIndex {options.RunIndex}, but Log had only {runs.Count} runs.");
}
// Get 'results' from map. If log was too small, page using the object model
if (!runs.Nodes.TryGetValue(options.RunIndex.ToString(), out run) ||
run.Nodes == null ||
run.Nodes.TryGetValue("results", out results) == false ||
results.ArrayStarts == null)
{
// Log too small; convert via OM
PageViaOm(options);
return;
}
if (options.Index >= results.Count)
{
throw new ArgumentOutOfRangeException($"Index requested was {options.Index} but Run has only {results.Count} results.");
}
if (options.Index + options.Count > results.Count)
{
Console.WriteLine($"Page requested from Result {options.Index} to {options.Index + options.Count} but Run has only {results.Count} results.");
options.Count = results.Count - options.Index;
}
Console.WriteLine($"Run {options.RunIndex} in \"{options.InputFilePath}\" has {results.Count:n0} results.");
Func <Stream> inputStreamProvider = () => _fileSystem.FileOpenRead(options.InputFilePath);
long firstResultStart = results.FindArrayStart(options.Index, inputStreamProvider);
long lastResultEnd = results.FindArrayStart(options.Index + options.Count, inputStreamProvider) - 1;
// Ensure output directory exists
string outputFolder = Path.GetDirectoryName(Path.GetFullPath(options.OutputFilePath));
Directory.CreateDirectory(outputFolder);
// Build the Sarif Log subset
long lengthWritten = 0;
byte[] buffer = new byte[64 * 1024];
using (Stream output = _fileSystem.FileCreate(options.OutputFilePath))
using (Stream source = _fileSystem.FileOpenRead(options.InputFilePath))
{
// Copy everything up to 'runs' (includes the '[')
JsonMapNode.CopyStreamBytes(source, output, 0, runs.Start, buffer);
// In the run, copy everything to 'results' (includes the '[')
JsonMapNode.CopyStreamBytes(source, output, run.Start, results.Start, buffer);
// Find and copy the desired range of results, excluding the last ','
JsonMapNode.CopyStreamBytes(source, output, firstResultStart, lastResultEnd, buffer, omitFromLast: (byte)',');
// Copy everything after the results array to the end of the run (includes the '}')
JsonMapNode.CopyStreamBytes(source, output, results.End, run.End, buffer);
// Omit all subsequent runs
// Copy everything after all runs (includes runs ']' and log '}')
JsonMapNode.CopyStreamBytes(source, output, runs.End, root.End, buffer);
lengthWritten = output.Length;
}
w.Stop();
Console.WriteLine($"Done; wrote {(lengthWritten / (double)(1024 * 1024)):n2} MB in {w.Elapsed.TotalSeconds:n1}s.");
}
public static JsonMapNode FindByPath(this JsonMapNode node, string jsonPath, bool throwIfNotFound = true)
{
return(FindByPath(node, SplitJsonPath(jsonPath), throwIfNotFound));
}
static void WriteBasics(JsonMapNode root, string jsonPath, TextWriter writer)
{
JsonMapNode current = root.FindByPath(jsonPath);
writer.WriteLine($"{jsonPath} is {ToSizeString(current.Length)} and has {current.Count:n0} elements");
}
static void Main(string[] args)
{
try
{
if (args.Length < 2)
{
Console.WriteLine(@"Usage: Map <jsonFilePath> <mode> [args]
Build <ratio> - Build JSON map from source document; file path is document, not map
Basics <jsonPath?> - Show size and element count under item. (over length, if included)
ArrayLengths <jsonPath?> <over?> - Show the size of each element in an array (reveal outliers)
Tree <threshold?> <jsonPath?> - Show tree of elements over threshold (% of parent or absolute size)
TreeToDepth <depth> <jsonPath?> - Show tree to a depth limit
Extract <jsonPath> <toFile> - Write element at JsonPath to a new file
Indent <toFile> - Write indented copy of file
Minify <toFile> - Write minified copy of file
Convert <fromFile> <toFile> - Convert to/from JSON, BSON
Parse <filePath> - Parse JSON/BSON files and time
Consolidate <toFile> - Write SarifConsolidator-compressed copy of file
LoadSarif <filePath> - Load into SARIF OM from JSON/BSON
");
return;
}
string jsonFilePath = args[0];
string mapFilePath = Path.ChangeExtension(jsonFilePath, ".map.json");
string mode = args[1];
// Handle modes which don't use a file map
switch (mode.ToLowerInvariant())
{
case "indent":
JsonIndent(
jsonFilePath,
(args.Length > 2 ? args[2] : Path.ChangeExtension(jsonFilePath, ".indent.json")));
return;
case "minify":
JsonMinify(
jsonFilePath,
(args.Length > 2 ? args[2] : Path.ChangeExtension(jsonFilePath, ".min.json")));
return;
case "convert":
Convert(
jsonFilePath,
(args.Length > 2 ? args[2] : Path.ChangeExtension(jsonFilePath, ".bson")));
return;
case "consolidate":
Consolidate(
jsonFilePath,
(args.Length > 2 ? args[2] : Path.ChangeExtension(jsonFilePath, ".trim.sarif")));
return;
case "parse":
Parse(jsonFilePath);
return;
case "loadsarif":
LoadSarif(jsonFilePath);
return;
case "build":
Build(jsonFilePath, (args.Length > 2 ? double.Parse(args[2]) : 0.01));
return;
}
// Build map if it is outdated or missing
if (!File.Exists(mapFilePath) || (File.GetLastWriteTimeUtc(jsonFilePath) > File.GetLastWriteTimeUtc(mapFilePath)))
{
Build(jsonFilePath);
}
// Load map
JsonMapNode root = JsonConvert.DeserializeObject <JsonMapNode>(File.ReadAllText(mapFilePath));
switch (mode.ToLowerInvariant())
{
case "basics":
WriteBasics(
root,
(args.Length > 2 ? args[2] : ""),
Console.Out);
break;
case "arraylengths":
WriteArrayEntryLengths(
root,
(args.Length > 2 ? args[2] : ""),
(args.Length > 3 ? int.Parse(args[3]) : -1),
Console.Out);
break;
case "tree":
WriteTree(
root,
(args.Length > 2 ? double.Parse(args[2]) : 0),
(args.Length > 3 ? args[3] : ""),
Console.Out);
break;
case "treetodepth":
WriteTreeToDepth(
root,
(args.Length > 2 ? int.Parse(args[2]) : -1),
(args.Length > 3 ? args[3] : ""),
Console.Out);
break;
case "extract":
Extract(
jsonFilePath,
root,
(args.Length > 2 ? args[2] : ""),
(args.Length > 3 ? args[3] : Path.ChangeExtension(jsonFilePath, ".extract.json")));
break;
default:
throw new ArgumentException($"Unknown Mode \"{mode}\".");
}
}
catch (Exception ex)
{
Console.WriteLine($"Unhandled exception: {ex}");
}
}
