public static JsonMapNode FindByPath(this JsonMapNode node, IReadOnlyList <string> jsonPath, bool throwIfNotFound = true) { JsonMapNode current = node; for (int i = 0; i < jsonPath.Count; ++i) { if (current.Nodes == null) { if (throwIfNotFound) { throw new ArgumentException($"Under \"{BuildJsonPath(jsonPath, i - 1)}\", no nodes were included in map to find \"{jsonPath[i]}\""); } return(null); } if (!current.Nodes.TryGetValue(jsonPath[i], out JsonMapNode child)) { if (throwIfNotFound) { throw new ArgumentException($"Under \"{BuildJsonPath(jsonPath, i - 1)}\", node \"{jsonPath[i]}\" was not found in map. Available Nodes: \r\n{String.Join("\r\n", current.Nodes.Keys)}"); } return(null); } current = child; } return(current); }
static void WriteTreeToDepth(JsonMapNode current, int depthLimit, string name, TextWriter writer, int indent = 0) { // At root, ... if (indent == 0) { // Find the desired start element if a json path was passed current = current.FindByPath(name); // Write the column names for subsequent output writer.WriteLine($"{Pad("Name", 40)} {PadRight("Count", 15)} {PadRight("Bytes", 15)}"); // Call this element 'Root' in the output if (String.IsNullOrEmpty(name)) { name = "[Root]"; } } // Write basics for this element writer.WriteLine($"{Pad(new string(' ', indent * 2) + name, 40)} {PadRight(current.Count.ToString("n0"), 15)} {PadRight(current.Length.ToString("n0"), 15)}"); // Recurse for children to depth limit if (current.Nodes != null && (indent < depthLimit || depthLimit < 0)) { foreach (KeyValuePair <string, JsonMapNode> child in current.Nodes.OrderByDescending(kvp => kvp.Value.Length)) { WriteTreeToDepth(child.Value, depthLimit, child.Key, writer, indent + 1); } } }
public void RunWithoutCatch(PageOptions options) { if (options.RunIndex < 0) { throw new ArgumentOutOfRangeException("runIndex"); } if (options.Index < 0) { throw new ArgumentOutOfRangeException("index"); } if (options.Count < 0) { throw new ArgumentOutOfRangeException("count"); } if (!_fileSystem.FileExists(options.InputFilePath)) { throw new FileNotFoundException($"Input file \"{options.InputFilePath}\" not found."); } if (options.Force == false && _fileSystem.FileExists(options.OutputFilePath)) { Console.WriteLine($"Output file \"{options.OutputFilePath}\" already exists. Stopping."); return; } // Load the JsonMap, if previously built and up-to-date, or rebuild it JsonMapNode root = LoadOrRebuildMap(options); // Write the desired page from the Sarif file ExtractPage(options, root); }
static void WriteArrayEntryLengths(JsonMapNode root, string jsonPath, int threshold, TextWriter writer) { JsonMapNode current = root.FindByPath(jsonPath); int countWritten = 0; long totalLength = 0; if (current.ArrayStarts == null) { throw new ArgumentException($"No ArrayStarts in map for node at \"{jsonPath}\"."); } if (current.Every != 1) { writer.WriteLine($"Warning: Array Starts only included for every {current.Every} elements; true count {current.Count:n0}"); } writer.WriteLine($"{jsonPath} is {ToSizeString(current.Length)} and has {current.Count:n0} elements with lengths:"); // ArrayStarts are converted to absolute in JsonMapNode for (int i = 1; i < current.ArrayStarts.Count; ++i) { // Difference between [i] and [i - 1] is the length of [i - 1] long length = current.ArrayStarts[i] - current.ArrayStarts[i - 1]; if (length > threshold) { writer.WriteLine($"[{(i - 1) * current.Every}] => {length:n0}"); countWritten++; totalLength += length; } } // Last element length is approximately from the last start to the array end if (current.Count > 0 && current.Every == 1) { long lastLength = current.End - current.ArrayStarts[current.ArrayStarts.Count - 1] - 1; if (lastLength > threshold) { writer.WriteLine($"[{current.Count - 1}] => {lastLength:n0}"); countWritten++; totalLength += lastLength; } } if (threshold > 0) { double containerPortion = totalLength / (double)root.Length; writer.WriteLine($"Elements with length >= {threshold:n0} bytes:"); writer.WriteLine($" {countWritten:n0} / {current.Count:n0} of elements ({(countWritten / (double)current.Count):p0})."); writer.WriteLine($" {totalLength:n0} / {current.Length:n0} bytes ({(totalLength / (double)current.Length):p0})."); } }
static void Build(string jsonFilePath, double ratio = 0.01) { string mapPath = Path.ChangeExtension(jsonFilePath, ".map.json"); Console.WriteLine($"Building {ratio:p1} map of \"{jsonFilePath}\"..."); Stopwatch w = Stopwatch.StartNew(); JsonMapNode root = JsonMapBuilder.Build(jsonFilePath, new JsonMapSettings(ratio, (10 * JsonMapSettings.Megabyte) * (ratio / 0.01))); File.WriteAllText(mapPath, JsonConvert.SerializeObject(root, Formatting.None)); Console.WriteLine($"Done in {w.Elapsed.TotalSeconds:n1}s; map is {ToSizeString(new FileInfo(mapPath).Length)}"); Console.WriteLine(); }
public int RunWithoutCatch(PageOptions options) { if (!ValidateOptions(options, _fileSystem)) { return(1); } // Load the JsonMap, if previously built and up-to-date, or rebuild it JsonMapNode root = LoadOrRebuildMap(options); // Write the desired page from the Sarif file ExtractPage(options, root); return(SUCCESS); }
static bool IsOverThreshold(JsonMapNode parent, JsonMapNode child, double threshold) { if (threshold <= 0) { return(true); } else if (threshold <= 1) { // Threshold under 1: Percentage of parent return(child.Length >= parent.Length * threshold); } else { // Threshold over 1: Absolute size return(child.Length >= threshold); } }
static void Extract(string jsonFilePath, JsonMapNode root, string jsonPath, string outputPath) { JsonMapNode current = root.FindByPath(jsonPath); Console.WriteLine($"Extracting \"{jsonPath}\" from \"{jsonFilePath}\" into \"{outputPath}\"..."); byte[] buffer = new byte[64 * 1024]; long lengthToCopy = current.Length; using (FileStream source = File.OpenRead(jsonFilePath)) using (FileStream target = File.Create(outputPath)) { source.Seek(current.Start, SeekOrigin.Begin); while (lengthToCopy > 0) { int lengthRead = source.Read(buffer, 0, (int)Math.Min(buffer.Length, lengthToCopy)); target.Write(buffer, 0, lengthRead); lengthToCopy -= lengthRead; } } Console.WriteLine($"Done. {ToSizeString(current.Length)} extracted."); }
private void ExtractPage(PageOptions options, JsonMapNode root) { Stopwatch w = Stopwatch.StartNew(); Console.WriteLine($"Extracting {options.Count:n0} results from index {options.Index:n0}\r\n from \"{options.InputFilePath}\"\r\n into \"{options.OutputFilePath}\"..."); JsonMapNode runs, run, results; // Get 'runs' node from map. If log was too small, page using the object model if (root == null || root.Nodes == null || root.Nodes.TryGetValue("runs", out runs) == false) { PageViaOm(options); return; } // Verify RunIndex in range if (options.RunIndex >= runs.Count) { throw new ArgumentOutOfRangeException($"Page requested for RunIndex {options.RunIndex}, but Log had only {runs.Count} runs."); } // Get 'results' from map. If log was too small, page using the object model if (!runs.Nodes.TryGetValue(options.RunIndex.ToString(), out run) || run.Nodes == null || run.Nodes.TryGetValue("results", out results) == false || results.ArrayStarts == null) { // Log too small; convert via OM PageViaOm(options); return; } if (options.Index >= results.Count) { throw new ArgumentOutOfRangeException($"Index requested was {options.Index} but Run has only {results.Count} results."); } if (options.Index + options.Count > results.Count) { Console.WriteLine($"Page requested from Result {options.Index} to {options.Index + options.Count} but Run has only {results.Count} results."); options.Count = results.Count - options.Index; } Console.WriteLine($"Run {options.RunIndex} in \"{options.InputFilePath}\" has {results.Count:n0} results."); Func <Stream> inputStreamProvider = () => _fileSystem.FileOpenRead(options.InputFilePath); long firstResultStart = results.FindArrayStart(options.Index, inputStreamProvider); long lastResultEnd = results.FindArrayStart(options.Index + options.Count, inputStreamProvider) - 1; // Ensure output directory exists string outputFolder = Path.GetDirectoryName(Path.GetFullPath(options.OutputFilePath)); Directory.CreateDirectory(outputFolder); // Build the Sarif Log subset long lengthWritten = 0; byte[] buffer = new byte[64 * 1024]; using (Stream output = _fileSystem.FileCreate(options.OutputFilePath)) using (Stream source = _fileSystem.FileOpenRead(options.InputFilePath)) { // Copy everything up to 'runs' (includes the '[') JsonMapNode.CopyStreamBytes(source, output, 0, runs.Start, buffer); // In the run, copy everything to 'results' (includes the '[') JsonMapNode.CopyStreamBytes(source, output, run.Start, results.Start, buffer); // Find and copy the desired range of results, excluding the last ',' JsonMapNode.CopyStreamBytes(source, output, firstResultStart, lastResultEnd, buffer, omitFromLast: (byte)','); // Copy everything after the results array to the end of the run (includes the '}') JsonMapNode.CopyStreamBytes(source, output, results.End, run.End, buffer); // Omit all subsequent runs // Copy everything after all runs (includes runs ']' and log '}') JsonMapNode.CopyStreamBytes(source, output, runs.End, root.End, buffer); lengthWritten = output.Length; } w.Stop(); Console.WriteLine($"Done; wrote {(lengthWritten / (double)(1024 * 1024)):n2} MB in {w.Elapsed.TotalSeconds:n1}s."); }
public static JsonMapNode FindByPath(this JsonMapNode node, string jsonPath, bool throwIfNotFound = true) { return(FindByPath(node, SplitJsonPath(jsonPath), throwIfNotFound)); }
static void WriteBasics(JsonMapNode root, string jsonPath, TextWriter writer) { JsonMapNode current = root.FindByPath(jsonPath); writer.WriteLine($"{jsonPath} is {ToSizeString(current.Length)} and has {current.Count:n0} elements"); }
static void Main(string[] args) { try { if (args.Length < 2) { Console.WriteLine(@"Usage: Map <jsonFilePath> <mode> [args] Build <ratio> - Build JSON map from source document; file path is document, not map Basics <jsonPath?> - Show size and element count under item. (over length, if included) ArrayLengths <jsonPath?> <over?> - Show the size of each element in an array (reveal outliers) Tree <threshold?> <jsonPath?> - Show tree of elements over threshold (% of parent or absolute size) TreeToDepth <depth> <jsonPath?> - Show tree to a depth limit Extract <jsonPath> <toFile> - Write element at JsonPath to a new file Indent <toFile> - Write indented copy of file Minify <toFile> - Write minified copy of file Convert <fromFile> <toFile> - Convert to/from JSON, BSON Parse <filePath> - Parse JSON/BSON files and time Consolidate <toFile> - Write SarifConsolidator-compressed copy of file LoadSarif <filePath> - Load into SARIF OM from JSON/BSON "); return; } string jsonFilePath = args[0]; string mapFilePath = Path.ChangeExtension(jsonFilePath, ".map.json"); string mode = args[1]; // Handle modes which don't use a file map switch (mode.ToLowerInvariant()) { case "indent": JsonIndent( jsonFilePath, (args.Length > 2 ? args[2] : Path.ChangeExtension(jsonFilePath, ".indent.json"))); return; case "minify": JsonMinify( jsonFilePath, (args.Length > 2 ? args[2] : Path.ChangeExtension(jsonFilePath, ".min.json"))); return; case "convert": Convert( jsonFilePath, (args.Length > 2 ? args[2] : Path.ChangeExtension(jsonFilePath, ".bson"))); return; case "consolidate": Consolidate( jsonFilePath, (args.Length > 2 ? args[2] : Path.ChangeExtension(jsonFilePath, ".trim.sarif"))); return; case "parse": Parse(jsonFilePath); return; case "loadsarif": LoadSarif(jsonFilePath); return; case "build": Build(jsonFilePath, (args.Length > 2 ? double.Parse(args[2]) : 0.01)); return; } // Build map if it is outdated or missing if (!File.Exists(mapFilePath) || (File.GetLastWriteTimeUtc(jsonFilePath) > File.GetLastWriteTimeUtc(mapFilePath))) { Build(jsonFilePath); } // Load map JsonMapNode root = JsonConvert.DeserializeObject <JsonMapNode>(File.ReadAllText(mapFilePath)); switch (mode.ToLowerInvariant()) { case "basics": WriteBasics( root, (args.Length > 2 ? args[2] : ""), Console.Out); break; case "arraylengths": WriteArrayEntryLengths( root, (args.Length > 2 ? args[2] : ""), (args.Length > 3 ? int.Parse(args[3]) : -1), Console.Out); break; case "tree": WriteTree( root, (args.Length > 2 ? double.Parse(args[2]) : 0), (args.Length > 3 ? args[3] : ""), Console.Out); break; case "treetodepth": WriteTreeToDepth( root, (args.Length > 2 ? int.Parse(args[2]) : -1), (args.Length > 3 ? args[3] : ""), Console.Out); break; case "extract": Extract( jsonFilePath, root, (args.Length > 2 ? args[2] : ""), (args.Length > 3 ? args[3] : Path.ChangeExtension(jsonFilePath, ".extract.json"))); break; default: throw new ArgumentException($"Unknown Mode \"{mode}\"."); } } catch (Exception ex) { Console.WriteLine($"Unhandled exception: {ex}"); } }