private static void ExpectToken(JsonToken expected, JsonPositionedTextReader reader)
{
if (reader.TokenType != expected)
{
throw new JsonReaderException($"Expect: {expected} at {reader.TokenPosition:n0}, was {reader.TokenType}.");
}
}
/// <summary>
/// Build the JsonMap for a given Json file, given a stream provider.
/// Returns null if the source file was too small for any map nodes to fit the size budget.
/// </summary>
/// <param name="streamProvider">A function which will open the stream for the desired file</param>
/// <param name="settings">JsonMapSettings for map; null for defaults</param>
/// <returns>JsonMap for file or null if file too small for map</returns>
public static JsonMapNode Build(Func <Stream> streamProvider, JsonMapSettings settings = null)
{
JsonMapRunSettings runSettings = null;
using (Stream stream = streamProvider())
{
long length = stream.Length;
// Compute JsonMapSettings for this specific file
runSettings = new JsonMapRunSettings(length, settings ?? JsonMapRunSettings.DefaultSettings);
// Don't build the map at all if the file is too small for anything to be mapped
if (length <= runSettings.MinimumSizeForNode)
{
return(null);
}
}
// Parse file using JsonPositionedTextReader so map can get byte locations of elements
using (JsonPositionedTextReader reader = new JsonPositionedTextReader(streamProvider))
{
if (!reader.Read())
{
return(null);
}
return(Build(reader, runSettings, startPosition: 0, out long _));
}
}
public static void ConvertSarifLog(JsonSerializer serializer, string sarifFilePath, CsvWriter writer, IEnumerable <Action <CsvWriter, Result, PhysicalLocation> > selectedWriters)
{
SarifLog log = null;
// Read the SarifLog with the deferred reader
using (JsonPositionedTextReader jtr = new JsonPositionedTextReader(sarifFilePath))
{
log = serializer.Deserialize <SarifLog>(jtr);
}
// Write a row in the output file for each result location
if (log.Runs != null)
{
foreach (var run in log.Runs)
{
if (run.Results != null)
{
foreach (var result in run.Results)
{
foreach (PhysicalLocation location in result.PhysicalLocations())
{
foreach (Action <CsvWriter, Result, PhysicalLocation> columnWriter in selectedWriters)
{
columnWriter(writer, result, location);
}
writer.NextRow();
}
}
}
}
}
}
/// <summary>
/// Load a SARIF file into a SarifLog object model instance using deferred loading.
/// [Less memory use, but slower; safe for large Sarif]
/// </summary>
/// <param name="sarifFilePath">File Path to Sarif file to load</param>
/// <returns>SarifLog instance for file</returns>
public static SarifLog LoadDeferred(string sarifFilePath)
{
JsonSerializer serializer = new JsonSerializer();
serializer.ContractResolver = new SarifDeferredContractResolver();
using (JsonPositionedTextReader jptr = new JsonPositionedTextReader(sarifFilePath))
{
return(serializer.Deserialize <SarifLog>(jptr));
}
}
public static void ConvertSarifLog(JsonSerializer serializer, string sarifFilePath, CsvWriter writer, IEnumerable <Action <WriteContext> > selectedWriters)
{
SarifLog log = null;
// Read the SarifLog with the deferred reader
using (JsonPositionedTextReader jtr = new JsonPositionedTextReader(sarifFilePath))
{
log = serializer.Deserialize <SarifLog>(jtr);
}
WriteContext context = new WriteContext();
context.Writer = writer;
// Write a row in the output file for each result location
if (log.Runs != null)
{
context.RunIndex = 0;
foreach (var run in log.Runs)
{
context.Run = run;
context.ResultIndex = 0;
if (run.Results != null)
{
foreach (var result in run.Results)
{
context.Result = result;
if (result.Locations == null || result.Locations.Count == 0)
{
context.PLoc = null;
Write(context, selectedWriters);
}
else
{
foreach (PhysicalLocation location in result.PhysicalLocations())
{
context.PLoc = location;
Write(context, selectedWriters);
}
}
context.ResultIndex++;
}
}
context.RunIndex++;
}
}
}
/// <summary>
/// Load a SARIF file into a SarifLog object model instance using deferred loading.
/// [Less memory use, but slower; safe for large Sarif]
/// </summary>
/// <param name="sarifFilePath">File Path to Sarif file to load</param>
/// <returns>SarifLog instance for file</returns>
public static SarifLog LoadDeferred(string sarifFilePath)
{
JsonSerializer serializer = new JsonSerializer();
serializer.ContractResolver = new SarifDeferredContractResolver();
// NOTE: Deferred reading needs JsonPositionedTextReader to identify where deferred collection items are
// (to deserialize them 'just in time' later) and needs a 'streamProvider' function (how to open the file again)
// so that deferred collections know how to open the file again to seek to and read elements 'just in time'
using (JsonPositionedTextReader jtr = new JsonPositionedTextReader(sarifFilePath))
{
// NOTE: Load with JsonSerializer.Deserialize, not JsonConvert.DeserializeObject, to avoid a string of the whole file in memory.
return(serializer.Deserialize <SarifLog>(jtr));
}
}
/// <summary>
/// Build the JsonMap for a given Json file, given a stream provider.
/// Returns null if the source file was too small for any map nodes to fit the size budget.
/// </summary>
/// <param name="streamProvider">A function which will open the stream for the desired file</param>
/// <returns>JsonMap for file or null if file too small for map</returns>
public JsonMapNode Build(Func <Stream> streamProvider)
{
using (Stream stream = streamProvider())
{
if (stream.Length <= MinimumSizeForNode)
{
return(null);
}
}
using (JsonPositionedTextReader reader = new JsonPositionedTextReader(streamProvider))
{
if (!reader.Read())
{
return(null);
}
return(Build(reader, 0, out long unused));
}
}
/// <summary>
/// Find the start position of the desired item index.
/// If the node knows it, the known value is returned.
/// If not, the code seeks to the nearest node and reads enough of the document to find it.
/// </summary>
/// <param name="index">Index of array element to find</param>
/// <param name="inputStreamProvider">Function which can open original file, if needed</param>
/// <returns>Absolute byte offset of the start array[index] within this array</returns>
public long FindArrayStart(int index, Func <Stream> inputStreamProvider)
{
if (index < 0 || index > this.Count)
{
throw new ArgumentOutOfRangeException("index");
}
if (index == this.Count)
{
// If item after last is requested, return array end
return(this.End);
}
else if (this.ArrayStarts != null && this.Every == 1)
{
// If we know every element position, just return it
return(this.ArrayStarts[index]);
}
else if (index % this.Every == 0)
{
// If we know this element position, just return it
return(this.ArrayStarts[index / this.Every]);
}
// Otherwise, find the closest span of the file we must read
long readFromPosition;
long readToPosition;
int startIndex;
if (this.ArrayStarts == null)
{
// If there are no array positions, we must read the whole array (it should be small)
readFromPosition = this.Start + 1;
readToPosition = this.End;
startIndex = 0;
}
else
{
// If there are array positions, read from the nearest previous element available
int startToRead = index / this.Every;
readFromPosition = this.ArrayStarts[startToRead];
readToPosition = (this.ArrayStarts.Count > startToRead + 1 ? this.ArrayStarts[startToRead + 1] : this.End);
startIndex = startToRead * this.Every;
}
using (Stream source = inputStreamProvider())
{
int lengthToRead = (int)(1 + readToPosition - readFromPosition);
byte[] buffer = new byte[lengthToRead + 1];
// Read the array slice
source.Seek(readFromPosition, SeekOrigin.Begin);
source.Read(buffer, 1, lengthToRead);
// Make it a valid array prefix (it must start with '[', which will look like the root of the Json document
buffer[0] = (byte)'[';
using (JsonPositionedTextReader reader = new JsonPositionedTextReader(() => new MemoryStream(buffer)))
{
// Find the desired array item index in the buffer
long relativePosition = reader.ReadToArrayIndex(index - startIndex);
// Convert back to an absolute position (buffer[0] was (readFromPosition - 1)
return((readFromPosition - 1) + relativePosition);
}
}
}
private JsonMapNode Build(JsonPositionedTextReader reader, long startPosition, out long endPosition)
{
// For tiny types, we know we won't create a node
switch (reader.TokenType)
{
case JsonToken.Null:
case JsonToken.Boolean:
case JsonToken.Integer:
case JsonToken.Float:
case JsonToken.Date:
case JsonToken.Comment:
endPosition = reader.TokenPosition;
reader.Read();
return(null);
}
// For Strings, create a node only if the string is big enough
if (reader.TokenType == JsonToken.String)
{
JsonMapNode result = null;
endPosition = reader.TokenPosition;
if (reader.Value.ToString().Length >= MinimumSizeForNode)
{
result = new JsonMapNode()
{
Start = startPosition, End = endPosition
};
}
reader.Read();
return(result);
}
// For objects and arrays, capture the exact position, then build a node and look inside...
JsonMapNode node = new JsonMapNode();
node.Start = reader.TokenPosition;
node.Count = 0;
if (reader.TokenType == JsonToken.StartObject)
{
reader.Read();
while (reader.TokenType != JsonToken.EndObject)
{
// Value start is one after the ':' reader.TokenPosition is pointing to
long valueStartPosition = reader.TokenPosition + 1;
ExpectToken(JsonToken.PropertyName, reader);
string propertyName = reader.Value.ToString();
reader.Read();
JsonMapNode child = Build(reader, valueStartPosition, out long unused);
if (child != null)
{
if (node.Nodes == null)
{
node.Nodes = new Dictionary <string, JsonMapNode>();
}
node.Nodes[propertyName] = child;
}
node.Count++;
}
ExpectToken(JsonToken.EndObject, reader);
endPosition = reader.TokenPosition;
node.End = endPosition;
reader.Read();
}
else if (reader.TokenType == JsonToken.StartArray)
{
node.ArrayStarts = new List <long>();
long absoluteNextItemStart = reader.TokenPosition + 1;
reader.Read();
while (reader.TokenType != JsonToken.EndArray)
{
// Consider building children if nodes are large enough
JsonMapNode child = Build(reader, absoluteNextItemStart, out long itemEnd);
if (child != null)
{
if (node.Nodes == null)
{
node.Nodes = new Dictionary <string, JsonMapNode>();
}
long itemIndex = node.Count;
node.Nodes[itemIndex.ToString()] = child;
absoluteNextItemStart = child.Start;
}
// Track the start of every array item
node.ArrayStarts.Add(absoluteNextItemStart);
// Next value start is two after the last value character itemEnd is pointing to (after last byte and comma)
absoluteNextItemStart = itemEnd + 2;
node.Count++;
}
endPosition = reader.TokenPosition;
node.End = endPosition;
reader.Read();
FilterArrayStarts(node);
}
else
{
throw new NotImplementedException($"Build not implemented for node type {reader.TokenType}.");
}
// Return the node if it was big enough
if (node.Length >= MinimumSizeForNode)
{
return(node);
}
else
{
return(null);
}
}
