public void JsonMapBuilder_EveryTwo()
{
string sampleFilePath = "Map.TinyArray.json";
File.WriteAllText(sampleFilePath, Extractor.GetResourceText("Map.TinyArray.json"));
// File: 300b.
// Map: Root (90b) + Array (5b x 50 (Count 100 / Every 2)) = 340b
JsonMapNode root = JsonMapBuilder.Build(sampleFilePath, new JsonMapSettings(350.0 / 300.0));
// 100 array elements, only 50 starts, every = 2
Assert.Equal(100, root.Count);
Assert.Equal(50, root.ArrayStarts.Count);
Assert.Equal(2, root.Every);
// Verify we see a[0], a[2], a[4], ...
Assert.Equal(1, root.ArrayStarts[0]);
Assert.Equal(7, root.ArrayStarts[1]);
Assert.Equal(13, root.ArrayStarts[2]);
// Ask the lookup method to find the positions of every element (every 3 bytes for "#, "
for (int i = 0; i < root.Count; ++i)
{
long position = root.FindArrayStart(i, () => File.OpenRead(sampleFilePath));
Assert.Equal(1 + 3 * i, position);
}
}
public void JsonMapBuilder_EveryTwo()
{
string sampleFilePath = "Map.TinyArray.json";
File.WriteAllText(sampleFilePath, Extractor.GetResourceText("Map.TinyArray.json"));
// Allow a map 1x the size of the original file
JsonMapBuilder builder = new JsonMapBuilder(1);
JsonMapNode root = builder.Build(sampleFilePath);
// 100 array elements, only 50 starts, every = 2
Assert.Equal(100, root.Count);
Assert.Equal(50, root.ArrayStarts.Count);
Assert.Equal(2, root.Every);
// Verify we see a[0], a[2], a[4], ...
Assert.Equal(1, root.ArrayStarts[0]);
Assert.Equal(7, root.ArrayStarts[1]);
Assert.Equal(13, root.ArrayStarts[2]);
// Ask the lookup method to find the positions of every element (every 3 bytes for "#, "
for (int i = 0; i < root.Count; ++i)
{
long position = root.FindArrayStart(i, () => File.OpenRead(sampleFilePath));
Assert.Equal(1 + 3 * i, position);
}
}
public void JsonMapBuilder_Basic_20x()
{
string sampleFilePath = "Map.Sample.json";
File.WriteAllText(sampleFilePath, Extractor.GetResourceText("Map.Sample.json"));
// Allow a map 20x the size of the original file
JsonMapBuilder builder = new JsonMapBuilder(20);
JsonMapNode root = builder.Build(sampleFilePath);
Assert.NotNull(root);
Assert.Equal(0, root.Start); // Index of root '{'
Assert.Equal(154, root.End); // Index of root '}'
Assert.Null(root.ArrayStarts); // Not an Array
Assert.Equal(0, root.Every);
Assert.Equal(3, root.Count); // Version, Schema, Results
Assert.Equal(3, root.Nodes.Count);
JsonMapNode version;
Assert.True(root.Nodes.TryGetValue("version", out version));
Assert.Equal(12, version.Start);
Assert.Equal(19, version.End);
JsonMapNode schema;
Assert.True(root.Nodes.TryGetValue("schema", out schema));
Assert.Equal(31, schema.Start);
Assert.Equal(67, schema.End);
JsonMapNode results;
Assert.True(root.Nodes.TryGetValue("results", out results));
Assert.Equal(10, results.Count);
Assert.Equal(10, results.ArrayStarts.Count);
Assert.Equal(1, results.Every);
long[] absoluteStarts = new long[results.ArrayStarts.Count];
long previous = 0;
for (int i = 0; i < results.ArrayStarts.Count; ++i)
{
previous += results.ArrayStarts[i];
absoluteStarts[i] = previous;
}
// Positions are two after the previous end for literals and exact starts ('[' or '{') for nested objects/arrays
Assert.Equal(new long[] { 82, 85, 91, 97, 104, 113, 120, 134, 140, 146 }, results.ArrayStarts.ToArray());
}
public void JsonMapBuilder_Basic_2x()
{
string sampleFilePath = "Map.Sample.json";
File.WriteAllText(sampleFilePath, Extractor.GetResourceText("Map.Sample.json"));
// Allow a map 2x the size of the original file
JsonMapNode root = JsonMapBuilder.Build(sampleFilePath, new JsonMapSettings(2));
// Verify three properties but only one big enough for a node
Assert.Equal(3, root.Count);
Assert.Equal(1, root.Nodes?.Count ?? 0);
// Verify all array starts fit
JsonMapNode results;
Assert.True(root.Nodes.TryGetValue("results", out results));
Assert.Equal(10, results.Count);
Assert.Equal(10, results.ArrayStarts.Count);
Assert.Equal(1, results.Every);
}
private static void FilterArrayStarts(JsonMapNode node, JsonMapRunSettings settings)
{
long arraySizeBytes = node.End - node.Start;
double sizeBudget = arraySizeBytes * settings.CurrentSizeRatio;
double countBudget = (double)((sizeBudget - JsonMapSettings.NodeSizeEstimateBytes) / JsonMapSettings.ArrayStartSizeEstimateBytes);
if (arraySizeBytes < settings.MinimumSizeForNode || node.Count < 2 || countBudget < 2)
{
// Overall object too small: Keep nothing.
node.ArrayStarts = null;
return;
}
else
{
// Determine what proportion of items we'll keep
int every = (int)Math.Ceiling(node.Count / countBudget);
if (every < 1)
{
every = 1;
}
node.Every = every;
// If not every item, build a new array with every Nth item
if (every > 1)
{
int newCount = (int)(node.Count / every);
List <long> newStarts = new List <long>(newCount);
for (int i = 0; i *every < node.Count; ++i)
{
newStarts.Add(node.ArrayStarts[i * every]);
}
node.ArrayStarts = newStarts;
}
}
}
private JsonMapNode Build(JsonPositionedTextReader reader, long startPosition, out long endPosition)
{
// For tiny types, we know we won't create a node
switch (reader.TokenType)
{
case JsonToken.Null:
case JsonToken.Boolean:
case JsonToken.Integer:
case JsonToken.Float:
case JsonToken.Date:
case JsonToken.Comment:
endPosition = reader.TokenPosition;
reader.Read();
return(null);
}
// For Strings, create a node only if the string is big enough
if (reader.TokenType == JsonToken.String)
{
JsonMapNode result = null;
endPosition = reader.TokenPosition;
if (reader.Value.ToString().Length >= MinimumSizeForNode)
{
result = new JsonMapNode()
{
Start = startPosition, End = endPosition
};
}
reader.Read();
return(result);
}
// For objects and arrays, capture the exact position, then build a node and look inside...
JsonMapNode node = new JsonMapNode();
node.Start = reader.TokenPosition;
node.Count = 0;
if (reader.TokenType == JsonToken.StartObject)
{
reader.Read();
while (reader.TokenType != JsonToken.EndObject)
{
// Value start is one after the ':' reader.TokenPosition is pointing to
long valueStartPosition = reader.TokenPosition + 1;
ExpectToken(JsonToken.PropertyName, reader);
string propertyName = reader.Value.ToString();
reader.Read();
JsonMapNode child = Build(reader, valueStartPosition, out long unused);
if (child != null)
{
if (node.Nodes == null)
{
node.Nodes = new Dictionary <string, JsonMapNode>();
}
node.Nodes[propertyName] = child;
}
node.Count++;
}
ExpectToken(JsonToken.EndObject, reader);
endPosition = reader.TokenPosition;
node.End = endPosition;
reader.Read();
}
else if (reader.TokenType == JsonToken.StartArray)
{
node.ArrayStarts = new List <long>();
long absoluteNextItemStart = reader.TokenPosition + 1;
reader.Read();
while (reader.TokenType != JsonToken.EndArray)
{
// Consider building children if nodes are large enough
JsonMapNode child = Build(reader, absoluteNextItemStart, out long itemEnd);
if (child != null)
{
if (node.Nodes == null)
{
node.Nodes = new Dictionary <string, JsonMapNode>();
}
long itemIndex = node.Count;
node.Nodes[itemIndex.ToString()] = child;
absoluteNextItemStart = child.Start;
}
// Track the start of every array item
node.ArrayStarts.Add(absoluteNextItemStart);
// Next value start is two after the last value character itemEnd is pointing to (after last byte and comma)
absoluteNextItemStart = itemEnd + 2;
node.Count++;
}
endPosition = reader.TokenPosition;
node.End = endPosition;
reader.Read();
FilterArrayStarts(node);
}
else
{
throw new NotImplementedException($"Build not implemented for node type {reader.TokenType}.");
}
// Return the node if it was big enough
if (node.Length >= MinimumSizeForNode)
{
return(node);
}
else
{
return(null);
}
}