/// <summary> /// Build the JsonMap for a given Json file, given a stream provider. /// Returns null if the source file was too small for any map nodes to fit the size budget. /// </summary> /// <param name="streamProvider">A function which will open the stream for the desired file</param> /// <param name="settings">JsonMapSettings for map; null for defaults</param> /// <returns>JsonMap for file or null if file too small for map</returns> public static JsonMapNode Build(Func <Stream> streamProvider, JsonMapSettings settings = null) { JsonMapRunSettings runSettings = null; using (Stream stream = streamProvider()) { long length = stream.Length; // Compute JsonMapSettings for this specific file runSettings = new JsonMapRunSettings(length, settings ?? JsonMapRunSettings.DefaultSettings); // Don't build the map at all if the file is too small for anything to be mapped if (length <= runSettings.MinimumSizeForNode) { return(null); } } // Parse file using JsonPositionedTextReader so map can get byte locations of elements using (JsonPositionedTextReader reader = new JsonPositionedTextReader(streamProvider)) { if (!reader.Read()) { return(null); } return(Build(reader, runSettings, startPosition: 0, out long _)); } }
private static void FilterArrayStarts(JsonMapNode node, JsonMapRunSettings settings) { long arraySizeBytes = node.End - node.Start; double sizeBudget = arraySizeBytes * settings.CurrentSizeRatio; double countBudget = (double)((sizeBudget - JsonMapSettings.NodeSizeEstimateBytes) / JsonMapSettings.ArrayStartSizeEstimateBytes); if (arraySizeBytes < settings.MinimumSizeForNode || node.Count < 2 || countBudget < 2) { // Overall object too small: Keep nothing. node.ArrayStarts = null; return; } else { // Determine what proportion of items we'll keep int every = (int)Math.Ceiling(node.Count / countBudget); if (every < 1) { every = 1; } node.Every = every; // If not every item, build a new array with every Nth item if (every > 1) { int newCount = (int)(node.Count / every); List <long> newStarts = new List <long>(newCount); for (int i = 0; i *every < node.Count; ++i) { newStarts.Add(node.ArrayStarts[i * every]); } node.ArrayStarts = newStarts; } } }
private static JsonMapNode Build(JsonPositionedTextReader reader, JsonMapRunSettings settings, long startPosition, out long endPosition) { // For tiny types, we know we won't create a node switch (reader.TokenType) { case JsonToken.Null: case JsonToken.Boolean: case JsonToken.Integer: case JsonToken.Float: case JsonToken.Date: case JsonToken.Comment: endPosition = reader.TokenPosition; reader.Read(); return(null); } // For Strings, create a node only if the string is big enough if (reader.TokenType == JsonToken.String) { JsonMapNode result = null; endPosition = reader.TokenPosition; if (reader.Value.ToString().Length >= settings.MinimumSizeForNode) { result = new JsonMapNode() { Start = startPosition, End = endPosition }; } reader.Read(); return(result); } // For objects and arrays, capture the exact position, then build a node and look inside... JsonMapNode node = new JsonMapNode(); node.Start = reader.TokenPosition; node.Count = 0; if (reader.TokenType == JsonToken.StartObject) { reader.Read(); while (reader.TokenType != JsonToken.EndObject) { // Value start is one after the ':' reader.TokenPosition is pointing to long valueStartPosition = reader.TokenPosition + 1; ExpectToken(JsonToken.PropertyName, reader); string propertyName = reader.Value.ToString(); reader.Read(); JsonMapNode child = Build(reader, settings, valueStartPosition, out long unused); if (child != null) { if (node.Nodes == null) { node.Nodes = new Dictionary <string, JsonMapNode>(); } node.Nodes[propertyName] = child; } node.Count++; } ExpectToken(JsonToken.EndObject, reader); endPosition = reader.TokenPosition; node.End = endPosition; reader.Read(); } else if (reader.TokenType == JsonToken.StartArray) { node.ArrayStarts = new List <long>(); long absoluteNextItemStart = reader.TokenPosition + 1; reader.Read(); while (reader.TokenType != JsonToken.EndArray) { // Consider building children if nodes are large enough JsonMapNode child = Build(reader, settings, absoluteNextItemStart, out long itemEnd); if (child != null) { if (node.Nodes == null) { node.Nodes = new Dictionary <string, JsonMapNode>(); } long itemIndex = node.Count; node.Nodes[itemIndex.ToString()] = child; absoluteNextItemStart = child.Start; } // Track the start of every array item node.ArrayStarts.Add(absoluteNextItemStart); // Next value start is two after the last value character itemEnd is pointing to (after last byte and comma) absoluteNextItemStart = itemEnd + 2; node.Count++; } endPosition = reader.TokenPosition; node.End = endPosition; reader.Read(); FilterArrayStarts(node, settings); } else { throw new NotImplementedException($"Build not implemented for node type {reader.TokenType}."); } // Return the node if it was big enough if (node.Length >= settings.MinimumSizeForNode) { return(node); } else { return(null); } }