private static TestDom ReadJson(string jsonString) { var json = new TestDom(); if (string.IsNullOrEmpty(jsonString)) { return(json); } var jsonReader = new JsonUtf8Reader(Encoding.UTF8.GetBytes(jsonString)); jsonReader.Read(); switch (jsonReader.TokenType) { case JsonTokenType.StartArray: json.Array = ReadArray(ref jsonReader); break; case JsonTokenType.StartObject: json.Object = ReadObject(ref jsonReader); break; default: Assert.True(false, "The test JSON does not start with an array or object token"); break; } return(json); }
private static Array ReadArray(ref JsonUtf8Reader jsonReader) { // NOTE: We should be sitting on a StartArray token. Assert.Equal(JsonTokenType.StartArray, jsonReader.TokenType); Array jsonArray = new Array(); List <Value> jsonValues = new List <Value>(); while (jsonReader.Read()) { switch (jsonReader.TokenType) { case JsonTokenType.EndArray: jsonArray.Values = jsonValues; return(jsonArray); case JsonTokenType.StartArray: case JsonTokenType.StartObject: case JsonTokenType.False: case JsonTokenType.True: case JsonTokenType.Null: case JsonTokenType.Number: case JsonTokenType.String: jsonValues.Add(GetValue(ref jsonReader)); break; default: throw new ArgumentOutOfRangeException(); } } throw new FormatException("Json array was started but never ended."); }
private static void GetJsonReaderException(ref JsonUtf8Reader json, ExceptionResource resource, byte nextByte, ReadOnlySpan <byte> bytes) { string message = GetResourceString(ref json, resource, (char)nextByte, Encoding.UTF8.GetString(bytes.ToArray(), 0, bytes.Length)); message += $" LineNumber: {json._lineNumber} | BytePosition: {json._position}."; throw new JsonReaderException(message, json._lineNumber, json._position); }
public byte[] ReaderSystemTextJsonLabReturnBytes() { var outputArray = new byte[_dataUtf8.Length * 2]; Span <byte> destination = outputArray; var json = new JsonUtf8Reader(_dataUtf8); while (json.Read()) { JsonTokenType tokenType = json.TokenType; ReadOnlySpan <byte> valueSpan = json.ValueSpan; switch (tokenType) { case JsonTokenType.PropertyName: valueSpan.CopyTo(destination); destination[valueSpan.Length] = (byte)','; destination[valueSpan.Length + 1] = (byte)' '; destination = destination.Slice(valueSpan.Length + 2); break; case JsonTokenType.Number: case JsonTokenType.String: valueSpan.CopyTo(destination); destination[valueSpan.Length] = (byte)','; destination[valueSpan.Length + 1] = (byte)' '; destination = destination.Slice(valueSpan.Length + 2); break; case JsonTokenType.True: // Special casing True/False so that the casing matches with Json.NET destination[0] = (byte)'T'; destination[1] = (byte)'r'; destination[2] = (byte)'u'; destination[3] = (byte)'e'; destination[valueSpan.Length] = (byte)','; destination[valueSpan.Length + 1] = (byte)' '; destination = destination.Slice(valueSpan.Length + 2); break; case JsonTokenType.False: destination[0] = (byte)'F'; destination[1] = (byte)'a'; destination[2] = (byte)'l'; destination[3] = (byte)'s'; destination[4] = (byte)'e'; destination[valueSpan.Length] = (byte)','; destination[valueSpan.Length + 1] = (byte)' '; destination = destination.Slice(valueSpan.Length + 2); break; case JsonTokenType.Null: // Special casing Null so that it matches what JSON.NET does break; default: break; } } return(outputArray); }
private static (JsonReaderState, SequencePosition, string) ProcessDataSpan(ReadOnlySequence <byte> ros, bool isFinalBlock, JsonReaderState state = default) { var builder = new StringBuilder(); ReadOnlySpan <byte> leftOver = default; byte[] pooledArray = null; JsonUtf8Reader json = default; long totalConsumed = 0; foreach (ReadOnlyMemory <byte> mem in ros) { ReadOnlySpan <byte> span = mem.Span; if (leftOver.Length > int.MaxValue - span.Length) { throw new ArgumentOutOfRangeException("Current sequence segment size is too large to fit left over data from the previous segment into a 2 GB buffer."); } pooledArray = ArrayPool <byte> .Shared.Rent(span.Length + leftOver.Length); // This is guaranteed to not overflow Span <byte> bufferSpan = pooledArray.AsSpan(0, leftOver.Length + span.Length); leftOver.CopyTo(bufferSpan); span.CopyTo(bufferSpan.Slice(leftOver.Length)); json = new JsonUtf8Reader(bufferSpan, isFinalBlock, state); while (json.Read()) { switch (json.TokenType) { case JsonTokenType.PropertyName: json.TryGetValueAsString(out string value); builder.Append(value); break; } } if (json.BytesConsumed < bufferSpan.Length) { leftOver = bufferSpan.Slice((int)json.BytesConsumed); } else { leftOver = default; } totalConsumed += json.BytesConsumed; Assert.Equal(json.BytesConsumed, json.CurrentState.BytesConsumed); Assert.Equal(json.Position, json.CurrentState.Position); if (pooledArray != null) // TODO: Will this work if data spans more than two segments? { ArrayPool <byte> .Shared.Return(pooledArray); } state = json.CurrentState; } return(json.CurrentState, ros.GetPosition(totalConsumed), builder.ToString()); }
public void ReaderSystemTextJsonLabSpanEmptyLoop() { var json = new JsonUtf8Reader(_dataUtf8); while (json.Read()) { ; } }
public void ReaderSystemTextJsonLabMultiSpanSequenceEmptyLoop() { var json = new JsonUtf8Reader(_sequence); while (json.Read()) { ; } }
public void ValidateReaderIndexOf() { var json = new JsonUtf8Reader(_dataUtf8); while (json.Read()) { ; } }
public Utf8JsonReaderStream(Stream jsonStream) { if (!jsonStream.CanRead || !jsonStream.CanSeek) { JsonThrowHelper.ThrowArgumentException("Stream must be readable and seekable."); } _pooledArray = ArrayPool <byte> .Shared.Rent(FirstSegmentSize); int numberOfBytes = jsonStream.Read(_pooledArray, 0, FirstSegmentSize); _buffer = _pooledArray.AsSpan(0, numberOfBytes); _stream = jsonStream; _isFinalBlock = numberOfBytes == 0; _jsonReader = new JsonUtf8Reader(_buffer, _isFinalBlock); _consumed = 0; }
private static (JsonReaderState, string) ProcessData(ReadOnlySequence <byte> ros, bool isFinalBlock, JsonReaderState state = default) { var builder = new StringBuilder(); var json = new JsonUtf8Reader(ros, isFinalBlock, state); while (json.Read()) { switch (json.TokenType) { case JsonTokenType.PropertyName: json.TryGetValueAsString(out string value); builder.Append(value); break; } } Assert.Equal(json.BytesConsumed, json.CurrentState.BytesConsumed); Assert.Equal(json.Position, json.CurrentState.Position); return(json.CurrentState, builder.ToString()); }
private bool ReadNext() { bool result = false; do { _consumed += _jsonReader.BytesConsumed; int leftOver = _buffer.Length - (int)_jsonReader.BytesConsumed; int amountToRead = StreamSegmentSize; if (leftOver > 0) { _stream.Position -= leftOver; if (_jsonReader.BytesConsumed == 0) { if (leftOver > 1_000_000_000) { JsonThrowHelper.ThrowArgumentException("Cannot fit left over data from the previous chunk and the next chunk of data into a 2 GB buffer."); } // This is guaranteed to not overflow due to the check above. amountToRead += leftOver * 2; ResizeBuffer(amountToRead); } } if (_pooledArray.Length < StreamSegmentSize) { ResizeBuffer(StreamSegmentSize); } int numberOfBytes = _stream.Read(_pooledArray, 0, amountToRead); _isFinalBlock = numberOfBytes == 0; // TODO: Can this be inferred differently based on leftOver and numberOfBytes _buffer = _pooledArray.AsSpan(0, numberOfBytes); _jsonReader = new JsonUtf8Reader(_buffer, _isFinalBlock, _jsonReader.CurrentState); result = _jsonReader.Read(); } while (!result && !_isFinalBlock); return(result); }
private static Value GetValue(ref JsonUtf8Reader jsonReader) { var value = new Value { Type = MapValueType(jsonReader.TokenType) }; switch (value.Type) { case Value.ValueType.String: value.StringValue = ReadUtf8String(ref jsonReader); break; case Value.ValueType.Number: CustomParser.TryParseDecimal(jsonReader.ValueSpan, out decimal num, out int consumed); value.NumberValue = Convert.ToDouble(num); break; case Value.ValueType.True: break; case Value.ValueType.False: break; case Value.ValueType.Null: break; case Value.ValueType.Object: value.ObjectValue = ReadObject(ref jsonReader); break; case Value.ValueType.Array: value.ArrayValue = ReadArray(ref jsonReader); break; default: throw new ArgumentOutOfRangeException(); } return(value); }
private static Object ReadObject(ref JsonUtf8Reader jsonReader) { // NOTE: We should be sitting on a StartObject token. Assert.Equal(JsonTokenType.StartObject, jsonReader.TokenType); var jsonObject = new Object(); List <Pair> jsonPairs = new List <Pair>(); while (jsonReader.Read()) { switch (jsonReader.TokenType) { case JsonTokenType.EndObject: jsonObject.Pairs = jsonPairs; return(jsonObject); case JsonTokenType.PropertyName: ReadOnlyMemory <byte> name = ReadUtf8String(ref jsonReader); jsonReader.Read(); // Move to value token var pair = new Pair { Name = name, Value = GetValue(ref jsonReader) }; if (jsonPairs != null) { jsonPairs.Add(pair); } break; default: throw new ArgumentOutOfRangeException(); } } throw new FormatException("Json object was started but never ended."); }
private static byte[] ReadUtf8String(ref JsonUtf8Reader jsonReader) { return(jsonReader.ValueSpan.ToArray()); }
public static JsonDynamicObject Parse(ReadOnlySpan <byte> utf8, int expectedNumberOfProperties = -1) { Stack <JsonDynamicObject> stack = new Stack <JsonDynamicObject>(); if (expectedNumberOfProperties == -1) { expectedNumberOfProperties = utf8.Length >> 3; } var properties = new Dictionary <JsonProperty, JsonValue>(expectedNumberOfProperties); stack.Push(new JsonDynamicObject(properties)); var reader = new JsonUtf8Reader(utf8); while (reader.Read()) { switch (reader.TokenType) { case JsonTokenType.PropertyName: var name = new Utf8String(reader.ValueSpan); reader.Read(); // Move to the value token var type = reader.TokenType; var current = stack.Peek(); var property = new JsonProperty(current, name); switch (type) { case JsonTokenType.String: current._properties[property] = new JsonValue(new Utf8String(reader.ValueSpan)); break; case JsonTokenType.StartObject: // TODO: could this be lazy? Could this reuse the root JsonObject (which would store non-allocating JsonDom)? var newObj = new JsonDynamicObject(properties); current._properties[property] = new JsonValue(newObj); stack.Push(newObj); break; case JsonTokenType.True: current._properties[property] = new JsonValue(type); break; case JsonTokenType.False: current._properties[property] = new JsonValue(type); break; case JsonTokenType.Null: current._properties[property] = new JsonValue(type); break; case JsonTokenType.Number: current._properties[property] = new JsonValue(new Utf8String(reader.ValueSpan), type); break; case JsonTokenType.StartArray: throw new NotImplementedException("array support not implemented yet."); default: throw new NotSupportedException(); } break; case JsonTokenType.StartObject: break; case JsonTokenType.EndObject: if (stack.Count != 1) { stack.Pop(); } break; case JsonTokenType.StartArray: throw new NotImplementedException("array support not implemented yet."); case JsonTokenType.EndArray: case JsonTokenType.String: case JsonTokenType.True: case JsonTokenType.False: case JsonTokenType.Null: case JsonTokenType.Number: break; default: throw new NotSupportedException(); } } return(stack.Peek()); }
public static T Deserialize <T>(ReadOnlySpan <byte> utf8) { if (!TypeCache.TryGetValue(typeof(T), out Dictionary <int, PropertyInfoLinkedList> dictionary)) { dictionary = GetTypeMap(typeof(T)); TypeCache.Add(typeof(T), dictionary); } T instance = Create <T> .CreateInstanceOfType(utf8); var reader = new JsonUtf8Reader(utf8); while (reader.Read()) { switch (reader.TokenType) { case JsonTokenType.PropertyName: int key = GetHashCode(reader.ValueSpan); reader.Read(); // Move to the value token JsonTokenType type = reader.TokenType; switch (type) { case JsonTokenType.String: PropertyInfo pi = GetPropertyInfo(dictionary, key, reader.ValueSpan); pi.SetValue(instance, new Utf8String(reader.ValueSpan)); // TODO: Use Ref.Emit instead of Reflection break; case JsonTokenType.StartObject: // TODO: could this be lazy? Could this reuse the root JsonObject (which would store non-allocating JsonDom)? throw new NotImplementedException("object support not implemented yet."); case JsonTokenType.True: pi = GetPropertyInfo(dictionary, key, reader.ValueSpan); pi.SetValue(instance, true); break; case JsonTokenType.False: pi = GetPropertyInfo(dictionary, key, reader.ValueSpan); pi.SetValue(instance, false); break; case JsonTokenType.Null: pi = GetPropertyInfo(dictionary, key, reader.ValueSpan); pi.SetValue(instance, null); break; case JsonTokenType.Number: pi = GetPropertyInfo(dictionary, key, reader.ValueSpan); // TODO: Add support for other numeric types like double, long, etc. if (!Utf8Parser.TryParse(reader.ValueSpan, out int result, out _)) { throw new InvalidCastException(); } pi.SetValue(instance, result); break; case JsonTokenType.StartArray: throw new NotImplementedException("array support not implemented yet."); default: throw new NotSupportedException(); } break; case JsonTokenType.StartObject: break; case JsonTokenType.EndObject: break; case JsonTokenType.StartArray: throw new NotImplementedException("array support not implemented yet."); case JsonTokenType.EndArray: case JsonTokenType.String: case JsonTokenType.True: case JsonTokenType.False: case JsonTokenType.Null: case JsonTokenType.Number: break; default: throw new NotSupportedException(); } } return(instance); }
private static string GetResourceString(ref JsonUtf8Reader json, ExceptionResource resource, char character, string characters) { Debug.Assert(Enum.IsDefined(typeof(ExceptionResource), resource), "The enum value is not defined, please check the ExceptionResource Enum."); string formatString = ExceptionStrings.ResourceManager.GetString(resource.ToString()); string message = formatString; switch (resource) { case ExceptionResource.ArrayDepthTooLarge: message = string.Format(formatString, json.CurrentDepth, json.MaxDepth); break; case ExceptionResource.ArrayEndWithinObject: if (json.CurrentDepth <= 0) { formatString = ExceptionStrings.ResourceManager.GetString(ExceptionResource.DepthMustBePositive.ToString()); message = string.Format(formatString, json.CurrentDepth); } else { message = string.Format(formatString); } break; case ExceptionResource.EndOfStringNotFound: break; case ExceptionResource.ExpectedDigitNotFound: message = string.Format(formatString, character); break; case ExceptionResource.ExpectedDigitNotFoundEndOfData: message = string.Format(formatString, character); break; case ExceptionResource.ExpectedEndAfterSingleJson: message = string.Format(formatString, character); break; case ExceptionResource.ExpectedEndOfDigitNotFound: message = string.Format(formatString, character); break; case ExceptionResource.ExpectedNextDigitComponentNotFound: message = string.Format(formatString, character); break; case ExceptionResource.ExpectedNextDigitEValueNotFound: message = string.Format(formatString, character); break; case ExceptionResource.ExpectedSeparaterAfterPropertyNameNotFound: message = string.Format(formatString, character); break; case ExceptionResource.ExpectedStartOfPropertyNotFound: message = string.Format(formatString, character); break; case ExceptionResource.ExpectedStartOfPropertyOrValueNotFound: break; case ExceptionResource.ExpectedStartOfValueNotFound: message = string.Format(formatString, character); break; case ExceptionResource.ExpectedValueAfterPropertyNameNotFound: break; case ExceptionResource.FoundInvalidCharacter: message = string.Format(formatString, character); break; case ExceptionResource.InvalidEndOfJson: message = string.Format(formatString, json.TokenType); break; case ExceptionResource.ObjectDepthTooLarge: message = string.Format(formatString, json.CurrentDepth, json.MaxDepth); break; case ExceptionResource.ObjectEndWithinArray: if (json.CurrentDepth <= 0) { formatString = ExceptionStrings.ResourceManager.GetString(ExceptionResource.DepthMustBePositive.ToString()); message = string.Format(formatString, json.CurrentDepth); } else { message = string.Format(formatString); } break; case ExceptionResource.Default: break; case ExceptionResource.ExpectedFalse: message = string.Format(formatString, characters); break; case ExceptionResource.ExpectedNull: message = string.Format(formatString, characters); break; case ExceptionResource.ExpectedTrue: message = string.Format(formatString, characters); break; // This case is covered between ArrayEndWithinObject and ObjectEndWithinArray /*case ExceptionResource.DepthMustBePositive: * break;*/ case ExceptionResource.InvalidCharacterWithinString: message = string.Format(formatString, character); break; case ExceptionResource.EndOfCommentNotFound: break; } return(message); }
public static void ThrowJsonReaderException(ref JsonUtf8Reader json, ExceptionResource resource = ExceptionResource.Default, byte nextByte = default, ReadOnlySpan <byte> bytes = default) { GetJsonReaderException(ref json, resource, nextByte, bytes); }