public static byte[] SequenceReturnBytesHelper(byte[] data, out int length, JsonCommentHandling commentHandling = JsonCommentHandling.Disallow, int maxDepth = 64) { ReadOnlySequence <byte> sequence = CreateSegments(data); var state = new JsonReaderState(new JsonReaderOptions { CommentHandling = commentHandling, MaxDepth = maxDepth }); var reader = new Utf8JsonReader(sequence, true, state); return(ReaderLoop(data.Length, out length, ref reader)); }
/// <summary> /// Parse the text representing a single JSON value into a <paramref name="returnType"/>. /// </summary> /// <returns>A <paramref name="returnType"/> representation of the JSON value.</returns> /// <param name="json">JSON text to parse.</param> /// <param name="returnType">The type of the object to convert to and return.</param> /// <param name="options">Options to control the behavior during parsing.</param> /// <exception cref="System.ArgumentNullException"> /// Thrown if <paramref name="json"/> or <paramref name="returnType"/> is null. /// </exception> /// <exception cref="JsonException"> /// Thrown when the JSON is invalid, /// the <paramref name="returnType"/> is not compatible with the JSON, /// or when there is remaining data in the Stream. /// </exception> /// <remarks>Using a <see cref="System.String"/> is not as efficient as using the /// UTF-8 methods since the implementation natively uses UTF-8. /// </remarks> public static object Deserialize(string json, Type returnType, JsonSerializerOptions options = null) { const long ArrayPoolMaxSizeBeforeUsingNormalAlloc = 1024 * 1024; if (json == null) { throw new ArgumentNullException(nameof(json)); } if (returnType == null) { throw new ArgumentNullException(nameof(returnType)); } if (options == null) { options = JsonSerializerOptions.s_defaultOptions; } object result; byte[] tempArray = null; // For performance, avoid obtaining actual byte count unless memory usage is higher than the threshold. Span <byte> utf8 = json.Length <= (ArrayPoolMaxSizeBeforeUsingNormalAlloc / JsonConstants.MaxExpansionFactorWhileTranscoding) ? // Use a pooled alloc. tempArray = ArrayPool <byte> .Shared.Rent(json.Length *JsonConstants.MaxExpansionFactorWhileTranscoding) : // Use a normal alloc since the pool would create a normal alloc anyway based on the threshold (per current implementation) // and by using a normal alloc we can avoid the Clear(). new byte[JsonReaderHelper.GetUtf8ByteCount(json.AsSpan())]; try { int actualByteCount = JsonReaderHelper.GetUtf8FromText(json.AsSpan(), utf8); utf8 = utf8.Slice(0, actualByteCount); var readerState = new JsonReaderState(options.GetReaderOptions()); var reader = new Utf8JsonReader(utf8, isFinalBlock: true, readerState); result = ReadCore(returnType, options, ref reader); // The reader should have thrown if we have remaining bytes. Debug.Assert(reader.BytesConsumed == actualByteCount); } finally { if (tempArray != null) { utf8.Clear(); ArrayPool <byte> .Shared.Return(tempArray); } } return(result); }
/// <summary> /// Parse the UTF-8 encoded text representing a single JSON value into a <typeparamref name="TValue"/>. /// </summary> /// <returns>A <typeparamref name="TValue"/> representation of the JSON value.</returns> /// <param name="utf8Json">JSON text to parse.</param> /// <param name="options">Options to control the behavior during parsing.</param> /// <exception cref="JsonException"> /// Thrown when the JSON is invalid, /// <typeparamref name="TValue"/> is not compatible with the JSON, /// or when there is remaining data in the Stream. /// </exception> /// <exception cref="NotSupportedException"> /// There is no compatible <see cref="System.Text.Json.Serialization.JsonConverter"/> /// for <typeparamref name="TValue"/> or its serializable members. /// </exception> public static TValue?Deserialize <[DynamicallyAccessedMembers(JsonHelpers.MembersAccessedOnRead)] TValue>(ReadOnlySpan <byte> utf8Json, JsonSerializerOptions?options = null) { if (options == null) { options = JsonSerializerOptions.s_defaultOptions; } var readerState = new JsonReaderState(options.GetReaderOptions()); var reader = new Utf8JsonReader(utf8Json, isFinalBlock: true, readerState); return(ReadCore <TValue>(ref reader, typeof(TValue), options)); }
public static TValue Deserialize <TValue>(ReadOnlySpan <byte> utf8Json, JsonSerializerOptions?options = null) { if (options == null) { options = JsonSerializerOptions.s_defaultOptions; } var readerState = new JsonReaderState(options.GetReaderOptions()); var reader = new Utf8JsonReader(utf8Json, isFinalBlock: true, readerState); return(ReadCore <TValue>(ref reader, typeof(TValue), options)); }
private static async IAsyncEnumerable <TValue> CreateAsyncEnumerableDeserializer <TValue>( Stream utf8Json, JsonTypeInfo jsonTypeInfo, [EnumeratorCancellation] CancellationToken cancellationToken) { JsonSerializerOptions options = jsonTypeInfo.Options; JsonTypeInfo <Queue <TValue> > queueTypeInfo = JsonMetadataServices.CreateQueueInfo <Queue <TValue>, TValue>( options: options, collectionInfo: new() { ObjectCreator = () => new Queue <TValue>(), ElementInfo = jsonTypeInfo, NumberHandling = options.NumberHandling }); var bufferState = new ReadBufferState(options.DefaultBufferSize); ReadStack readStack = default; queueTypeInfo.EnsureConfigured(); readStack.Initialize(queueTypeInfo, supportContinuation: true); var jsonReaderState = new JsonReaderState(options.GetReaderOptions()); try { do { bufferState = await ReadFromStreamAsync(utf8Json, bufferState, cancellationToken).ConfigureAwait(false); ContinueDeserialize <Queue <TValue> >( ref bufferState, ref jsonReaderState, ref readStack, queueTypeInfo.PropertyInfoForTypeInfo.ConverterBase, options); if (readStack.Current.ReturnValue is Queue <TValue> queue) { while (queue.Count > 0) { yield return(queue.Dequeue()); } } }while (!bufferState.IsFinalBlock); } finally { bufferState.Dispose(); } }
private static object?ParseCore(ReadOnlySpan <byte> utf8Json, Type returnType, JsonSerializerOptions?options) { if (options == null) { options = JsonSerializerOptions.s_defaultOptions; } var readerState = new JsonReaderState(options.GetReaderOptions()); var reader = new Utf8JsonReader(utf8Json, isFinalBlock: true, readerState); object?result = ReadCore(returnType, options, ref reader); // The reader should have thrown if we have remaining bytes. Debug.Assert(reader.BytesConsumed == utf8Json.Length); return(result); }
/// <summary> /// Parse the UTF-8 encoded text representing a single JSON value into a <paramref name="returnType"/>. /// </summary> /// <returns>A <paramref name="returnType"/> representation of the JSON value.</returns> /// <param name="utf8Json">JSON text to parse.</param> /// <param name="returnType">The type of the object to convert to and return.</param> /// <param name="options">Options to control the behavior during parsing.</param> /// <exception cref="System.ArgumentNullException"> /// <paramref name="returnType"/> is <see langword="null"/>. /// </exception> /// <exception cref="JsonException"> /// Thrown when the JSON is invalid, /// <paramref name="returnType"/> is not compatible with the JSON, /// or when there is remaining data in the Stream. /// </exception> /// <exception cref="NotSupportedException"> /// There is no compatible <see cref="System.Text.Json.Serialization.JsonConverter"/> /// for <paramref name="returnType"/> or its serializable members. /// </exception> public static object?Deserialize(ReadOnlySpan <byte> utf8Json, Type returnType, JsonSerializerOptions?options = null) { if (returnType == null) { throw new ArgumentNullException(nameof(returnType)); } if (options == null) { options = JsonSerializerOptions.s_defaultOptions; } var readerState = new JsonReaderState(options.GetReaderOptions()); var reader = new Utf8JsonReader(utf8Json, isFinalBlock: true, readerState); return(ReadCore <object>(ref reader, returnType, options)); }
private static TValue?ReadUsingMetadata <TValue>(ReadOnlySpan <byte> utf8Json, JsonTypeInfo jsonTypeInfo, int?actualByteCount = null) { JsonSerializerOptions options = jsonTypeInfo.Options; var readerState = new JsonReaderState(options.GetReaderOptions()); var reader = new Utf8JsonReader(utf8Json, isFinalBlock: true, readerState); ReadStack state = default; state.Initialize(jsonTypeInfo); TValue?value = ReadCore <TValue>(jsonTypeInfo.PropertyInfoForTypeInfo.ConverterBase, ref reader, options, ref state); // The reader should have thrown if we have remaining bytes. Debug.Assert(reader.BytesConsumed == (actualByteCount ?? utf8Json.Length)); return(value); }
/// <summary> /// Parse the UTF-8 encoded text representing a single JSON value into a <typeparamref name="TValue"/>. /// </summary> /// <returns>A <typeparamref name="TValue"/> representation of the JSON value.</returns> /// <param name="utf8Json">JSON text to parse.</param> /// <param name="jsonTypeInfo">Metadata about the type to convert.</param> /// <exception cref="JsonException"> /// Thrown when the JSON is invalid, /// <typeparamref name="TValue"/> is not compatible with the JSON, /// or when there is remaining data in the Stream. /// </exception> /// <exception cref="NotSupportedException"> /// There is no compatible <see cref="System.Text.Json.Serialization.JsonConverter"/> /// for <typeparamref name="TValue"/> or its serializable members. /// </exception> public static TValue?Deserialize <TValue>(ReadOnlySpan <byte> utf8Json, JsonTypeInfo <TValue> jsonTypeInfo) { if (jsonTypeInfo == null) { throw new ArgumentNullException(nameof(jsonTypeInfo)); } JsonSerializerOptions options = jsonTypeInfo.Options; var readerState = new JsonReaderState(options.GetReaderOptions()); var reader = new Utf8JsonReader(utf8Json, isFinalBlock: true, readerState); ReadStack state = default; state.Initialize(jsonTypeInfo); return(ReadCore <TValue>(jsonTypeInfo.PropertyInfoForTypeInfo.ConverterBase, ref reader, options, ref state)); }
/// <summary> /// Parse the UTF-8 encoded text representing a single JSON value into a <paramref name="returnType"/>. /// </summary> /// <returns>A <paramref name="returnType"/> representation of the JSON value.</returns> /// <param name="utf8Json">JSON text to parse.</param> /// <param name="returnType">The type of the object to convert to and return.</param> /// <param name="options">Options to control the behavior during parsing.</param> /// <exception cref="System.ArgumentNullException"> /// <paramref name="returnType"/> is <see langword="null"/>. /// </exception> /// <exception cref="JsonException"> /// Thrown when the JSON is invalid, /// <paramref name="returnType"/> is not compatible with the JSON, /// or when there is remaining data in the Stream. /// </exception> /// <exception cref="NotSupportedException"> /// There is no compatible <see cref="System.Text.Json.Serialization.JsonConverter"/> /// for <paramref name="returnType"/> or its serializable members. /// </exception> public static object?Deserialize(ReadOnlySpan <byte> utf8Json, [DynamicallyAccessedMembers(JsonHelpers.MembersAccessedOnRead)] Type returnType, JsonSerializerOptions?options = null) { if (returnType == null) { throw new ArgumentNullException(nameof(returnType)); } if (options == null) { options = JsonSerializerOptions.s_defaultOptions; } options.RootBuiltInConvertersAndTypeInfoCreator(); var readerState = new JsonReaderState(options.GetReaderOptions()); var reader = new Utf8JsonReader(utf8Json, isFinalBlock: true, readerState); return(ReadCore <object>(ref reader, returnType, options)); }
private static TValue?Deserialize <TValue>( JsonConverter jsonConverter, ReadOnlySpan <char> json, JsonSerializerOptions options, ref ReadStack state) { const long ArrayPoolMaxSizeBeforeUsingNormalAlloc = 1024 * 1024; byte[]? tempArray = null; // For performance, avoid obtaining actual byte count unless memory usage is higher than the threshold. Span <byte> utf8 = json.Length <= (ArrayPoolMaxSizeBeforeUsingNormalAlloc / JsonConstants.MaxExpansionFactorWhileTranscoding) ? // Use a pooled alloc. tempArray = ArrayPool <byte> .Shared.Rent(json.Length *JsonConstants.MaxExpansionFactorWhileTranscoding) : // Use a normal alloc since the pool would create a normal alloc anyway based on the threshold (per current implementation) // and by using a normal alloc we can avoid the Clear(). new byte[JsonReaderHelper.GetUtf8ByteCount(json)]; try { int actualByteCount = JsonReaderHelper.GetUtf8FromText(json, utf8); utf8 = utf8.Slice(0, actualByteCount); var readerState = new JsonReaderState(options.GetReaderOptions()); var reader = new Utf8JsonReader(utf8, isFinalBlock: true, readerState); TValue?value = ReadCore <TValue>(jsonConverter, ref reader, options, ref state); // The reader should have thrown if we have remaining bytes. Debug.Assert(reader.BytesConsumed == actualByteCount); return(value); } finally { if (tempArray != null) { utf8.Clear(); ArrayPool <byte> .Shared.Return(tempArray); } } }
private static TValue ReadCore <TValue>( ref JsonReaderState readerState, bool isFinalBlock, ReadOnlySpan <byte> buffer, JsonSerializerOptions options, ref ReadStack state, JsonConverter converterBase) { var reader = new Utf8JsonReader(buffer, isFinalBlock, readerState); // If we haven't read in the entire stream's payload we'll need to signify that we want // to enable read ahead behaviors to ensure we have complete json objects and arrays // ({}, []) when needed. (Notably to successfully parse JsonElement via JsonDocument // to assign to object and JsonElement properties in the constructed .NET object.) state.ReadAhead = !isFinalBlock; state.BytesConsumed = 0; TValue?value = ReadCore <TValue>(converterBase, ref reader, options, ref state); readerState = reader.CurrentState; return(value !); }
private static bool HandleObjectAsValue( JsonTokenType tokenType, JsonSerializerOptions options, ref Utf8JsonReader reader, ref ReadStack readStack, ref JsonReaderState initialState, long initialBytesConsumed) { if (readStack.ReadAhead) { // Attempt to skip to make sure we have all the data we need. bool complete = reader.TrySkip(); // We need to restore the state in all cases as we need to be positioned back before // the current token to either attempt to skip again or to actually read the value in // HandleValue below. reader = new Utf8JsonReader( reader.OriginalSpan.Slice(checked ((int)initialBytesConsumed)), isFinalBlock: reader.IsFinalBlock, state: initialState); Debug.Assert(reader.BytesConsumed == 0); readStack.BytesConsumed += initialBytesConsumed; if (!complete) { // Couldn't read to the end of the object, exit out to get more data in the buffer. return(false); } // Success, requeue the reader to the token for HandleValue. reader.Read(); Debug.Assert(tokenType == reader.TokenType); } HandleValue(tokenType, options, ref reader, ref readStack); return(true); }
private static void ReadCore( ref JsonReaderState readerState, bool isFinalBlock, Span <byte> buffer, JsonSerializerOptions options, ref ReadStack readStack) { var reader = new Utf8JsonReader(buffer, isFinalBlock, readerState); // If we haven't read in the entire stream's payload we'll need to signify that we want // to enable read ahead behaviors to ensure we have complete json objects and arrays // ({}, []) when needed. (Notably to successfully parse JsonElement via JsonDocument // to assign to object and JsonElement properties in the constructed .NET object.) readStack.ReadAhead = !isFinalBlock; readStack.BytesConsumed = 0; ReadCore( options, ref reader, ref readStack); readerState = reader.CurrentState; }
static async IAsyncEnumerable <TValue> CreateAsyncEnumerableDeserializer( Stream utf8Json, JsonSerializerOptions options, [EnumeratorCancellation] CancellationToken cancellationToken) { var bufferState = new ReadBufferState(options.DefaultBufferSize); // Hardcode the queue converter to avoid accidental use of custom converters JsonConverter converter = QueueOfTConverter <Queue <TValue>, TValue> .Instance; JsonTypeInfo jsonTypeInfo = CreateQueueJsonTypeInfo <TValue>(converter, options); ReadStack readStack = default; jsonTypeInfo.EnsureConfigured(); readStack.Initialize(jsonTypeInfo, supportContinuation: true); var jsonReaderState = new JsonReaderState(options.GetReaderOptions()); try { do { bufferState = await ReadFromStreamAsync(utf8Json, bufferState, cancellationToken).ConfigureAwait(false); ContinueDeserialize <Queue <TValue> >(ref bufferState, ref jsonReaderState, ref readStack, converter, options); if (readStack.Current.ReturnValue is Queue <TValue> queue) { while (queue.Count > 0) { yield return(queue.Dequeue()); } } }while (!bufferState.IsFinalBlock); } finally { bufferState.Dispose(); } }
/// <summary> /// Parse the UTF-8 encoded text representing a single JSON value into a <paramref name="returnType"/>. /// </summary> /// <returns>A <paramref name="returnType"/> representation of the JSON value.</returns> /// <param name="utf8Json">JSON text to parse.</param> /// <param name="returnType">The type of the object to convert to and return.</param> /// <param name="context">A metadata provider for serializable types.</param> /// <exception cref="System.ArgumentNullException"> /// <paramref name="returnType"/> is <see langword="null"/>. /// </exception> /// <exception cref="JsonException"> /// Thrown when the JSON is invalid, /// <paramref name="returnType"/> is not compatible with the JSON, /// or when there is remaining data in the Stream. /// </exception> /// <exception cref="NotSupportedException"> /// There is no compatible <see cref="System.Text.Json.Serialization.JsonConverter"/> /// for <paramref name="returnType"/> or its serializable members. /// </exception> /// <exception cref="InvalidOperationException"> /// The <see cref="JsonSerializerContext.GetTypeInfo(Type)"/> method on the provided <paramref name="context"/> /// did not return a compatible <see cref="JsonTypeInfo"/> for <paramref name="returnType"/>. /// </exception> public static object?Deserialize(ReadOnlySpan <byte> utf8Json, Type returnType, JsonSerializerContext context) { if (returnType == null) { throw new ArgumentNullException(nameof(returnType)); } if (context == null) { throw new ArgumentNullException(nameof(context)); } JsonTypeInfo jsonTypeInfo = JsonHelpers.GetTypeInfo(context, returnType); JsonSerializerOptions options = jsonTypeInfo.Options; var readerState = new JsonReaderState(options.GetReaderOptions()); var reader = new Utf8JsonReader(utf8Json, isFinalBlock: true, readerState); ReadStack state = default; state.Initialize(jsonTypeInfo); return(ReadCore <object?>(jsonTypeInfo.PropertyInfoForTypeInfo.ConverterBase, ref reader, options, ref state)); }
private static void ReadCore( JsonSerializerOptions options, ref Utf8JsonReader reader, ref ReadStack readStack) { try { JsonReaderState initialState = default; long initialBytesConsumed = default; while (true) { if (readStack.ReadAhead) { // When we're reading ahead we always have to save the state // as we don't know if the next token is an opening object or // array brace. initialState = reader.CurrentState; initialBytesConsumed = reader.BytesConsumed; } if (!reader.Read()) { // Need more data break; } JsonTokenType tokenType = reader.TokenType; if (options.ReferenceHandling.ShouldReadPreservedReferences()) { CheckValidTokenAfterMetadataValues(ref readStack, tokenType); } if (JsonHelpers.IsInRangeInclusive(tokenType, JsonTokenType.String, JsonTokenType.False)) { Debug.Assert(tokenType == JsonTokenType.String || tokenType == JsonTokenType.Number || tokenType == JsonTokenType.True || tokenType == JsonTokenType.False); HandleValue(tokenType, options, ref reader, ref readStack); } else if (tokenType == JsonTokenType.PropertyName) { HandlePropertyName(options, ref reader, ref readStack); } else if (tokenType == JsonTokenType.StartObject) { if (readStack.Current.SkipProperty) { readStack.Push(); readStack.Current.Drain = true; } else if (readStack.Current.IsProcessingValue()) { if (!HandleObjectAsValue(tokenType, options, ref reader, ref readStack, ref initialState, initialBytesConsumed)) { // Need more data break; } } else if (readStack.Current.IsProcessingDictionary()) { HandleStartDictionary(options, ref readStack); } else { HandleStartObject(options, ref readStack); } } else if (tokenType == JsonTokenType.EndObject) { if (readStack.Current.Drain) { readStack.Pop(); // Clear the current property in case it is a dictionary, since dictionaries must have EndProperty() called when completed. // A non-dictionary property can also have EndProperty() called when completed, although it is redundant. readStack.Current.EndProperty(); } else if (readStack.Current.ReferenceId != null) { Debug.Assert(options.ReferenceHandling.ShouldReadPreservedReferences()); HandleReference(ref readStack); } else if (readStack.Current.IsProcessingDictionary()) { HandleEndDictionary(options, ref readStack); } else { HandleEndObject(ref readStack); } } else if (tokenType == JsonTokenType.StartArray) { if (!readStack.Current.IsProcessingValue()) { HandleStartArray(options, ref readStack); } else if (!HandleObjectAsValue(tokenType, options, ref reader, ref readStack, ref initialState, initialBytesConsumed)) { // Need more data break; } } else if (tokenType == JsonTokenType.EndArray) { HandleEndArray(options, ref readStack); } else if (tokenType == JsonTokenType.Null) { HandleNull(options, ref reader, ref readStack); } } } catch (JsonReaderException ex) { // Re-throw with Path information. ThrowHelper.ReThrowWithPath(readStack, ex); } catch (FormatException ex) when(ex.Source == ThrowHelper.ExceptionSourceValueToRethrowAsJsonException) { ThrowHelper.ReThrowWithPath(readStack, reader, ex); } catch (InvalidOperationException ex) when(ex.Source == ThrowHelper.ExceptionSourceValueToRethrowAsJsonException) { ThrowHelper.ReThrowWithPath(readStack, reader, ex); } catch (JsonException ex) { ThrowHelper.AddExceptionInformation(readStack, reader, ex); throw; } readStack.BytesConsumed += reader.BytesConsumed; }
private static async ValueTask <TValue> ReadAsync <TValue>( Stream utf8Json, Type returnType, JsonSerializerOptions options = null, CancellationToken cancellationToken = default) { if (options == null) { options = JsonSerializerOptions.s_defaultOptions; } ReadStack readStack = default; readStack.Current.Initialize(returnType, options); var readerState = new JsonReaderState(options.GetReaderOptions()); // todo: switch to ArrayBuffer implementation to handle and simplify the allocs? int utf8BomLength = JsonConstants.Utf8Bom.Length; byte[] buffer = ArrayPool <byte> .Shared.Rent(Math.Max(options.DefaultBufferSize, utf8BomLength)); int bytesInBuffer = 0; long totalBytesRead = 0; int clearMax = 0; bool firstIteration = true; try { while (true) { // Read from the stream until either our buffer is filled or we hit EOF. // Calling ReadCore is relatively expensive, so we minimize the number of times // we need to call it. bool isFinalBlock = false; while (true) { int bytesRead = await utf8Json.ReadAsync( #if BUILDING_INBOX_LIBRARY buffer.AsMemory(bytesInBuffer), #else buffer, bytesInBuffer, buffer.Length - bytesInBuffer, #endif cancellationToken).ConfigureAwait(false); if (bytesRead == 0) { isFinalBlock = true; break; } totalBytesRead += bytesRead; bytesInBuffer += bytesRead; if (bytesInBuffer == buffer.Length) { break; } } if (bytesInBuffer > clearMax) { clearMax = bytesInBuffer; } int start = 0; if (firstIteration) { firstIteration = false; // Handle the UTF-8 BOM if present Debug.Assert(buffer.Length >= JsonConstants.Utf8Bom.Length); if (buffer.AsSpan().StartsWith(JsonConstants.Utf8Bom)) { start += utf8BomLength; bytesInBuffer -= utf8BomLength; } } // Process the data available ReadCore( ref readerState, isFinalBlock, new ReadOnlySpan <byte>(buffer, start, bytesInBuffer), options, ref readStack); Debug.Assert(readStack.BytesConsumed <= bytesInBuffer); int bytesConsumed = checked ((int)readStack.BytesConsumed); bytesInBuffer -= bytesConsumed; if (isFinalBlock) { break; } // Check if we need to shift or expand the buffer because there wasn't enough data to complete deserialization. if ((uint)bytesInBuffer > ((uint)buffer.Length / 2)) { // We have less than half the buffer available, double the buffer size. byte[] dest = ArrayPool <byte> .Shared.Rent((buffer.Length < (int.MaxValue / 2))?buffer.Length * 2 : int.MaxValue); // Copy the unprocessed data to the new buffer while shifting the processed bytes. Buffer.BlockCopy(buffer, bytesConsumed + start, dest, 0, bytesInBuffer); new Span <byte>(buffer, 0, clearMax).Clear(); ArrayPool <byte> .Shared.Return(buffer); clearMax = bytesInBuffer; buffer = dest; } else if (bytesInBuffer != 0) { // Shift the processed bytes to the beginning of buffer to make more room. Buffer.BlockCopy(buffer, bytesConsumed + start, buffer, 0, bytesInBuffer); } } } finally { // Clear only what we used and return the buffer to the pool new Span <byte>(buffer, 0, clearMax).Clear(); ArrayPool <byte> .Shared.Return(buffer); } // The reader should have thrown if we have remaining bytes. Debug.Assert(bytesInBuffer == 0); return((TValue)readStack.Current.ReturnValue); }
public static byte[] ReturnBytesHelper(byte[] data, out int length, JsonCommentHandling commentHandling = JsonCommentHandling.Disallow, int maxDepth = 64) { var state = new JsonReaderState(new JsonReaderOptions { CommentHandling = commentHandling, MaxDepth = maxDepth }); var reader = new Utf8JsonReader(data, true, state); return ReaderLoop(data.Length, out length, ref reader); }
private static bool TryParseValue(ref Utf8JsonReader reader, out JsonDocument document, bool shouldThrow) { JsonReaderState state = reader.CurrentState; CheckSupportedOptions(state.Options, nameof(reader)); // Value copy to overwrite the ref on an exception and undo the destructive reads. Utf8JsonReader restore = reader; // Only used for StartArray or StartObject, // the beginning of the token is one byte earlier. long startingOffset = state.BytesConsumed; ReadOnlySpan <byte> valueSpan = default; ReadOnlySequence <byte> valueSequence = default; try { switch (reader.TokenType) { // A new reader was created and has never been read, // so we need to move to the first token. // (or a reader has terminated and we're about to throw) case JsonTokenType.None: // Using a reader loop the caller has identified a property they wish to // hydrate into a JsonDocument. Move to the value first. case JsonTokenType.PropertyName: { if (!reader.Read()) { if (shouldThrow) { ThrowHelper.ThrowJsonReaderException( ref reader, ExceptionResource.ExpectedJsonTokens); } reader = restore; document = null; return(false); } // Reset the starting position since we moved. startingOffset = reader.BytesConsumed; break; } } switch (reader.TokenType) { // Any of the "value start" states are acceptable. case JsonTokenType.StartObject: case JsonTokenType.StartArray: { // Placeholder until reader.Skip() is written (#33295) { int depth = reader.CurrentDepth; // CurrentDepth rises late and falls fast, // a payload of "[ 1, 2, 3, 4 ]" will report post-Read() // CurrentDepth values of { 0, 1, 1, 1, 1, 0 }, // Since we're logically at 0 ([), Read() once and keep // reading until we've come back down to 0 (]). do { if (!reader.Read()) { if (shouldThrow) { ThrowHelper.ThrowJsonReaderException( ref reader, ExceptionResource.ExpectedJsonTokens); } reader = restore; document = null; return(false); } } while (reader.CurrentDepth > depth); } // Back up to be at the beginning of the { or [, vs the end. startingOffset--; long totalLength = reader.BytesConsumed - startingOffset; ReadOnlySequence <byte> sequence = reader.OriginalSequence; if (sequence.IsEmpty) { valueSpan = reader.OriginalSpan.Slice( checked ((int)startingOffset), checked ((int)totalLength)); } else { valueSequence = sequence.Slice(startingOffset, totalLength); } Debug.Assert( reader.TokenType == JsonTokenType.EndObject || reader.TokenType == JsonTokenType.EndArray); break; } // Single-token values case JsonTokenType.Number: case JsonTokenType.True: case JsonTokenType.False: case JsonTokenType.Null: { if (reader.HasValueSequence) { valueSequence = reader.ValueSequence; } else { valueSpan = reader.ValueSpan; } break; } // String's ValueSequence/ValueSpan omits the quotes, we need them back. case JsonTokenType.String: { ReadOnlySequence <byte> sequence = reader.OriginalSequence; if (sequence.IsEmpty) { // Since the quoted string fit in a ReadOnlySpan originally // the contents length plus the two quotes can't overflow. int payloadLength = reader.ValueSpan.Length + 2; Debug.Assert(payloadLength > 1); int openQuote = checked ((int)startingOffset) - payloadLength; ReadOnlySpan <byte> readerSpan = reader.OriginalSpan; Debug.Assert( readerSpan[openQuote] == (byte)'"', $"Calculated span starts with {readerSpan[openQuote]}"); Debug.Assert( readerSpan[(int)startingOffset - 1] == (byte)'"', $"Calculated span ends with {readerSpan[(int)startingOffset - 1]}"); valueSpan = readerSpan.Slice(openQuote, payloadLength); } else { long payloadLength = 2; if (reader.HasValueSequence) { payloadLength += reader.ValueSequence.Length; } else { payloadLength += reader.ValueSpan.Length; } valueSequence = sequence.Slice(startingOffset - payloadLength, payloadLength); Debug.Assert( valueSequence.First.Span[0] == (byte)'"', $"Calculated sequence starts with {valueSequence.First.Span[0]}"); } break; } default: { if (shouldThrow) { byte displayByte; if (reader.HasValueSequence) { displayByte = reader.ValueSequence.First.Span[0]; } else { displayByte = reader.ValueSpan[0]; } ThrowHelper.ThrowJsonReaderException( ref reader, ExceptionResource.ExpectedStartOfValueNotFound, displayByte); } reader = restore; document = null; return(false); } } } catch { reader = restore; throw; } int length = valueSpan.IsEmpty ? checked ((int)valueSequence.Length) : valueSpan.Length; byte[] rented = ArrayPool <byte> .Shared.Rent(length); Span <byte> rentedSpan = rented.AsSpan(0, length); try { if (valueSpan.IsEmpty) { valueSequence.CopyTo(rentedSpan); } else { valueSpan.CopyTo(rentedSpan); } document = Parse(rented.AsMemory(0, length), state.Options, rented); return(true); } catch { // This really shouldn't happen since the document was already checked // for consistency by Skip. But if data mutations happened just after // the calls to Read then the copy may not be valid. rentedSpan.Clear(); ArrayPool <byte> .Shared.Return(rented); throw; } }
private static void ReadCore( JsonSerializerOptions options, ref Utf8JsonReader reader, ref ReadStack readStack) { try { JsonReaderState initialState = default; long initialBytesConsumed = default; while (true) { if (readStack.ReadAhead) { // When we're reading ahead we always have to save the state // as we don't know if the next token is an opening object or // array brace. initialState = reader.CurrentState; initialBytesConsumed = reader.BytesConsumed; } if (!reader.Read()) { // Need more data break; } JsonTokenType tokenType = reader.TokenType; if (JsonHelpers.IsInRangeInclusive(tokenType, JsonTokenType.String, JsonTokenType.False)) { Debug.Assert(tokenType == JsonTokenType.String || tokenType == JsonTokenType.Number || tokenType == JsonTokenType.True || tokenType == JsonTokenType.False); HandleValue(tokenType, options, ref reader, ref readStack); } else if (tokenType == JsonTokenType.PropertyName) { HandlePropertyName(options, ref reader, ref readStack); } else if (tokenType == JsonTokenType.StartObject) { if (readStack.Current.SkipProperty) { readStack.Push(); readStack.Current.Drain = true; } else if (readStack.Current.IsProcessingValue(tokenType)) { if (!HandleObjectAsValue(tokenType, options, ref reader, ref readStack, ref initialState, initialBytesConsumed)) { // Need more data break; } } else if (readStack.Current.IsProcessingDictionary || readStack.Current.IsProcessingIDictionaryConstructible) { HandleStartDictionary(options, ref reader, ref readStack); } else { HandleStartObject(options, ref reader, ref readStack); } } else if (tokenType == JsonTokenType.EndObject) { if (readStack.Current.Drain) { readStack.Pop(); } else if (readStack.Current.IsProcessingDictionary || readStack.Current.IsProcessingIDictionaryConstructible) { HandleEndDictionary(options, ref reader, ref readStack); } else { HandleEndObject(options, ref reader, ref readStack); } } else if (tokenType == JsonTokenType.StartArray) { if (!readStack.Current.IsProcessingValue(tokenType)) { HandleStartArray(options, ref reader, ref readStack); } else if (!HandleObjectAsValue(tokenType, options, ref reader, ref readStack, ref initialState, initialBytesConsumed)) { // Need more data break; } } else if (tokenType == JsonTokenType.EndArray) { HandleEndArray(options, ref reader, ref readStack); } else if (tokenType == JsonTokenType.Null) { HandleNull(ref reader, ref readStack, options); } } } catch (JsonReaderException ex) { // Re-throw with Path information. ThrowHelper.ReThrowWithPath(readStack, ex); } catch (FormatException ex) when(ex.Source == ThrowHelper.ExceptionSourceValueToRethrowAsJsonException) { ThrowHelper.ReThrowWithPath(readStack, reader, ex); } catch (InvalidOperationException ex) when(ex.Source == ThrowHelper.ExceptionSourceValueToRethrowAsJsonException) { ThrowHelper.ReThrowWithPath(readStack, reader, ex); } catch (JsonException ex) { ThrowHelper.AddExceptionInformation(readStack, reader, ex); throw; } readStack.BytesConsumed += reader.BytesConsumed; return; }
public static object ReturnObjectHelper(byte[] data, JsonCommentHandling commentHandling = JsonCommentHandling.Disallow) { var state = new JsonReaderState(options: new JsonReaderOptions { CommentHandling = commentHandling }); var reader = new Utf8JsonReader(data, true, state); return ReaderLoop(ref reader); }
internal static TValue ContinueDeserialize <TValue>( ref ReadBufferState bufferState, ref JsonReaderState jsonReaderState, ref ReadStack readStack, JsonConverter converter, JsonSerializerOptions options) { if (bufferState.BytesInBuffer > bufferState.ClearMax) { bufferState.ClearMax = bufferState.BytesInBuffer; } int start = 0; if (bufferState.IsFirstIteration) { bufferState.IsFirstIteration = false; // Handle the UTF-8 BOM if present Debug.Assert(bufferState.Buffer.Length >= JsonConstants.Utf8Bom.Length); if (bufferState.Buffer.AsSpan().StartsWith(JsonConstants.Utf8Bom)) { start += JsonConstants.Utf8Bom.Length; bufferState.BytesInBuffer -= JsonConstants.Utf8Bom.Length; } } // Process the data available TValue value = ReadCore <TValue>( ref jsonReaderState, bufferState.IsFinalBlock, new ReadOnlySpan <byte>(bufferState.Buffer, start, bufferState.BytesInBuffer), options, ref readStack, converter); Debug.Assert(readStack.BytesConsumed <= bufferState.BytesInBuffer); int bytesConsumed = checked ((int)readStack.BytesConsumed); bufferState.BytesInBuffer -= bytesConsumed; // The reader should have thrown if we have remaining bytes. Debug.Assert(!bufferState.IsFinalBlock || bufferState.BytesInBuffer == 0); if (!bufferState.IsFinalBlock) { // Check if we need to shift or expand the buffer because there wasn't enough data to complete deserialization. if ((uint)bufferState.BytesInBuffer > ((uint)bufferState.Buffer.Length / 2)) { // We have less than half the buffer available, double the buffer size. byte[] oldBuffer = bufferState.Buffer; int oldClearMax = bufferState.ClearMax; byte[] newBuffer = ArrayPool <byte> .Shared.Rent((bufferState.Buffer.Length < (int.MaxValue / 2))?bufferState.Buffer.Length * 2 : int.MaxValue); // Copy the unprocessed data to the new buffer while shifting the processed bytes. Buffer.BlockCopy(oldBuffer, bytesConsumed + start, newBuffer, 0, bufferState.BytesInBuffer); bufferState.Buffer = newBuffer; bufferState.ClearMax = bufferState.BytesInBuffer; // Clear and return the old buffer new Span <byte>(oldBuffer, 0, oldClearMax).Clear(); ArrayPool <byte> .Shared.Return(oldBuffer); } else if (bufferState.BytesInBuffer != 0) { // Shift the processed bytes to the beginning of buffer to make more room. Buffer.BlockCopy(bufferState.Buffer, bytesConsumed + start, bufferState.Buffer, 0, bufferState.BytesInBuffer); } } return(value); }
private static void ReadValueCore(JsonSerializerOptions options, ref Utf8JsonReader reader, ref ReadStack readStack) { JsonReaderState state = reader.CurrentState; CheckSupportedOptions(state.Options, nameof(reader)); // Value copy to overwrite the ref on an exception and undo the destructive reads. Utf8JsonReader restore = reader; ReadOnlySpan <byte> valueSpan = default; ReadOnlySequence <byte> valueSequence = default; try { switch (reader.TokenType) { // A new reader was created and has never been read, // so we need to move to the first token. // (or a reader has terminated and we're about to throw) case JsonTokenType.None: // Using a reader loop the caller has identified a property they wish to // hydrate into a JsonDocument. Move to the value first. case JsonTokenType.PropertyName: { if (!reader.Read()) { ThrowHelper.ThrowJsonReaderException(ref reader, ExceptionResource.ExpectedOneCompleteToken); } break; } } switch (reader.TokenType) { // Any of the "value start" states are acceptable. case JsonTokenType.StartObject: case JsonTokenType.StartArray: { long startingOffset = reader.TokenStartIndex; if (!reader.TrySkip()) { ThrowHelper.ThrowJsonReaderException(ref reader, ExceptionResource.NotEnoughData); } long totalLength = reader.BytesConsumed - startingOffset; ReadOnlySequence <byte> sequence = reader.OriginalSequence; if (sequence.IsEmpty) { valueSpan = reader.OriginalSpan.Slice( checked ((int)startingOffset), checked ((int)totalLength)); } else { valueSequence = sequence.Slice(startingOffset, totalLength); } Debug.Assert( reader.TokenType == JsonTokenType.EndObject || reader.TokenType == JsonTokenType.EndArray); break; } // Single-token values case JsonTokenType.Number: case JsonTokenType.True: case JsonTokenType.False: case JsonTokenType.Null: { if (reader.HasValueSequence) { valueSequence = reader.ValueSequence; } else { valueSpan = reader.ValueSpan; } break; } // String's ValueSequence/ValueSpan omits the quotes, we need them back. case JsonTokenType.String: { ReadOnlySequence <byte> sequence = reader.OriginalSequence; if (sequence.IsEmpty) { // Since the quoted string fit in a ReadOnlySpan originally // the contents length plus the two quotes can't overflow. int payloadLength = reader.ValueSpan.Length + 2; Debug.Assert(payloadLength > 1); ReadOnlySpan <byte> readerSpan = reader.OriginalSpan; Debug.Assert( readerSpan[(int)reader.TokenStartIndex] == (byte)'"', $"Calculated span starts with {readerSpan[(int)reader.TokenStartIndex]}"); Debug.Assert( readerSpan[(int)reader.TokenStartIndex + payloadLength - 1] == (byte)'"', $"Calculated span ends with {readerSpan[(int)reader.TokenStartIndex + payloadLength - 1]}"); valueSpan = readerSpan.Slice((int)reader.TokenStartIndex, payloadLength); } else { long payloadLength = 2; if (reader.HasValueSequence) { payloadLength += reader.ValueSequence.Length; } else { payloadLength += reader.ValueSpan.Length; } valueSequence = sequence.Slice(reader.TokenStartIndex, payloadLength); Debug.Assert( valueSequence.First.Span[0] == (byte)'"', $"Calculated sequence starts with {valueSequence.First.Span[0]}"); Debug.Assert( valueSequence.ToArray()[payloadLength - 1] == (byte)'"', $"Calculated sequence ends with {valueSequence.ToArray()[payloadLength - 1]}"); } break; } default: { byte displayByte; if (reader.HasValueSequence) { displayByte = reader.ValueSequence.First.Span[0]; } else { displayByte = reader.ValueSpan[0]; } ThrowHelper.ThrowJsonReaderException( ref reader, ExceptionResource.ExpectedStartOfValueNotFound, displayByte); break; } } } catch (JsonReaderException ex) { reader = restore; // Re-throw with Path information. ThrowHelper.ReThrowWithPath(readStack, ex); } int length = valueSpan.IsEmpty ? checked ((int)valueSequence.Length) : valueSpan.Length; byte[] rented = ArrayPool <byte> .Shared.Rent(length); Span <byte> rentedSpan = rented.AsSpan(0, length); try { if (valueSpan.IsEmpty) { valueSequence.CopyTo(rentedSpan); } else { valueSpan.CopyTo(rentedSpan); } var newReader = new Utf8JsonReader(rentedSpan, isFinalBlock: true, state: default); ReadCore(options, ref newReader, ref readStack); // The reader should have thrown if we have remaining bytes. Debug.Assert(newReader.BytesConsumed == length); } catch (JsonException) { reader = restore; throw; } finally { rentedSpan.Clear(); ArrayPool <byte> .Shared.Return(rented); } }
internal static bool TryParseValue( ref Utf8JsonReader reader, [NotNullWhen(true)] out JsonDocument?document, bool shouldThrow, bool useArrayPools) { JsonReaderState state = reader.CurrentState; CheckSupportedOptions(state.Options, nameof(reader)); // Value copy to overwrite the ref on an exception and undo the destructive reads. Utf8JsonReader restore = reader; ReadOnlySpan <byte> valueSpan = default; ReadOnlySequence <byte> valueSequence = default; try { switch (reader.TokenType) { // A new reader was created and has never been read, // so we need to move to the first token. // (or a reader has terminated and we're about to throw) case JsonTokenType.None: // Using a reader loop the caller has identified a property they wish to // hydrate into a JsonDocument. Move to the value first. case JsonTokenType.PropertyName: { if (!reader.Read()) { if (shouldThrow) { ThrowHelper.ThrowJsonReaderException( ref reader, ExceptionResource.ExpectedJsonTokens); } reader = restore; document = null; return(false); } break; } } switch (reader.TokenType) { // Any of the "value start" states are acceptable. case JsonTokenType.StartObject: case JsonTokenType.StartArray: { long startingOffset = reader.TokenStartIndex; if (!reader.TrySkip()) { if (shouldThrow) { ThrowHelper.ThrowJsonReaderException( ref reader, ExceptionResource.ExpectedJsonTokens); } reader = restore; document = null; return(false); } long totalLength = reader.BytesConsumed - startingOffset; ReadOnlySequence <byte> sequence = reader.OriginalSequence; if (sequence.IsEmpty) { valueSpan = reader.OriginalSpan.Slice( checked ((int)startingOffset), checked ((int)totalLength)); } else { valueSequence = sequence.Slice(startingOffset, totalLength); } Debug.Assert( reader.TokenType == JsonTokenType.EndObject || reader.TokenType == JsonTokenType.EndArray); break; } case JsonTokenType.False: case JsonTokenType.True: case JsonTokenType.Null: if (useArrayPools) { if (reader.HasValueSequence) { valueSequence = reader.ValueSequence; } else { valueSpan = reader.ValueSpan; } break; } document = CreateForLiteral(reader.TokenType); return(true); case JsonTokenType.Number: { if (reader.HasValueSequence) { valueSequence = reader.ValueSequence; } else { valueSpan = reader.ValueSpan; } break; } // String's ValueSequence/ValueSpan omits the quotes, we need them back. case JsonTokenType.String: { ReadOnlySequence <byte> sequence = reader.OriginalSequence; if (sequence.IsEmpty) { // Since the quoted string fit in a ReadOnlySpan originally // the contents length plus the two quotes can't overflow. int payloadLength = reader.ValueSpan.Length + 2; Debug.Assert(payloadLength > 1); ReadOnlySpan <byte> readerSpan = reader.OriginalSpan; Debug.Assert( readerSpan[(int)reader.TokenStartIndex] == (byte)'"', $"Calculated span starts with {readerSpan[(int)reader.TokenStartIndex]}"); Debug.Assert( readerSpan[(int)reader.TokenStartIndex + payloadLength - 1] == (byte)'"', $"Calculated span ends with {readerSpan[(int)reader.TokenStartIndex + payloadLength - 1]}"); valueSpan = readerSpan.Slice((int)reader.TokenStartIndex, payloadLength); } else { long payloadLength = 2; if (reader.HasValueSequence) { payloadLength += reader.ValueSequence.Length; } else { payloadLength += reader.ValueSpan.Length; } valueSequence = sequence.Slice(reader.TokenStartIndex, payloadLength); Debug.Assert( valueSequence.First.Span[0] == (byte)'"', $"Calculated sequence starts with {valueSequence.First.Span[0]}"); Debug.Assert( valueSequence.ToArray()[payloadLength - 1] == (byte)'"', $"Calculated sequence ends with {valueSequence.ToArray()[payloadLength - 1]}"); } break; } default: { if (shouldThrow) { // Default case would only hit if TokenType equals JsonTokenType.EndObject or JsonTokenType.EndArray in which case it would never be sequence Debug.Assert(!reader.HasValueSequence); byte displayByte = reader.ValueSpan[0]; ThrowHelper.ThrowJsonReaderException( ref reader, ExceptionResource.ExpectedStartOfValueNotFound, displayByte); } reader = restore; document = null; return(false); } } } catch { reader = restore; throw; } int length = valueSpan.IsEmpty ? checked ((int)valueSequence.Length) : valueSpan.Length; if (useArrayPools) { byte[] rented = ArrayPool <byte> .Shared.Rent(length); Span <byte> rentedSpan = rented.AsSpan(0, length); try { if (valueSpan.IsEmpty) { valueSequence.CopyTo(rentedSpan); } else { valueSpan.CopyTo(rentedSpan); } document = Parse(rented.AsMemory(0, length), state.Options, rented); } catch { // This really shouldn't happen since the document was already checked // for consistency by Skip. But if data mutations happened just after // the calls to Read then the copy may not be valid. rentedSpan.Clear(); ArrayPool <byte> .Shared.Return(rented); throw; } } else { byte[] owned; if (valueSpan.IsEmpty) { owned = valueSequence.ToArray(); } else { owned = valueSpan.ToArray(); } document = ParseUnrented(owned, state.Options, reader.TokenType); } return(true); }
/// <summary> /// Parse the text representing a single JSON value into a <paramref name="returnType"/>. /// </summary> /// <returns>A <paramref name="returnType"/> representation of the JSON value.</returns> /// <param name="json">JSON text to parse.</param> /// <param name="returnType">The type of the object to convert to and return.</param> /// <param name="options">Options to control the behavior during parsing.</param> /// <exception cref="System.ArgumentNullException"> /// Thrown if <paramref name="json"/> or <paramref name="returnType"/> is null. /// </exception> /// <exception cref="JsonException"> /// Thrown when the JSON is invalid, /// the <paramref name="returnType"/> is not compatible with the JSON, /// or when there is remaining data in the Stream. /// </exception> /// <remarks>Using a <see cref="string"/> is not as efficient as using the /// UTF-8 methods since the implementation natively uses UTF-8. /// </remarks> public static object Deserialize(string json, Type returnType, JsonSerializerOptions options = null) { if (json == null) { throw new ArgumentNullException(nameof(json)); } if (returnType == null) { throw new ArgumentNullException(nameof(returnType)); } if (options == null) { options = JsonSerializerOptions.s_defaultOptions; } object result; byte[] tempArray = null; int maxBytes; // For performance, avoid asking for the actual byte count unless necessary. if (json.Length > MaxArrayLengthBeforeCalculatingSize) { // Get the actual byte count in order to handle large input. maxBytes = JsonReaderHelper.GetUtf8ByteCount(json.AsSpan()); } else { maxBytes = json.Length * JsonConstants.MaxExpansionFactorWhileTranscoding; } Span <byte> utf8 = maxBytes <= JsonConstants.StackallocThreshold ? stackalloc byte[maxBytes] : (tempArray = ArrayPool <byte> .Shared.Rent(maxBytes)); try { int actualByteCount = JsonReaderHelper.GetUtf8FromText(json.AsSpan(), utf8); utf8 = utf8.Slice(0, actualByteCount); var readerState = new JsonReaderState(options.GetReaderOptions()); var reader = new Utf8JsonReader(utf8, isFinalBlock: true, readerState); result = ReadCore(returnType, options, ref reader); if (reader.BytesConsumed != actualByteCount) { ThrowHelper.ThrowJsonException_DeserializeDataRemaining( actualByteCount, actualByteCount - reader.BytesConsumed); } } finally { if (tempArray != null) { utf8.Clear(); ArrayPool <byte> .Shared.Return(tempArray); } } return(result); }