static async IAsyncEnumerable <TValue> CreateAsyncEnumerableDeserializer( Stream utf8Json, JsonSerializerOptions options, [EnumeratorCancellation] CancellationToken cancellationToken) { var bufferState = new ReadBufferState(options.DefaultBufferSize); // Hardcode the queue converter to avoid accidental use of custom converters JsonConverter converter = QueueOfTConverter <Queue <TValue>, TValue> .Instance; JsonTypeInfo jsonTypeInfo = CreateQueueJsonTypeInfo <TValue>(converter, options); ReadStack readStack = default; readStack.Initialize(jsonTypeInfo, supportContinuation: true); var jsonReaderState = new JsonReaderState(options.GetReaderOptions()); try { do { bufferState = await ReadFromStreamAsync(utf8Json, bufferState, cancellationToken).ConfigureAwait(false); ContinueDeserialize <Queue <TValue> >(ref bufferState, ref jsonReaderState, ref readStack, converter, options); if (readStack.Current.ReturnValue is Queue <TValue> queue) { while (queue.Count > 0) { yield return(queue.Dequeue()); } } }while (!bufferState.IsFinalBlock); } finally { bufferState.Dispose(); } }
private static TValue?ReadFromSpan <TValue>(ReadOnlySpan <byte> utf8Json, JsonTypeInfo jsonTypeInfo, int?actualByteCount = null) { JsonSerializerOptions options = jsonTypeInfo.Options; var readerState = new JsonReaderState(options.GetReaderOptions()); var reader = new Utf8JsonReader(utf8Json, isFinalBlock: true, readerState); ReadStack state = default; state.Initialize(jsonTypeInfo); TValue? value; JsonConverter jsonConverter = jsonTypeInfo.PropertyInfoForTypeInfo.ConverterBase; // For performance, the code below is a lifted ReadCore() above. if (jsonConverter is JsonConverter <TValue> converter) { // Call the strongly-typed ReadCore that will not box structs. value = converter.ReadCore(ref reader, options, ref state); } else { // The non-generic API was called or we have a polymorphic case where TValue is not equal to the T in JsonConverter<T>. object?objValue = jsonConverter.ReadCoreAsObject(ref reader, options, ref state); Debug.Assert(objValue == null || objValue is TValue); value = (TValue?)objValue; } // The reader should have thrown if we have remaining bytes. Debug.Assert(reader.BytesConsumed == (actualByteCount ?? utf8Json.Length)); return(value); }
internal static TValue?ReadAll <TValue>( Stream utf8Json, JsonTypeInfo jsonTypeInfo) { JsonSerializerOptions options = jsonTypeInfo.Options; var bufferState = new ReadBufferState(options.DefaultBufferSize); ReadStack readStack = default; jsonTypeInfo.EnsureConfigured(); readStack.Initialize(jsonTypeInfo, supportContinuation: true); JsonConverter converter = readStack.Current.JsonPropertyInfo !.ConverterBase; var jsonReaderState = new JsonReaderState(options.GetReaderOptions()); try { while (true) { bufferState = ReadFromStream(utf8Json, bufferState); TValue value = ContinueDeserialize <TValue>(ref bufferState, ref jsonReaderState, ref readStack, converter, options); if (bufferState.IsFinalBlock) { return(value !); } } } finally { bufferState.Dispose(); } }
static async IAsyncEnumerable <TValue> CreateAsyncEnumerableDeserializer( Stream utf8Json, JsonSerializerOptions options, [EnumeratorCancellation] CancellationToken cancellationToken) { var bufferState = new ReadAsyncBufferState(options.DefaultBufferSize); ReadStack readStack = default; readStack.Initialize(typeof(Queue <TValue>), options, supportContinuation: true); JsonConverter converter = readStack.Current.JsonPropertyInfo !.ConverterBase; var jsonReaderState = new JsonReaderState(options.GetReaderOptions()); try { do { bufferState = await ReadFromStreamAsync(utf8Json, bufferState, cancellationToken).ConfigureAwait(false); ContinueDeserialize <Queue <TValue> >(ref bufferState, ref jsonReaderState, ref readStack, converter, options); if (readStack.Current.ReturnValue is Queue <TValue> queue) { while (queue.Count > 0) { yield return(queue.Dequeue()); } } }while (!bufferState.IsFinalBlock); } finally { bufferState.Dispose(); } }
private static TValue?ReadFromSpan <TValue>(ReadOnlySpan <byte> utf8Json, JsonTypeInfo jsonTypeInfo, int?actualByteCount = null) { Debug.Assert(jsonTypeInfo.IsConfigured); JsonSerializerOptions options = jsonTypeInfo.Options; var readerState = new JsonReaderState(options.GetReaderOptions()); var reader = new Utf8JsonReader(utf8Json, isFinalBlock: true, readerState); ReadStack state = default; state.Initialize(jsonTypeInfo); TValue?value; // For performance, the code below is a lifted ReadCore() above. if (jsonTypeInfo is JsonTypeInfo <TValue> typedInfo) { // Call the strongly-typed ReadCore that will not box structs. value = typedInfo.EffectiveConverter.ReadCore(ref reader, options, ref state); } else { // The non-generic API was called. object?objValue = jsonTypeInfo.Converter.ReadCoreAsObject(ref reader, options, ref state); Debug.Assert(objValue is null or TValue); value = (TValue?)objValue; } // The reader should have thrown if we have remaining bytes. Debug.Assert(reader.BytesConsumed == (actualByteCount ?? utf8Json.Length)); return(value); }
/// <summary> /// Parse the text representing a single JSON value into a <paramref name="returnType"/>. /// </summary> /// <returns>A <paramref name="returnType"/> representation of the JSON value.</returns> /// <param name="json">JSON text to parse.</param> /// <param name="returnType">The type of the object to convert to and return.</param> /// <param name="options">Options to control the behavior during parsing.</param> /// <exception cref="System.ArgumentNullException"> /// Thrown if <paramref name="json"/> or <paramref name="returnType"/> is null. /// </exception> /// <exception cref="JsonException"> /// Thrown when the JSON is invalid, /// the <paramref name="returnType"/> is not compatible with the JSON, /// or when there is remaining data in the Stream. /// </exception> /// <remarks>Using a <see cref="System.String"/> is not as efficient as using the /// UTF-8 methods since the implementation natively uses UTF-8. /// </remarks> public static object Deserialize(string json, Type returnType, JsonSerializerOptions options = null) { const long ArrayPoolMaxSizeBeforeUsingNormalAlloc = 1024 * 1024; if (json == null) { throw new ArgumentNullException(nameof(json)); } if (returnType == null) { throw new ArgumentNullException(nameof(returnType)); } if (options == null) { options = JsonSerializerOptions.s_defaultOptions; } object result; byte[] tempArray = null; // For performance, avoid obtaining actual byte count unless memory usage is higher than the threshold. Span <byte> utf8 = json.Length <= (ArrayPoolMaxSizeBeforeUsingNormalAlloc / JsonConstants.MaxExpansionFactorWhileTranscoding) ? // Use a pooled alloc. tempArray = ArrayPool <byte> .Shared.Rent(json.Length *JsonConstants.MaxExpansionFactorWhileTranscoding) : // Use a normal alloc since the pool would create a normal alloc anyway based on the threshold (per current implementation) // and by using a normal alloc we can avoid the Clear(). new byte[JsonReaderHelper.GetUtf8ByteCount(json.AsSpan())]; try { int actualByteCount = JsonReaderHelper.GetUtf8FromText(json.AsSpan(), utf8); utf8 = utf8.Slice(0, actualByteCount); var readerState = new JsonReaderState(options.GetReaderOptions()); var reader = new Utf8JsonReader(utf8, isFinalBlock: true, readerState); result = ReadCore(returnType, options, ref reader); // The reader should have thrown if we have remaining bytes. Debug.Assert(reader.BytesConsumed == actualByteCount); } finally { if (tempArray != null) { utf8.Clear(); ArrayPool <byte> .Shared.Return(tempArray); } } return(result); }
private static async IAsyncEnumerable <TValue> CreateAsyncEnumerableDeserializer <TValue>( Stream utf8Json, JsonTypeInfo jsonTypeInfo, [EnumeratorCancellation] CancellationToken cancellationToken) { JsonSerializerOptions options = jsonTypeInfo.Options; JsonTypeInfo <Queue <TValue> > queueTypeInfo = JsonMetadataServices.CreateQueueInfo <Queue <TValue>, TValue>( options: options, collectionInfo: new() { ObjectCreator = () => new Queue <TValue>(), ElementInfo = jsonTypeInfo, NumberHandling = options.NumberHandling }); var bufferState = new ReadBufferState(options.DefaultBufferSize); ReadStack readStack = default; queueTypeInfo.EnsureConfigured(); readStack.Initialize(queueTypeInfo, supportContinuation: true); var jsonReaderState = new JsonReaderState(options.GetReaderOptions()); try { do { bufferState = await ReadFromStreamAsync(utf8Json, bufferState, cancellationToken).ConfigureAwait(false); ContinueDeserialize <Queue <TValue> >( ref bufferState, ref jsonReaderState, ref readStack, queueTypeInfo.PropertyInfoForTypeInfo.ConverterBase, options); if (readStack.Current.ReturnValue is Queue <TValue> queue) { while (queue.Count > 0) { yield return(queue.Dequeue()); } } }while (!bufferState.IsFinalBlock); } finally { bufferState.Dispose(); } }
private static object ParseCore(ReadOnlySpan <byte> utf8Json, Type returnType, JsonSerializerOptions options) { if (options == null) { options = JsonSerializerOptions.s_defaultOptions; } var readerState = new JsonReaderState(options.GetReaderOptions()); var reader = new Utf8JsonReader(utf8Json, isFinalBlock: true, readerState); object result = ReadCore(returnType, options, ref reader); // The reader should have thrown if we have remaining bytes. Debug.Assert(reader.BytesConsumed == utf8Json.Length); return(result); }
private static TValue?ReadUsingMetadata <TValue>(ReadOnlySpan <byte> utf8Json, JsonTypeInfo jsonTypeInfo, int?actualByteCount = null) { JsonSerializerOptions options = jsonTypeInfo.Options; var readerState = new JsonReaderState(options.GetReaderOptions()); var reader = new Utf8JsonReader(utf8Json, isFinalBlock: true, readerState); ReadStack state = default; state.Initialize(jsonTypeInfo); TValue?value = ReadCore <TValue>(jsonTypeInfo.PropertyInfoForTypeInfo.ConverterBase, ref reader, options, ref state); // The reader should have thrown if we have remaining bytes. Debug.Assert(reader.BytesConsumed == (actualByteCount ?? utf8Json.Length)); return(value); }
/// <summary> /// Parse the UTF-8 encoded text representing a single JSON value into a <typeparamref name="TValue"/>. /// </summary> /// <returns>A <typeparamref name="TValue"/> representation of the JSON value.</returns> /// <param name="utf8Json">JSON text to parse.</param> /// <param name="jsonTypeInfo">Metadata about the type to convert.</param> /// <exception cref="JsonException"> /// Thrown when the JSON is invalid, /// <typeparamref name="TValue"/> is not compatible with the JSON, /// or when there is remaining data in the Stream. /// </exception> /// <exception cref="NotSupportedException"> /// There is no compatible <see cref="System.Text.Json.Serialization.JsonConverter"/> /// for <typeparamref name="TValue"/> or its serializable members. /// </exception> public static TValue?Deserialize <TValue>(ReadOnlySpan <byte> utf8Json, JsonTypeInfo <TValue> jsonTypeInfo) { if (jsonTypeInfo == null) { throw new ArgumentNullException(nameof(jsonTypeInfo)); } JsonSerializerOptions options = jsonTypeInfo.Options; var readerState = new JsonReaderState(options.GetReaderOptions()); var reader = new Utf8JsonReader(utf8Json, isFinalBlock: true, readerState); ReadStack state = default; state.Initialize(jsonTypeInfo); return(ReadCore <TValue>(jsonTypeInfo.PropertyInfoForTypeInfo.ConverterBase, ref reader, options, ref state)); }
private static TValue?Deserialize <TValue>( JsonConverter jsonConverter, ReadOnlySpan <char> json, JsonSerializerOptions options, ref ReadStack state) { const long ArrayPoolMaxSizeBeforeUsingNormalAlloc = 1024 * 1024; byte[]? tempArray = null; // For performance, avoid obtaining actual byte count unless memory usage is higher than the threshold. Span <byte> utf8 = json.Length <= (ArrayPoolMaxSizeBeforeUsingNormalAlloc / JsonConstants.MaxExpansionFactorWhileTranscoding) ? // Use a pooled alloc. tempArray = ArrayPool <byte> .Shared.Rent(json.Length *JsonConstants.MaxExpansionFactorWhileTranscoding) : // Use a normal alloc since the pool would create a normal alloc anyway based on the threshold (per current implementation) // and by using a normal alloc we can avoid the Clear(). new byte[JsonReaderHelper.GetUtf8ByteCount(json)]; try { int actualByteCount = JsonReaderHelper.GetUtf8FromText(json, utf8); utf8 = utf8.Slice(0, actualByteCount); var readerState = new JsonReaderState(options.GetReaderOptions()); var reader = new Utf8JsonReader(utf8, isFinalBlock: true, readerState); TValue?value = ReadCore <TValue>(jsonConverter, ref reader, options, ref state); // The reader should have thrown if we have remaining bytes. Debug.Assert(reader.BytesConsumed == actualByteCount); return(value); } finally { if (tempArray != null) { utf8.Clear(); ArrayPool <byte> .Shared.Return(tempArray); } } }
/// <summary> /// Parse the UTF-8 encoded text representing a single JSON value into a <paramref name="returnType"/>. /// </summary> /// <returns>A <paramref name="returnType"/> representation of the JSON value.</returns> /// <param name="utf8Json">JSON text to parse.</param> /// <param name="returnType">The type of the object to convert to and return.</param> /// <param name="context">A metadata provider for serializable types.</param> /// <exception cref="System.ArgumentNullException"> /// <paramref name="returnType"/> is <see langword="null"/>. /// </exception> /// <exception cref="JsonException"> /// Thrown when the JSON is invalid, /// <paramref name="returnType"/> is not compatible with the JSON, /// or when there is remaining data in the Stream. /// </exception> /// <exception cref="NotSupportedException"> /// There is no compatible <see cref="System.Text.Json.Serialization.JsonConverter"/> /// for <paramref name="returnType"/> or its serializable members. /// </exception> /// <exception cref="InvalidOperationException"> /// The <see cref="JsonSerializerContext.GetTypeInfo(Type)"/> method on the provided <paramref name="context"/> /// did not return a compatible <see cref="JsonTypeInfo"/> for <paramref name="returnType"/>. /// </exception> public static object?Deserialize(ReadOnlySpan <byte> utf8Json, Type returnType, JsonSerializerContext context) { if (returnType == null) { throw new ArgumentNullException(nameof(returnType)); } if (context == null) { throw new ArgumentNullException(nameof(context)); } JsonTypeInfo jsonTypeInfo = JsonHelpers.GetTypeInfo(context, returnType); JsonSerializerOptions options = jsonTypeInfo.Options; var readerState = new JsonReaderState(options.GetReaderOptions()); var reader = new Utf8JsonReader(utf8Json, isFinalBlock: true, readerState); ReadStack state = default; state.Initialize(jsonTypeInfo); return(ReadCore <object?>(jsonTypeInfo.PropertyInfoForTypeInfo.ConverterBase, ref reader, options, ref state)); }
private static async ValueTask <TValue> ReadAsync <TValue>( Stream utf8Json, Type returnType, JsonSerializerOptions options = null, CancellationToken cancellationToken = default) { if (options == null) { options = JsonSerializerOptions.s_defaultOptions; } ReadStack readStack = default; readStack.Current.Initialize(returnType, options); var readerState = new JsonReaderState(options.GetReaderOptions()); // todo: switch to ArrayBuffer implementation to handle and simplify the allocs? int utf8BomLength = JsonConstants.Utf8Bom.Length; byte[] buffer = ArrayPool <byte> .Shared.Rent(Math.Max(options.DefaultBufferSize, utf8BomLength)); int bytesInBuffer = 0; long totalBytesRead = 0; int clearMax = 0; bool firstIteration = true; try { while (true) { // Read from the stream until either our buffer is filled or we hit EOF. // Calling ReadCore is relatively expensive, so we minimize the number of times // we need to call it. bool isFinalBlock = false; while (true) { int bytesRead = await utf8Json.ReadAsync( #if BUILDING_INBOX_LIBRARY buffer.AsMemory(bytesInBuffer), #else buffer, bytesInBuffer, buffer.Length - bytesInBuffer, #endif cancellationToken).ConfigureAwait(false); if (bytesRead == 0) { isFinalBlock = true; break; } totalBytesRead += bytesRead; bytesInBuffer += bytesRead; if (bytesInBuffer == buffer.Length) { break; } } if (bytesInBuffer > clearMax) { clearMax = bytesInBuffer; } int start = 0; if (firstIteration) { firstIteration = false; // Handle the UTF-8 BOM if present Debug.Assert(buffer.Length >= JsonConstants.Utf8Bom.Length); if (buffer.AsSpan().StartsWith(JsonConstants.Utf8Bom)) { start += utf8BomLength; bytesInBuffer -= utf8BomLength; } } // Process the data available ReadCore( ref readerState, isFinalBlock, new ReadOnlySpan <byte>(buffer, start, bytesInBuffer), options, ref readStack); Debug.Assert(readStack.BytesConsumed <= bytesInBuffer); int bytesConsumed = checked ((int)readStack.BytesConsumed); bytesInBuffer -= bytesConsumed; if (isFinalBlock) { break; } // Check if we need to shift or expand the buffer because there wasn't enough data to complete deserialization. if ((uint)bytesInBuffer > ((uint)buffer.Length / 2)) { // We have less than half the buffer available, double the buffer size. byte[] dest = ArrayPool <byte> .Shared.Rent((buffer.Length < (int.MaxValue / 2))?buffer.Length * 2 : int.MaxValue); // Copy the unprocessed data to the new buffer while shifting the processed bytes. Buffer.BlockCopy(buffer, bytesConsumed + start, dest, 0, bytesInBuffer); new Span <byte>(buffer, 0, clearMax).Clear(); ArrayPool <byte> .Shared.Return(buffer); clearMax = bytesInBuffer; buffer = dest; } else if (bytesInBuffer != 0) { // Shift the processed bytes to the beginning of buffer to make more room. Buffer.BlockCopy(buffer, bytesConsumed + start, buffer, 0, bytesInBuffer); } } } finally { // Clear only what we used and return the buffer to the pool new Span <byte>(buffer, 0, clearMax).Clear(); ArrayPool <byte> .Shared.Return(buffer); } // The reader should have thrown if we have remaining bytes. Debug.Assert(bytesInBuffer == 0); return((TValue)readStack.Current.ReturnValue); }
/// <summary> /// Parse the text representing a single JSON value into a <paramref name="returnType"/>. /// </summary> /// <returns>A <paramref name="returnType"/> representation of the JSON value.</returns> /// <param name="json">JSON text to parse.</param> /// <param name="returnType">The type of the object to convert to and return.</param> /// <param name="options">Options to control the behavior during parsing.</param> /// <exception cref="System.ArgumentNullException"> /// Thrown if <paramref name="json"/> or <paramref name="returnType"/> is null. /// </exception> /// <exception cref="JsonException"> /// Thrown when the JSON is invalid, /// the <paramref name="returnType"/> is not compatible with the JSON, /// or when there is remaining data in the Stream. /// </exception> /// <remarks>Using a <see cref="string"/> is not as efficient as using the /// UTF-8 methods since the implementation natively uses UTF-8. /// </remarks> public static object Deserialize(string json, Type returnType, JsonSerializerOptions options = null) { if (json == null) { throw new ArgumentNullException(nameof(json)); } if (returnType == null) { throw new ArgumentNullException(nameof(returnType)); } if (options == null) { options = JsonSerializerOptions.s_defaultOptions; } object result; byte[] tempArray = null; int maxBytes; // For performance, avoid asking for the actual byte count unless necessary. if (json.Length > MaxArrayLengthBeforeCalculatingSize) { // Get the actual byte count in order to handle large input. maxBytes = JsonReaderHelper.GetUtf8ByteCount(json.AsSpan()); } else { maxBytes = json.Length * JsonConstants.MaxExpansionFactorWhileTranscoding; } Span <byte> utf8 = maxBytes <= JsonConstants.StackallocThreshold ? stackalloc byte[maxBytes] : (tempArray = ArrayPool <byte> .Shared.Rent(maxBytes)); try { int actualByteCount = JsonReaderHelper.GetUtf8FromText(json.AsSpan(), utf8); utf8 = utf8.Slice(0, actualByteCount); var readerState = new JsonReaderState(options.GetReaderOptions()); var reader = new Utf8JsonReader(utf8, isFinalBlock: true, readerState); result = ReadCore(returnType, options, ref reader); if (reader.BytesConsumed != actualByteCount) { ThrowHelper.ThrowJsonException_DeserializeDataRemaining( actualByteCount, actualByteCount - reader.BytesConsumed); } } finally { if (tempArray != null) { utf8.Clear(); ArrayPool <byte> .Shared.Return(tempArray); } } return(result); }