static async IAsyncEnumerable <TValue> CreateAsyncEnumerableDeserializer(
Stream utf8Json,
JsonSerializerOptions options,
[EnumeratorCancellation] CancellationToken cancellationToken)
{
var bufferState = new ReadBufferState(options.DefaultBufferSize);
// Hardcode the queue converter to avoid accidental use of custom converters
JsonConverter converter = QueueOfTConverter <Queue <TValue>, TValue> .Instance;
JsonTypeInfo jsonTypeInfo = CreateQueueJsonTypeInfo <TValue>(converter, options);
ReadStack readStack = default;
readStack.Initialize(jsonTypeInfo, supportContinuation: true);
var jsonReaderState = new JsonReaderState(options.GetReaderOptions());
try
{
do
{
bufferState = await ReadFromStreamAsync(utf8Json, bufferState, cancellationToken).ConfigureAwait(false);
ContinueDeserialize <Queue <TValue> >(ref bufferState, ref jsonReaderState, ref readStack, converter, options);
if (readStack.Current.ReturnValue is Queue <TValue> queue)
{
while (queue.Count > 0)
{
yield return(queue.Dequeue());
}
}
}while (!bufferState.IsFinalBlock);
}
finally
{
bufferState.Dispose();
}
}
private static TValue?ReadFromSpan <TValue>(ReadOnlySpan <byte> utf8Json, JsonTypeInfo jsonTypeInfo, int?actualByteCount = null)
{
JsonSerializerOptions options = jsonTypeInfo.Options;
var readerState = new JsonReaderState(options.GetReaderOptions());
var reader = new Utf8JsonReader(utf8Json, isFinalBlock: true, readerState);
ReadStack state = default;
state.Initialize(jsonTypeInfo);
TValue? value;
JsonConverter jsonConverter = jsonTypeInfo.PropertyInfoForTypeInfo.ConverterBase;
// For performance, the code below is a lifted ReadCore() above.
if (jsonConverter is JsonConverter <TValue> converter)
{
// Call the strongly-typed ReadCore that will not box structs.
value = converter.ReadCore(ref reader, options, ref state);
}
else
{
// The non-generic API was called or we have a polymorphic case where TValue is not equal to the T in JsonConverter<T>.
object?objValue = jsonConverter.ReadCoreAsObject(ref reader, options, ref state);
Debug.Assert(objValue == null || objValue is TValue);
value = (TValue?)objValue;
}
// The reader should have thrown if we have remaining bytes.
Debug.Assert(reader.BytesConsumed == (actualByteCount ?? utf8Json.Length));
return(value);
}
internal static TValue?ReadAll <TValue>(
Stream utf8Json,
JsonTypeInfo jsonTypeInfo)
{
JsonSerializerOptions options = jsonTypeInfo.Options;
var bufferState = new ReadBufferState(options.DefaultBufferSize);
ReadStack readStack = default;
jsonTypeInfo.EnsureConfigured();
readStack.Initialize(jsonTypeInfo, supportContinuation: true);
JsonConverter converter = readStack.Current.JsonPropertyInfo !.ConverterBase;
var jsonReaderState = new JsonReaderState(options.GetReaderOptions());
try
{
while (true)
{
bufferState = ReadFromStream(utf8Json, bufferState);
TValue value = ContinueDeserialize <TValue>(ref bufferState, ref jsonReaderState, ref readStack, converter, options);
if (bufferState.IsFinalBlock)
{
return(value !);
}
}
}
finally
{
bufferState.Dispose();
}
}
static async IAsyncEnumerable <TValue> CreateAsyncEnumerableDeserializer(
Stream utf8Json,
JsonSerializerOptions options,
[EnumeratorCancellation] CancellationToken cancellationToken)
{
var bufferState = new ReadAsyncBufferState(options.DefaultBufferSize);
ReadStack readStack = default;
readStack.Initialize(typeof(Queue <TValue>), options, supportContinuation: true);
JsonConverter converter = readStack.Current.JsonPropertyInfo !.ConverterBase;
var jsonReaderState = new JsonReaderState(options.GetReaderOptions());
try
{
do
{
bufferState = await ReadFromStreamAsync(utf8Json, bufferState, cancellationToken).ConfigureAwait(false);
ContinueDeserialize <Queue <TValue> >(ref bufferState, ref jsonReaderState, ref readStack, converter, options);
if (readStack.Current.ReturnValue is Queue <TValue> queue)
{
while (queue.Count > 0)
{
yield return(queue.Dequeue());
}
}
}while (!bufferState.IsFinalBlock);
}
finally
{
bufferState.Dispose();
}
}
private static TValue?ReadFromSpan <TValue>(ReadOnlySpan <byte> utf8Json, JsonTypeInfo jsonTypeInfo, int?actualByteCount = null)
{
Debug.Assert(jsonTypeInfo.IsConfigured);
JsonSerializerOptions options = jsonTypeInfo.Options;
var readerState = new JsonReaderState(options.GetReaderOptions());
var reader = new Utf8JsonReader(utf8Json, isFinalBlock: true, readerState);
ReadStack state = default;
state.Initialize(jsonTypeInfo);
TValue?value;
// For performance, the code below is a lifted ReadCore() above.
if (jsonTypeInfo is JsonTypeInfo <TValue> typedInfo)
{
// Call the strongly-typed ReadCore that will not box structs.
value = typedInfo.EffectiveConverter.ReadCore(ref reader, options, ref state);
}
else
{
// The non-generic API was called.
object?objValue = jsonTypeInfo.Converter.ReadCoreAsObject(ref reader, options, ref state);
Debug.Assert(objValue is null or TValue);
value = (TValue?)objValue;
}
// The reader should have thrown if we have remaining bytes.
Debug.Assert(reader.BytesConsumed == (actualByteCount ?? utf8Json.Length));
return(value);
}
/// <summary>
/// Parse the text representing a single JSON value into a <paramref name="returnType"/>.
/// </summary>
/// <returns>A <paramref name="returnType"/> representation of the JSON value.</returns>
/// <param name="json">JSON text to parse.</param>
/// <param name="returnType">The type of the object to convert to and return.</param>
/// <param name="options">Options to control the behavior during parsing.</param>
/// <exception cref="System.ArgumentNullException">
/// Thrown if <paramref name="json"/> or <paramref name="returnType"/> is null.
/// </exception>
/// <exception cref="JsonException">
/// Thrown when the JSON is invalid,
/// the <paramref name="returnType"/> is not compatible with the JSON,
/// or when there is remaining data in the Stream.
/// </exception>
/// <remarks>Using a <see cref="System.String"/> is not as efficient as using the
/// UTF-8 methods since the implementation natively uses UTF-8.
/// </remarks>
public static object Deserialize(string json, Type returnType, JsonSerializerOptions options = null)
{
const long ArrayPoolMaxSizeBeforeUsingNormalAlloc = 1024 * 1024;
if (json == null)
{
throw new ArgumentNullException(nameof(json));
}
if (returnType == null)
{
throw new ArgumentNullException(nameof(returnType));
}
if (options == null)
{
options = JsonSerializerOptions.s_defaultOptions;
}
object result;
byte[] tempArray = null;
// For performance, avoid obtaining actual byte count unless memory usage is higher than the threshold.
Span <byte> utf8 = json.Length <= (ArrayPoolMaxSizeBeforeUsingNormalAlloc / JsonConstants.MaxExpansionFactorWhileTranscoding) ?
// Use a pooled alloc.
tempArray = ArrayPool <byte> .Shared.Rent(json.Length *JsonConstants.MaxExpansionFactorWhileTranscoding) :
// Use a normal alloc since the pool would create a normal alloc anyway based on the threshold (per current implementation)
// and by using a normal alloc we can avoid the Clear().
new byte[JsonReaderHelper.GetUtf8ByteCount(json.AsSpan())];
try
{
int actualByteCount = JsonReaderHelper.GetUtf8FromText(json.AsSpan(), utf8);
utf8 = utf8.Slice(0, actualByteCount);
var readerState = new JsonReaderState(options.GetReaderOptions());
var reader = new Utf8JsonReader(utf8, isFinalBlock: true, readerState);
result = ReadCore(returnType, options, ref reader);
// The reader should have thrown if we have remaining bytes.
Debug.Assert(reader.BytesConsumed == actualByteCount);
}
finally
{
if (tempArray != null)
{
utf8.Clear();
ArrayPool <byte> .Shared.Return(tempArray);
}
}
return(result);
}
private static async IAsyncEnumerable <TValue> CreateAsyncEnumerableDeserializer <TValue>(
Stream utf8Json,
JsonTypeInfo jsonTypeInfo,
[EnumeratorCancellation] CancellationToken cancellationToken)
{
JsonSerializerOptions options = jsonTypeInfo.Options;
JsonTypeInfo <Queue <TValue> > queueTypeInfo =
JsonMetadataServices.CreateQueueInfo <Queue <TValue>, TValue>(
options: options,
collectionInfo: new()
{
ObjectCreator = () => new Queue <TValue>(),
ElementInfo = jsonTypeInfo,
NumberHandling = options.NumberHandling
});
var bufferState = new ReadBufferState(options.DefaultBufferSize);
ReadStack readStack = default;
queueTypeInfo.EnsureConfigured();
readStack.Initialize(queueTypeInfo, supportContinuation: true);
var jsonReaderState = new JsonReaderState(options.GetReaderOptions());
try
{
do
{
bufferState = await ReadFromStreamAsync(utf8Json, bufferState, cancellationToken).ConfigureAwait(false);
ContinueDeserialize <Queue <TValue> >(
ref bufferState,
ref jsonReaderState,
ref readStack,
queueTypeInfo.PropertyInfoForTypeInfo.ConverterBase,
options);
if (readStack.Current.ReturnValue is Queue <TValue> queue)
{
while (queue.Count > 0)
{
yield return(queue.Dequeue());
}
}
}while (!bufferState.IsFinalBlock);
}
finally
{
bufferState.Dispose();
}
}
private static object ParseCore(ReadOnlySpan <byte> utf8Json, Type returnType, JsonSerializerOptions options)
{
if (options == null)
{
options = JsonSerializerOptions.s_defaultOptions;
}
var readerState = new JsonReaderState(options.GetReaderOptions());
var reader = new Utf8JsonReader(utf8Json, isFinalBlock: true, readerState);
object result = ReadCore(returnType, options, ref reader);
// The reader should have thrown if we have remaining bytes.
Debug.Assert(reader.BytesConsumed == utf8Json.Length);
return(result);
}
private static TValue?ReadUsingMetadata <TValue>(ReadOnlySpan <byte> utf8Json, JsonTypeInfo jsonTypeInfo, int?actualByteCount = null)
{
JsonSerializerOptions options = jsonTypeInfo.Options;
var readerState = new JsonReaderState(options.GetReaderOptions());
var reader = new Utf8JsonReader(utf8Json, isFinalBlock: true, readerState);
ReadStack state = default;
state.Initialize(jsonTypeInfo);
TValue?value = ReadCore <TValue>(jsonTypeInfo.PropertyInfoForTypeInfo.ConverterBase, ref reader, options, ref state);
// The reader should have thrown if we have remaining bytes.
Debug.Assert(reader.BytesConsumed == (actualByteCount ?? utf8Json.Length));
return(value);
}
/// <summary>
/// Parse the UTF-8 encoded text representing a single JSON value into a <typeparamref name="TValue"/>.
/// </summary>
/// <returns>A <typeparamref name="TValue"/> representation of the JSON value.</returns>
/// <param name="utf8Json">JSON text to parse.</param>
/// <param name="jsonTypeInfo">Metadata about the type to convert.</param>
/// <exception cref="JsonException">
/// Thrown when the JSON is invalid,
/// <typeparamref name="TValue"/> is not compatible with the JSON,
/// or when there is remaining data in the Stream.
/// </exception>
/// <exception cref="NotSupportedException">
/// There is no compatible <see cref="System.Text.Json.Serialization.JsonConverter"/>
/// for <typeparamref name="TValue"/> or its serializable members.
/// </exception>
public static TValue?Deserialize <TValue>(ReadOnlySpan <byte> utf8Json, JsonTypeInfo <TValue> jsonTypeInfo)
{
if (jsonTypeInfo == null)
{
throw new ArgumentNullException(nameof(jsonTypeInfo));
}
JsonSerializerOptions options = jsonTypeInfo.Options;
var readerState = new JsonReaderState(options.GetReaderOptions());
var reader = new Utf8JsonReader(utf8Json, isFinalBlock: true, readerState);
ReadStack state = default;
state.Initialize(jsonTypeInfo);
return(ReadCore <TValue>(jsonTypeInfo.PropertyInfoForTypeInfo.ConverterBase, ref reader, options, ref state));
}
private static TValue?Deserialize <TValue>(
JsonConverter jsonConverter,
ReadOnlySpan <char> json,
JsonSerializerOptions options,
ref ReadStack state)
{
const long ArrayPoolMaxSizeBeforeUsingNormalAlloc = 1024 * 1024;
byte[]? tempArray = null;
// For performance, avoid obtaining actual byte count unless memory usage is higher than the threshold.
Span <byte> utf8 = json.Length <= (ArrayPoolMaxSizeBeforeUsingNormalAlloc / JsonConstants.MaxExpansionFactorWhileTranscoding) ?
// Use a pooled alloc.
tempArray = ArrayPool <byte> .Shared.Rent(json.Length *JsonConstants.MaxExpansionFactorWhileTranscoding) :
// Use a normal alloc since the pool would create a normal alloc anyway based on the threshold (per current implementation)
// and by using a normal alloc we can avoid the Clear().
new byte[JsonReaderHelper.GetUtf8ByteCount(json)];
try
{
int actualByteCount = JsonReaderHelper.GetUtf8FromText(json, utf8);
utf8 = utf8.Slice(0, actualByteCount);
var readerState = new JsonReaderState(options.GetReaderOptions());
var reader = new Utf8JsonReader(utf8, isFinalBlock: true, readerState);
TValue?value = ReadCore <TValue>(jsonConverter, ref reader, options, ref state);
// The reader should have thrown if we have remaining bytes.
Debug.Assert(reader.BytesConsumed == actualByteCount);
return(value);
}
finally
{
if (tempArray != null)
{
utf8.Clear();
ArrayPool <byte> .Shared.Return(tempArray);
}
}
}
/// <summary>
/// Parse the UTF-8 encoded text representing a single JSON value into a <paramref name="returnType"/>.
/// </summary>
/// <returns>A <paramref name="returnType"/> representation of the JSON value.</returns>
/// <param name="utf8Json">JSON text to parse.</param>
/// <param name="returnType">The type of the object to convert to and return.</param>
/// <param name="context">A metadata provider for serializable types.</param>
/// <exception cref="System.ArgumentNullException">
/// <paramref name="returnType"/> is <see langword="null"/>.
/// </exception>
/// <exception cref="JsonException">
/// Thrown when the JSON is invalid,
/// <paramref name="returnType"/> is not compatible with the JSON,
/// or when there is remaining data in the Stream.
/// </exception>
/// <exception cref="NotSupportedException">
/// There is no compatible <see cref="System.Text.Json.Serialization.JsonConverter"/>
/// for <paramref name="returnType"/> or its serializable members.
/// </exception>
/// <exception cref="InvalidOperationException">
/// The <see cref="JsonSerializerContext.GetTypeInfo(Type)"/> method on the provided <paramref name="context"/>
/// did not return a compatible <see cref="JsonTypeInfo"/> for <paramref name="returnType"/>.
/// </exception>
public static object?Deserialize(ReadOnlySpan <byte> utf8Json, Type returnType, JsonSerializerContext context)
{
if (returnType == null)
{
throw new ArgumentNullException(nameof(returnType));
}
if (context == null)
{
throw new ArgumentNullException(nameof(context));
}
JsonTypeInfo jsonTypeInfo = JsonHelpers.GetTypeInfo(context, returnType);
JsonSerializerOptions options = jsonTypeInfo.Options;
var readerState = new JsonReaderState(options.GetReaderOptions());
var reader = new Utf8JsonReader(utf8Json, isFinalBlock: true, readerState);
ReadStack state = default;
state.Initialize(jsonTypeInfo);
return(ReadCore <object?>(jsonTypeInfo.PropertyInfoForTypeInfo.ConverterBase, ref reader, options, ref state));
}
private static async ValueTask <TValue> ReadAsync <TValue>(
Stream utf8Json,
Type returnType,
JsonSerializerOptions options = null,
CancellationToken cancellationToken = default)
{
if (options == null)
{
options = JsonSerializerOptions.s_defaultOptions;
}
ReadStack readStack = default;
readStack.Current.Initialize(returnType, options);
var readerState = new JsonReaderState(options.GetReaderOptions());
// todo: switch to ArrayBuffer implementation to handle and simplify the allocs?
int utf8BomLength = JsonConstants.Utf8Bom.Length;
byte[] buffer = ArrayPool <byte> .Shared.Rent(Math.Max(options.DefaultBufferSize, utf8BomLength));
int bytesInBuffer = 0;
long totalBytesRead = 0;
int clearMax = 0;
bool firstIteration = true;
try
{
while (true)
{
// Read from the stream until either our buffer is filled or we hit EOF.
// Calling ReadCore is relatively expensive, so we minimize the number of times
// we need to call it.
bool isFinalBlock = false;
while (true)
{
int bytesRead = await utf8Json.ReadAsync(
#if BUILDING_INBOX_LIBRARY
buffer.AsMemory(bytesInBuffer),
#else
buffer, bytesInBuffer, buffer.Length - bytesInBuffer,
#endif
cancellationToken).ConfigureAwait(false);
if (bytesRead == 0)
{
isFinalBlock = true;
break;
}
totalBytesRead += bytesRead;
bytesInBuffer += bytesRead;
if (bytesInBuffer == buffer.Length)
{
break;
}
}
if (bytesInBuffer > clearMax)
{
clearMax = bytesInBuffer;
}
int start = 0;
if (firstIteration)
{
firstIteration = false;
// Handle the UTF-8 BOM if present
Debug.Assert(buffer.Length >= JsonConstants.Utf8Bom.Length);
if (buffer.AsSpan().StartsWith(JsonConstants.Utf8Bom))
{
start += utf8BomLength;
bytesInBuffer -= utf8BomLength;
}
}
// Process the data available
ReadCore(
ref readerState,
isFinalBlock,
new ReadOnlySpan <byte>(buffer, start, bytesInBuffer),
options,
ref readStack);
Debug.Assert(readStack.BytesConsumed <= bytesInBuffer);
int bytesConsumed = checked ((int)readStack.BytesConsumed);
bytesInBuffer -= bytesConsumed;
if (isFinalBlock)
{
break;
}
// Check if we need to shift or expand the buffer because there wasn't enough data to complete deserialization.
if ((uint)bytesInBuffer > ((uint)buffer.Length / 2))
{
// We have less than half the buffer available, double the buffer size.
byte[] dest = ArrayPool <byte> .Shared.Rent((buffer.Length < (int.MaxValue / 2))?buffer.Length * 2 : int.MaxValue);
// Copy the unprocessed data to the new buffer while shifting the processed bytes.
Buffer.BlockCopy(buffer, bytesConsumed + start, dest, 0, bytesInBuffer);
new Span <byte>(buffer, 0, clearMax).Clear();
ArrayPool <byte> .Shared.Return(buffer);
clearMax = bytesInBuffer;
buffer = dest;
}
else if (bytesInBuffer != 0)
{
// Shift the processed bytes to the beginning of buffer to make more room.
Buffer.BlockCopy(buffer, bytesConsumed + start, buffer, 0, bytesInBuffer);
}
}
}
finally
{
// Clear only what we used and return the buffer to the pool
new Span <byte>(buffer, 0, clearMax).Clear();
ArrayPool <byte> .Shared.Return(buffer);
}
// The reader should have thrown if we have remaining bytes.
Debug.Assert(bytesInBuffer == 0);
return((TValue)readStack.Current.ReturnValue);
}
/// <summary>
/// Parse the text representing a single JSON value into a <paramref name="returnType"/>.
/// </summary>
/// <returns>A <paramref name="returnType"/> representation of the JSON value.</returns>
/// <param name="json">JSON text to parse.</param>
/// <param name="returnType">The type of the object to convert to and return.</param>
/// <param name="options">Options to control the behavior during parsing.</param>
/// <exception cref="System.ArgumentNullException">
/// Thrown if <paramref name="json"/> or <paramref name="returnType"/> is null.
/// </exception>
/// <exception cref="JsonException">
/// Thrown when the JSON is invalid,
/// the <paramref name="returnType"/> is not compatible with the JSON,
/// or when there is remaining data in the Stream.
/// </exception>
/// <remarks>Using a <see cref="string"/> is not as efficient as using the
/// UTF-8 methods since the implementation natively uses UTF-8.
/// </remarks>
public static object Deserialize(string json, Type returnType, JsonSerializerOptions options = null)
{
if (json == null)
{
throw new ArgumentNullException(nameof(json));
}
if (returnType == null)
{
throw new ArgumentNullException(nameof(returnType));
}
if (options == null)
{
options = JsonSerializerOptions.s_defaultOptions;
}
object result;
byte[] tempArray = null;
int maxBytes;
// For performance, avoid asking for the actual byte count unless necessary.
if (json.Length > MaxArrayLengthBeforeCalculatingSize)
{
// Get the actual byte count in order to handle large input.
maxBytes = JsonReaderHelper.GetUtf8ByteCount(json.AsSpan());
}
else
{
maxBytes = json.Length * JsonConstants.MaxExpansionFactorWhileTranscoding;
}
Span <byte> utf8 = maxBytes <= JsonConstants.StackallocThreshold ?
stackalloc byte[maxBytes] :
(tempArray = ArrayPool <byte> .Shared.Rent(maxBytes));
try
{
int actualByteCount = JsonReaderHelper.GetUtf8FromText(json.AsSpan(), utf8);
utf8 = utf8.Slice(0, actualByteCount);
var readerState = new JsonReaderState(options.GetReaderOptions());
var reader = new Utf8JsonReader(utf8, isFinalBlock: true, readerState);
result = ReadCore(returnType, options, ref reader);
if (reader.BytesConsumed != actualByteCount)
{
ThrowHelper.ThrowJsonException_DeserializeDataRemaining(
actualByteCount, actualByteCount - reader.BytesConsumed);
}
}
finally
{
if (tempArray != null)
{
utf8.Clear();
ArrayPool <byte> .Shared.Return(tempArray);
}
}
return(result);
}
