private static TestDom ReadJson(string jsonString)
{
var json = new TestDom();
if (string.IsNullOrEmpty(jsonString))
{
return(json);
}
var jsonReader = new JsonUtf8Reader(Encoding.UTF8.GetBytes(jsonString));
jsonReader.Read();
switch (jsonReader.TokenType)
{
case JsonTokenType.StartArray:
json.Array = ReadArray(ref jsonReader);
break;
case JsonTokenType.StartObject:
json.Object = ReadObject(ref jsonReader);
break;
default:
Assert.True(false, "The test JSON does not start with an array or object token");
break;
}
return(json);
}
private static Array ReadArray(ref JsonUtf8Reader jsonReader)
{
// NOTE: We should be sitting on a StartArray token.
Assert.Equal(JsonTokenType.StartArray, jsonReader.TokenType);
Array jsonArray = new Array();
List <Value> jsonValues = new List <Value>();
while (jsonReader.Read())
{
switch (jsonReader.TokenType)
{
case JsonTokenType.EndArray:
jsonArray.Values = jsonValues;
return(jsonArray);
case JsonTokenType.StartArray:
case JsonTokenType.StartObject:
case JsonTokenType.False:
case JsonTokenType.True:
case JsonTokenType.Null:
case JsonTokenType.Number:
case JsonTokenType.String:
jsonValues.Add(GetValue(ref jsonReader));
break;
default:
throw new ArgumentOutOfRangeException();
}
}
throw new FormatException("Json array was started but never ended.");
}
private static void GetJsonReaderException(ref JsonUtf8Reader json, ExceptionResource resource, byte nextByte, ReadOnlySpan <byte> bytes)
{
string message = GetResourceString(ref json, resource, (char)nextByte, Encoding.UTF8.GetString(bytes.ToArray(), 0, bytes.Length));
message += $" LineNumber: {json._lineNumber} | BytePosition: {json._position}.";
throw new JsonReaderException(message, json._lineNumber, json._position);
}
public byte[] ReaderSystemTextJsonLabReturnBytes()
{
var outputArray = new byte[_dataUtf8.Length * 2];
Span <byte> destination = outputArray;
var json = new JsonUtf8Reader(_dataUtf8);
while (json.Read())
{
JsonTokenType tokenType = json.TokenType;
ReadOnlySpan <byte> valueSpan = json.ValueSpan;
switch (tokenType)
{
case JsonTokenType.PropertyName:
valueSpan.CopyTo(destination);
destination[valueSpan.Length] = (byte)',';
destination[valueSpan.Length + 1] = (byte)' ';
destination = destination.Slice(valueSpan.Length + 2);
break;
case JsonTokenType.Number:
case JsonTokenType.String:
valueSpan.CopyTo(destination);
destination[valueSpan.Length] = (byte)',';
destination[valueSpan.Length + 1] = (byte)' ';
destination = destination.Slice(valueSpan.Length + 2);
break;
case JsonTokenType.True:
// Special casing True/False so that the casing matches with Json.NET
destination[0] = (byte)'T';
destination[1] = (byte)'r';
destination[2] = (byte)'u';
destination[3] = (byte)'e';
destination[valueSpan.Length] = (byte)',';
destination[valueSpan.Length + 1] = (byte)' ';
destination = destination.Slice(valueSpan.Length + 2);
break;
case JsonTokenType.False:
destination[0] = (byte)'F';
destination[1] = (byte)'a';
destination[2] = (byte)'l';
destination[3] = (byte)'s';
destination[4] = (byte)'e';
destination[valueSpan.Length] = (byte)',';
destination[valueSpan.Length + 1] = (byte)' ';
destination = destination.Slice(valueSpan.Length + 2);
break;
case JsonTokenType.Null:
// Special casing Null so that it matches what JSON.NET does
break;
default:
break;
}
}
return(outputArray);
}
private static (JsonReaderState, SequencePosition, string) ProcessDataSpan(ReadOnlySequence <byte> ros, bool isFinalBlock, JsonReaderState state = default)
{
var builder = new StringBuilder();
ReadOnlySpan <byte> leftOver = default;
byte[] pooledArray = null;
JsonUtf8Reader json = default;
long totalConsumed = 0;
foreach (ReadOnlyMemory <byte> mem in ros)
{
ReadOnlySpan <byte> span = mem.Span;
if (leftOver.Length > int.MaxValue - span.Length)
{
throw new ArgumentOutOfRangeException("Current sequence segment size is too large to fit left over data from the previous segment into a 2 GB buffer.");
}
pooledArray = ArrayPool <byte> .Shared.Rent(span.Length + leftOver.Length); // This is guaranteed to not overflow
Span <byte> bufferSpan = pooledArray.AsSpan(0, leftOver.Length + span.Length);
leftOver.CopyTo(bufferSpan);
span.CopyTo(bufferSpan.Slice(leftOver.Length));
json = new JsonUtf8Reader(bufferSpan, isFinalBlock, state);
while (json.Read())
{
switch (json.TokenType)
{
case JsonTokenType.PropertyName:
json.TryGetValueAsString(out string value);
builder.Append(value);
break;
}
}
if (json.BytesConsumed < bufferSpan.Length)
{
leftOver = bufferSpan.Slice((int)json.BytesConsumed);
}
else
{
leftOver = default;
}
totalConsumed += json.BytesConsumed;
Assert.Equal(json.BytesConsumed, json.CurrentState.BytesConsumed);
Assert.Equal(json.Position, json.CurrentState.Position);
if (pooledArray != null) // TODO: Will this work if data spans more than two segments?
{
ArrayPool <byte> .Shared.Return(pooledArray);
}
state = json.CurrentState;
}
return(json.CurrentState, ros.GetPosition(totalConsumed), builder.ToString());
}
public void ReaderSystemTextJsonLabSpanEmptyLoop()
{
var json = new JsonUtf8Reader(_dataUtf8);
while (json.Read())
{
;
}
}
public void ReaderSystemTextJsonLabMultiSpanSequenceEmptyLoop()
{
var json = new JsonUtf8Reader(_sequence);
while (json.Read())
{
;
}
}
public void ValidateReaderIndexOf()
{
var json = new JsonUtf8Reader(_dataUtf8);
while (json.Read())
{
;
}
}
public Utf8JsonReaderStream(Stream jsonStream)
{
if (!jsonStream.CanRead || !jsonStream.CanSeek)
{
JsonThrowHelper.ThrowArgumentException("Stream must be readable and seekable.");
}
_pooledArray = ArrayPool <byte> .Shared.Rent(FirstSegmentSize);
int numberOfBytes = jsonStream.Read(_pooledArray, 0, FirstSegmentSize);
_buffer = _pooledArray.AsSpan(0, numberOfBytes);
_stream = jsonStream;
_isFinalBlock = numberOfBytes == 0;
_jsonReader = new JsonUtf8Reader(_buffer, _isFinalBlock);
_consumed = 0;
}
private static (JsonReaderState, string) ProcessData(ReadOnlySequence <byte> ros, bool isFinalBlock, JsonReaderState state = default)
{
var builder = new StringBuilder();
var json = new JsonUtf8Reader(ros, isFinalBlock, state);
while (json.Read())
{
switch (json.TokenType)
{
case JsonTokenType.PropertyName:
json.TryGetValueAsString(out string value);
builder.Append(value);
break;
}
}
Assert.Equal(json.BytesConsumed, json.CurrentState.BytesConsumed);
Assert.Equal(json.Position, json.CurrentState.Position);
return(json.CurrentState, builder.ToString());
}
private bool ReadNext()
{
bool result = false;
do
{
_consumed += _jsonReader.BytesConsumed;
int leftOver = _buffer.Length - (int)_jsonReader.BytesConsumed;
int amountToRead = StreamSegmentSize;
if (leftOver > 0)
{
_stream.Position -= leftOver;
if (_jsonReader.BytesConsumed == 0)
{
if (leftOver > 1_000_000_000)
{
JsonThrowHelper.ThrowArgumentException("Cannot fit left over data from the previous chunk and the next chunk of data into a 2 GB buffer.");
}
// This is guaranteed to not overflow due to the check above.
amountToRead += leftOver * 2;
ResizeBuffer(amountToRead);
}
}
if (_pooledArray.Length < StreamSegmentSize)
{
ResizeBuffer(StreamSegmentSize);
}
int numberOfBytes = _stream.Read(_pooledArray, 0, amountToRead);
_isFinalBlock = numberOfBytes == 0; // TODO: Can this be inferred differently based on leftOver and numberOfBytes
_buffer = _pooledArray.AsSpan(0, numberOfBytes);
_jsonReader = new JsonUtf8Reader(_buffer, _isFinalBlock, _jsonReader.CurrentState);
result = _jsonReader.Read();
} while (!result && !_isFinalBlock);
return(result);
}
private static Value GetValue(ref JsonUtf8Reader jsonReader)
{
var value = new Value {
Type = MapValueType(jsonReader.TokenType)
};
switch (value.Type)
{
case Value.ValueType.String:
value.StringValue = ReadUtf8String(ref jsonReader);
break;
case Value.ValueType.Number:
CustomParser.TryParseDecimal(jsonReader.ValueSpan, out decimal num, out int consumed);
value.NumberValue = Convert.ToDouble(num);
break;
case Value.ValueType.True:
break;
case Value.ValueType.False:
break;
case Value.ValueType.Null:
break;
case Value.ValueType.Object:
value.ObjectValue = ReadObject(ref jsonReader);
break;
case Value.ValueType.Array:
value.ArrayValue = ReadArray(ref jsonReader);
break;
default:
throw new ArgumentOutOfRangeException();
}
return(value);
}
private static Object ReadObject(ref JsonUtf8Reader jsonReader)
{
// NOTE: We should be sitting on a StartObject token.
Assert.Equal(JsonTokenType.StartObject, jsonReader.TokenType);
var jsonObject = new Object();
List <Pair> jsonPairs = new List <Pair>();
while (jsonReader.Read())
{
switch (jsonReader.TokenType)
{
case JsonTokenType.EndObject:
jsonObject.Pairs = jsonPairs;
return(jsonObject);
case JsonTokenType.PropertyName:
ReadOnlyMemory <byte> name = ReadUtf8String(ref jsonReader);
jsonReader.Read(); // Move to value token
var pair = new Pair
{
Name = name,
Value = GetValue(ref jsonReader)
};
if (jsonPairs != null)
{
jsonPairs.Add(pair);
}
break;
default:
throw new ArgumentOutOfRangeException();
}
}
throw new FormatException("Json object was started but never ended.");
}
private static byte[] ReadUtf8String(ref JsonUtf8Reader jsonReader)
{
return(jsonReader.ValueSpan.ToArray());
}
public static JsonDynamicObject Parse(ReadOnlySpan <byte> utf8, int expectedNumberOfProperties = -1)
{
Stack <JsonDynamicObject> stack = new Stack <JsonDynamicObject>();
if (expectedNumberOfProperties == -1)
{
expectedNumberOfProperties = utf8.Length >> 3;
}
var properties = new Dictionary <JsonProperty, JsonValue>(expectedNumberOfProperties);
stack.Push(new JsonDynamicObject(properties));
var reader = new JsonUtf8Reader(utf8);
while (reader.Read())
{
switch (reader.TokenType)
{
case JsonTokenType.PropertyName:
var name = new Utf8String(reader.ValueSpan);
reader.Read(); // Move to the value token
var type = reader.TokenType;
var current = stack.Peek();
var property = new JsonProperty(current, name);
switch (type)
{
case JsonTokenType.String:
current._properties[property] = new JsonValue(new Utf8String(reader.ValueSpan));
break;
case JsonTokenType.StartObject: // TODO: could this be lazy? Could this reuse the root JsonObject (which would store non-allocating JsonDom)?
var newObj = new JsonDynamicObject(properties);
current._properties[property] = new JsonValue(newObj);
stack.Push(newObj);
break;
case JsonTokenType.True:
current._properties[property] = new JsonValue(type);
break;
case JsonTokenType.False:
current._properties[property] = new JsonValue(type);
break;
case JsonTokenType.Null:
current._properties[property] = new JsonValue(type);
break;
case JsonTokenType.Number:
current._properties[property] = new JsonValue(new Utf8String(reader.ValueSpan), type);
break;
case JsonTokenType.StartArray:
throw new NotImplementedException("array support not implemented yet.");
default:
throw new NotSupportedException();
}
break;
case JsonTokenType.StartObject:
break;
case JsonTokenType.EndObject:
if (stack.Count != 1)
{
stack.Pop();
}
break;
case JsonTokenType.StartArray:
throw new NotImplementedException("array support not implemented yet.");
case JsonTokenType.EndArray:
case JsonTokenType.String:
case JsonTokenType.True:
case JsonTokenType.False:
case JsonTokenType.Null:
case JsonTokenType.Number:
break;
default:
throw new NotSupportedException();
}
}
return(stack.Peek());
}
public static T Deserialize <T>(ReadOnlySpan <byte> utf8)
{
if (!TypeCache.TryGetValue(typeof(T), out Dictionary <int, PropertyInfoLinkedList> dictionary))
{
dictionary = GetTypeMap(typeof(T));
TypeCache.Add(typeof(T), dictionary);
}
T instance = Create <T> .CreateInstanceOfType(utf8);
var reader = new JsonUtf8Reader(utf8);
while (reader.Read())
{
switch (reader.TokenType)
{
case JsonTokenType.PropertyName:
int key = GetHashCode(reader.ValueSpan);
reader.Read(); // Move to the value token
JsonTokenType type = reader.TokenType;
switch (type)
{
case JsonTokenType.String:
PropertyInfo pi = GetPropertyInfo(dictionary, key, reader.ValueSpan);
pi.SetValue(instance, new Utf8String(reader.ValueSpan)); // TODO: Use Ref.Emit instead of Reflection
break;
case JsonTokenType.StartObject: // TODO: could this be lazy? Could this reuse the root JsonObject (which would store non-allocating JsonDom)?
throw new NotImplementedException("object support not implemented yet.");
case JsonTokenType.True:
pi = GetPropertyInfo(dictionary, key, reader.ValueSpan);
pi.SetValue(instance, true);
break;
case JsonTokenType.False:
pi = GetPropertyInfo(dictionary, key, reader.ValueSpan);
pi.SetValue(instance, false);
break;
case JsonTokenType.Null:
pi = GetPropertyInfo(dictionary, key, reader.ValueSpan);
pi.SetValue(instance, null);
break;
case JsonTokenType.Number:
pi = GetPropertyInfo(dictionary, key, reader.ValueSpan);
// TODO: Add support for other numeric types like double, long, etc.
if (!Utf8Parser.TryParse(reader.ValueSpan, out int result, out _))
{
throw new InvalidCastException();
}
pi.SetValue(instance, result);
break;
case JsonTokenType.StartArray:
throw new NotImplementedException("array support not implemented yet.");
default:
throw new NotSupportedException();
}
break;
case JsonTokenType.StartObject:
break;
case JsonTokenType.EndObject:
break;
case JsonTokenType.StartArray:
throw new NotImplementedException("array support not implemented yet.");
case JsonTokenType.EndArray:
case JsonTokenType.String:
case JsonTokenType.True:
case JsonTokenType.False:
case JsonTokenType.Null:
case JsonTokenType.Number:
break;
default:
throw new NotSupportedException();
}
}
return(instance);
}
private static string GetResourceString(ref JsonUtf8Reader json, ExceptionResource resource, char character, string characters)
{
Debug.Assert(Enum.IsDefined(typeof(ExceptionResource), resource),
"The enum value is not defined, please check the ExceptionResource Enum.");
string formatString = ExceptionStrings.ResourceManager.GetString(resource.ToString());
string message = formatString;
switch (resource)
{
case ExceptionResource.ArrayDepthTooLarge:
message = string.Format(formatString, json.CurrentDepth, json.MaxDepth);
break;
case ExceptionResource.ArrayEndWithinObject:
if (json.CurrentDepth <= 0)
{
formatString = ExceptionStrings.ResourceManager.GetString(ExceptionResource.DepthMustBePositive.ToString());
message = string.Format(formatString, json.CurrentDepth);
}
else
{
message = string.Format(formatString);
}
break;
case ExceptionResource.EndOfStringNotFound:
break;
case ExceptionResource.ExpectedDigitNotFound:
message = string.Format(formatString, character);
break;
case ExceptionResource.ExpectedDigitNotFoundEndOfData:
message = string.Format(formatString, character);
break;
case ExceptionResource.ExpectedEndAfterSingleJson:
message = string.Format(formatString, character);
break;
case ExceptionResource.ExpectedEndOfDigitNotFound:
message = string.Format(formatString, character);
break;
case ExceptionResource.ExpectedNextDigitComponentNotFound:
message = string.Format(formatString, character);
break;
case ExceptionResource.ExpectedNextDigitEValueNotFound:
message = string.Format(formatString, character);
break;
case ExceptionResource.ExpectedSeparaterAfterPropertyNameNotFound:
message = string.Format(formatString, character);
break;
case ExceptionResource.ExpectedStartOfPropertyNotFound:
message = string.Format(formatString, character);
break;
case ExceptionResource.ExpectedStartOfPropertyOrValueNotFound:
break;
case ExceptionResource.ExpectedStartOfValueNotFound:
message = string.Format(formatString, character);
break;
case ExceptionResource.ExpectedValueAfterPropertyNameNotFound:
break;
case ExceptionResource.FoundInvalidCharacter:
message = string.Format(formatString, character);
break;
case ExceptionResource.InvalidEndOfJson:
message = string.Format(formatString, json.TokenType);
break;
case ExceptionResource.ObjectDepthTooLarge:
message = string.Format(formatString, json.CurrentDepth, json.MaxDepth);
break;
case ExceptionResource.ObjectEndWithinArray:
if (json.CurrentDepth <= 0)
{
formatString = ExceptionStrings.ResourceManager.GetString(ExceptionResource.DepthMustBePositive.ToString());
message = string.Format(formatString, json.CurrentDepth);
}
else
{
message = string.Format(formatString);
}
break;
case ExceptionResource.Default:
break;
case ExceptionResource.ExpectedFalse:
message = string.Format(formatString, characters);
break;
case ExceptionResource.ExpectedNull:
message = string.Format(formatString, characters);
break;
case ExceptionResource.ExpectedTrue:
message = string.Format(formatString, characters);
break;
// This case is covered between ArrayEndWithinObject and ObjectEndWithinArray
/*case ExceptionResource.DepthMustBePositive:
* break;*/
case ExceptionResource.InvalidCharacterWithinString:
message = string.Format(formatString, character);
break;
case ExceptionResource.EndOfCommentNotFound:
break;
}
return(message);
}
public static void ThrowJsonReaderException(ref JsonUtf8Reader json, ExceptionResource resource = ExceptionResource.Default, byte nextByte = default, ReadOnlySpan <byte> bytes = default)
{
GetJsonReaderException(ref json, resource, nextByte, bytes);
}
