public static JsonToken[] Tokenize(IJsonReader jsonReader, string json)
{
List <JsonToken> tokensFromReader = new List <JsonToken>();
while (jsonReader.Read())
{
switch (jsonReader.CurrentTokenType)
{
case JsonTokenType.NotStarted:
throw new ArgumentException(string.Format("Got an unexpected JsonTokenType: {0} as an expected token type", jsonReader.CurrentTokenType));
case JsonTokenType.BeginArray:
tokensFromReader.Add(JsonToken.ArrayStart());
break;
case JsonTokenType.EndArray:
tokensFromReader.Add(JsonToken.ArrayEnd());
break;
case JsonTokenType.BeginObject:
tokensFromReader.Add(JsonToken.ObjectStart());
break;
case JsonTokenType.EndObject:
tokensFromReader.Add(JsonToken.ObjectEnd());
break;
case JsonTokenType.String:
tokensFromReader.Add(JsonToken.String(jsonReader.GetStringValue()));
break;
case JsonTokenType.Number:
tokensFromReader.Add(JsonToken.Number(jsonReader.GetNumberValue()));
break;
case JsonTokenType.True:
tokensFromReader.Add(JsonToken.Boolean(true));
break;
case JsonTokenType.False:
tokensFromReader.Add(JsonToken.Boolean(false));
break;
case JsonTokenType.Null:
tokensFromReader.Add(JsonToken.Null());
break;
case JsonTokenType.FieldName:
tokensFromReader.Add(JsonToken.FieldName(jsonReader.GetStringValue()));
break;
default:
break;
}
}
return(tokensFromReader.ToArray());
}
internal static JsonTokenInfo[] GetTokensWithReader(IJsonReader jsonReader)
{
List <JsonTokenInfo> tokens = new List <JsonTokenInfo>();
while (jsonReader.Read())
{
switch (jsonReader.CurrentTokenType)
{
case JsonTokenType.NotStarted:
throw new InvalidOperationException();
case JsonTokenType.BeginArray:
tokens.Add(JsonTokenInfo.ArrayStart());
break;
case JsonTokenType.EndArray:
tokens.Add(JsonTokenInfo.ArrayEnd());
break;
case JsonTokenType.BeginObject:
tokens.Add(JsonTokenInfo.ObjectStart());
break;
case JsonTokenType.EndObject:
tokens.Add(JsonTokenInfo.ObjectEnd());
break;
case JsonTokenType.String:
tokens.Add(JsonTokenInfo.String(jsonReader.GetStringValue()));
break;
case JsonTokenType.Number:
tokens.Add(JsonTokenInfo.Number(jsonReader.GetNumberValue()));
break;
case JsonTokenType.True:
tokens.Add(JsonTokenInfo.Boolean(true));
break;
case JsonTokenType.False:
tokens.Add(JsonTokenInfo.Boolean(false));
break;
case JsonTokenType.Null:
tokens.Add(JsonTokenInfo.Null());
break;
case JsonTokenType.FieldName:
tokens.Add(JsonTokenInfo.FieldName(jsonReader.GetStringValue()));
break;
default:
break;
}
}
return(tokens.ToArray());
}
static void TokenizeInternal(IJsonReader jsonReader, List <JsonToken> tokenList)
{
while (jsonReader.Read())
{
switch (jsonReader.CurrentTokenType)
{
case JsonTokenType.NotStarted:
throw new ArgumentException(string.Format("Got an unexpected JsonTokenType: {0} as an expected token type", jsonReader.CurrentTokenType));
case JsonTokenType.BeginArray:
tokenList.Add(JsonToken.ArrayStart());
break;
case JsonTokenType.EndArray:
tokenList.Add(JsonToken.ArrayEnd());
break;
case JsonTokenType.BeginObject:
tokenList.Add(JsonToken.ObjectStart());
break;
case JsonTokenType.EndObject:
tokenList.Add(JsonToken.ObjectEnd());
break;
case JsonTokenType.String:
tokenList.Add(JsonToken.String(jsonReader.GetStringValue()));
break;
case JsonTokenType.Number:
tokenList.Add(JsonToken.Number(jsonReader.GetNumberValue()));
break;
case JsonTokenType.True:
tokenList.Add(JsonToken.Boolean(true));
break;
case JsonTokenType.False:
tokenList.Add(JsonToken.Boolean(false));
break;
case JsonTokenType.Null:
tokenList.Add(JsonToken.Null());
break;
case JsonTokenType.FieldName:
tokenList.Add(JsonToken.FieldName(jsonReader.GetStringValue()));
break;
default:
throw new ArgumentOutOfRangeException($"Unknown {nameof(JsonTokenType)}: '{jsonReader.CurrentTokenType}'.");
}
}
}
private static void RunReadBenchmark(
Payload payload,
bool useUtf8)
{
// Don't really need to test both serialization formats, since they are similiar.
IJsonReader jsonReader = JsonReader.Create(payload.Binary);
while (jsonReader.Read())
{
// Materialize the value
switch (jsonReader.CurrentTokenType)
{
case JsonTokenType.BeginArray:
case JsonTokenType.EndArray:
// Single byte tokens
break;
case JsonTokenType.String:
UtfAnyString value = jsonReader.GetStringValue();
if (useUtf8)
{
Utf8String _ = value.ToUtf8String();
}
else
{
string _ = value.ToString();
}
break;
default:
throw new ArgumentException("$Unknown token type.");
}
}
}
public void Utf8VsUtf16StringRead()
{
void RunPerf(string utf16String, JsonSerializationFormat jsonSerializationFormat, bool useUtf8)
{
Stopwatch stopWatch = new Stopwatch();
byte[] payload = JsonTestUtils.ConvertTextToBinary("\"" + utf16String + "\"");
for (int i = 0; i < 1000000; i++)
{
IJsonReader reader = JsonReader.Create(payload);
reader.Read();
stopWatch.Start();
if (useUtf8)
{
reader.TryGetBufferedUtf8StringValue(out ReadOnlyMemory <byte> bufferedUtf8StringValue);
}
else
{
string value = reader.GetStringValue();
}
stopWatch.Stop();
}
Console.WriteLine($"UTF {(useUtf8 ? 8 : 16)} {jsonSerializationFormat} reader + string length: {utf16String.Length} = {stopWatch.ElapsedMilliseconds} ms");
}
foreach (int stringLength in new int[] { 8, 32, 256, 1024, 4096 })
{
foreach (JsonSerializationFormat jsonSerializationFormat in new JsonSerializationFormat[] { JsonSerializationFormat.Text, JsonSerializationFormat.Binary })
{
foreach (bool useUtf8 in new bool[] { false, true })
{
RunPerf(
utf16String: new string('a', stringLength),
jsonSerializationFormat: jsonSerializationFormat,
useUtf8: useUtf8);
}
}
}
}
private static void RunReadBenchmark(
Payload payload,
bool useUtf8)
{
// Don't really need to test both serialization formats, since they are similiar.
IJsonReader jsonReader = JsonReader.Create(payload.Binary);
while (jsonReader.Read())
{
// Materialize the value
switch (jsonReader.CurrentTokenType)
{
case JsonTokenType.BeginArray:
case JsonTokenType.EndArray:
// Single byte tokens
break;
case JsonTokenType.String:
if (useUtf8)
{
if (!jsonReader.TryGetBufferedUtf8StringValue(out ReadOnlyMemory <byte> bufferedUtf8StringValue))
{
throw new InvalidOperationException("Failed to get utf8 string.");
}
}
else
{
string value = jsonReader.GetStringValue();
}
break;
default:
throw new ArgumentException("$Unknown token type.");
}
}
}
/// <summary>
/// Writes current token from a json reader to the internal buffer.
/// </summary>
/// <param name="jsonReader">The JsonReader to the get the current token from.</param>
public void WriteCurrentToken(IJsonReader jsonReader)
{
if (jsonReader == null)
{
throw new ArgumentNullException("jsonReader can not be null");
}
// For now we don't optimize for text, since the reader could be UTF-8 and the writer could be UTF-16.
// We need to add more enums for the different serialization formats.
bool sameFormat = jsonReader.SerializationFormat == this.SerializationFormat && (this.SerializationFormat == JsonSerializationFormat.Binary || this.SerializationFormat == JsonSerializationFormat.HybridRow);
JsonTokenType jsonTokenType = jsonReader.CurrentTokenType;
switch (jsonTokenType)
{
case JsonTokenType.NotStarted:
break;
case JsonTokenType.BeginArray:
this.WriteArrayStart();
break;
case JsonTokenType.EndArray:
this.WriteArrayEnd();
break;
case JsonTokenType.BeginObject:
this.WriteObjectStart();
break;
case JsonTokenType.EndObject:
this.WriteObjectEnd();
break;
case JsonTokenType.True:
this.WriteBoolValue(true);
break;
case JsonTokenType.False:
this.WriteBoolValue(false);
break;
case JsonTokenType.Null:
this.WriteNullValue();
break;
case JsonTokenType.String:
case JsonTokenType.Number:
case JsonTokenType.FieldName:
{
if (sameFormat)
{
IReadOnlyList <byte> bufferedRawJson = jsonReader.GetBufferedRawJsonToken();
this.WriteRawJsonToken(jsonTokenType, bufferedRawJson);
}
else
{
if (jsonTokenType == JsonTokenType.Number)
{
double number = jsonReader.GetNumberValue();
this.WriteNumberValue(number);
}
else
{
string value = jsonReader.GetStringValue();
if (jsonTokenType == JsonTokenType.FieldName)
{
this.WriteFieldName(value);
}
else
{
this.WriteStringValue(value);
}
}
}
break;
}
default:
throw new ArgumentException($"Unexpected JsonTokenType: {jsonTokenType}");
}
}
private static JsonToken[] GetTokensWithReader(IJsonReader jsonReader)
{
List <JsonToken> tokens = new List <JsonToken>();
while (jsonReader.Read())
{
JsonToken token;
switch (jsonReader.CurrentTokenType)
{
case JsonTokenType.NotStarted:
throw new InvalidOperationException();
case JsonTokenType.BeginArray:
token = JsonToken.ArrayStart();
break;
case JsonTokenType.EndArray:
token = JsonToken.ArrayEnd();
break;
case JsonTokenType.BeginObject:
token = JsonToken.ObjectStart();
break;
case JsonTokenType.EndObject:
token = JsonToken.ObjectEnd();
break;
case JsonTokenType.String:
token = JsonToken.String(jsonReader.GetStringValue());
break;
case JsonTokenType.Number:
token = JsonToken.Number(jsonReader.GetNumberValue());
break;
case JsonTokenType.True:
token = JsonToken.Boolean(true);
break;
case JsonTokenType.False:
token = JsonToken.Boolean(false);
break;
case JsonTokenType.Null:
token = JsonToken.Null();
break;
case JsonTokenType.FieldName:
token = JsonToken.FieldName(jsonReader.GetStringValue());
break;
case JsonTokenType.Int8:
token = JsonToken.Int8(jsonReader.GetInt8Value());
break;
case JsonTokenType.Int16:
token = JsonToken.Int16(jsonReader.GetInt16Value());
break;
case JsonTokenType.Int32:
token = JsonToken.Int32(jsonReader.GetInt32Value());
break;
case JsonTokenType.Int64:
token = JsonToken.Int64(jsonReader.GetInt64Value());
break;
case JsonTokenType.UInt32:
token = JsonToken.UInt32(jsonReader.GetUInt32Value());
break;
case JsonTokenType.Float32:
token = JsonToken.Float32(jsonReader.GetFloat32Value());
break;
case JsonTokenType.Float64:
token = JsonToken.Float64(jsonReader.GetFloat64Value());
break;
case JsonTokenType.Guid:
token = JsonToken.Guid(jsonReader.GetGuidValue());
break;
case JsonTokenType.Binary:
token = JsonToken.Binary(jsonReader.GetBinaryValue());
break;
default:
throw new ArgumentException($"Unknown {nameof(JsonTokenType)}: {jsonReader.CurrentTokenType}");
}
tokens.Add(token);
}
return(tokens.ToArray());
}
public static void DrainReader(IJsonReader jsonReader, bool materializeValue)
{
if (jsonReader == null)
{
throw new ArgumentNullException(nameof(jsonReader));
}
while (jsonReader.Read())
{
// Materialize the value
switch (jsonReader.CurrentTokenType)
{
case JsonTokenType.BeginArray:
case JsonTokenType.EndArray:
case JsonTokenType.BeginObject:
case JsonTokenType.EndObject:
case JsonTokenType.Null:
case JsonTokenType.True:
case JsonTokenType.False:
// Single byte tokens
break;
case JsonTokenType.String:
case JsonTokenType.FieldName:
if (materializeValue)
{
string stringValue = jsonReader.GetStringValue();
}
break;
case JsonTokenType.Number:
if (materializeValue)
{
Number64 number64Value = jsonReader.GetNumberValue();
}
break;
case JsonTokenType.Int8:
if (materializeValue)
{
sbyte int8Value = jsonReader.GetInt8Value();
}
break;
case JsonTokenType.Int16:
if (materializeValue)
{
short int16Value = jsonReader.GetInt16Value();
}
break;
case JsonTokenType.Int32:
if (materializeValue)
{
int int32Value = jsonReader.GetInt32Value();
}
break;
case JsonTokenType.Int64:
if (materializeValue)
{
long int64Value = jsonReader.GetInt64Value();
}
break;
case JsonTokenType.UInt32:
if (materializeValue)
{
uint uInt32Value = jsonReader.GetUInt32Value();
}
break;
case JsonTokenType.Float32:
if (materializeValue)
{
float float32Value = jsonReader.GetFloat32Value();
}
break;
case JsonTokenType.Float64:
if (materializeValue)
{
double doubleValue = jsonReader.GetFloat64Value();
}
break;
case JsonTokenType.Guid:
if (materializeValue)
{
Guid guidValue = jsonReader.GetGuidValue();
}
break;
case JsonTokenType.Binary:
if (materializeValue)
{
ReadOnlyMemory <byte> binaryValue = jsonReader.GetBinaryValue();
}
break;
default:
throw new ArgumentException("$Unknown token type.");
}
}
}
public static TimeSpan MeasureReadPerformance(IJsonReader jsonReader, int numberOfIterations = 1)
{
Stopwatch stopwatch = new Stopwatch();
for (int i = 0; i < numberOfIterations; i++)
{
GC.Collect();
GC.WaitForPendingFinalizers();
stopwatch.Start();
while (jsonReader.Read())
{
// Materialize the value
switch (jsonReader.CurrentTokenType)
{
case JsonTokenType.BeginArray:
case JsonTokenType.EndArray:
case JsonTokenType.BeginObject:
case JsonTokenType.EndObject:
case JsonTokenType.Null:
case JsonTokenType.True:
case JsonTokenType.False:
// Single byte tokens
break;
case JsonTokenType.String:
case JsonTokenType.FieldName:
string stringValue = jsonReader.GetStringValue();
break;
case JsonTokenType.Number:
Number64 number64Value = jsonReader.GetNumberValue();
break;
case JsonTokenType.Int8:
sbyte int8Value = jsonReader.GetInt8Value();
break;
case JsonTokenType.Int16:
short int16Value = jsonReader.GetInt16Value();
break;
case JsonTokenType.Int32:
int int32Value = jsonReader.GetInt32Value();
break;
case JsonTokenType.Int64:
long int64Value = jsonReader.GetInt64Value();
break;
case JsonTokenType.UInt32:
uint uInt32Value = jsonReader.GetUInt32Value();
break;
case JsonTokenType.Float32:
float float32Value = jsonReader.GetFloat32Value();
break;
case JsonTokenType.Float64:
double doubleValue = jsonReader.GetFloat64Value();
break;
case JsonTokenType.Guid:
Guid guidValue = jsonReader.GetGuidValue();
break;
case JsonTokenType.Binary:
ReadOnlyMemory <byte> binaryValue = jsonReader.GetBinaryValue();
break;
default:
throw new ArgumentException("$Unknown token type.");
}
}
stopwatch.Stop();
}
return(stopwatch.Elapsed);
}
/// <summary>
/// Writes current token from a json reader to the internal buffer.
/// </summary>
/// <param name="jsonReader">The JsonReader to the get the current token from.</param>
public void WriteCurrentToken(IJsonReader jsonReader)
{
if (jsonReader == null)
{
throw new ArgumentNullException("jsonReader can not be null");
}
// For now we don't optimize for text, since the reader could be UTF-8 and the writer could be UTF-16.
// We need to add more enums for the different serialization formats.
bool sameFormat = jsonReader.SerializationFormat == this.SerializationFormat && (this.SerializationFormat == JsonSerializationFormat.Binary || this.SerializationFormat == JsonSerializationFormat.HybridRow);
JsonTokenType jsonTokenType = jsonReader.CurrentTokenType;
switch (jsonTokenType)
{
case JsonTokenType.NotStarted:
break;
case JsonTokenType.BeginArray:
this.WriteArrayStart();
break;
case JsonTokenType.EndArray:
this.WriteArrayEnd();
break;
case JsonTokenType.BeginObject:
this.WriteObjectStart();
break;
case JsonTokenType.EndObject:
this.WriteObjectEnd();
break;
case JsonTokenType.True:
this.WriteBoolValue(true);
break;
case JsonTokenType.False:
this.WriteBoolValue(false);
break;
case JsonTokenType.Null:
this.WriteNullValue();
break;
default:
{
if (sameFormat)
{
ReadOnlySpan <byte> bufferedRawJson = jsonReader.GetBufferedRawJsonToken().Span;
this.WriteRawJsonToken(jsonTokenType, bufferedRawJson);
}
else
{
switch (jsonTokenType)
{
case JsonTokenType.String:
case JsonTokenType.FieldName:
{
string value = jsonReader.GetStringValue();
if (jsonTokenType == JsonTokenType.FieldName)
{
this.WriteFieldName(value);
}
else
{
this.WriteStringValue(value);
}
break;
}
case JsonTokenType.Number:
{
Number64 value = jsonReader.GetNumberValue();
this.WriteNumberValue(value);
}
break;
case JsonTokenType.Int8:
{
sbyte value = jsonReader.GetInt8Value();
this.WriteInt8Value(value);
}
break;
case JsonTokenType.Int16:
{
short value = jsonReader.GetInt16Value();
this.WriteInt16Value(value);
}
break;
case JsonTokenType.Int32:
{
int value = jsonReader.GetInt32Value();
this.WriteInt32Value(value);
}
break;
case JsonTokenType.Int64:
{
long value = jsonReader.GetInt64Value();
this.WriteInt64Value(value);
}
break;
case JsonTokenType.UInt32:
{
uint value = jsonReader.GetUInt32Value();
this.WriteUInt32Value(value);
}
break;
case JsonTokenType.Float32:
{
float value = jsonReader.GetFloat32Value();
this.WriteFloat32Value(value);
}
break;
case JsonTokenType.Float64:
{
double value = jsonReader.GetFloat64Value();
this.WriteFloat64Value(value);
}
break;
case JsonTokenType.Guid:
{
Guid value = jsonReader.GetGuidValue();
this.WriteGuidValue(value);
}
break;
case JsonTokenType.Binary:
{
ReadOnlyMemory <byte> value = jsonReader.GetBinaryValue();
this.WriteBinaryValue(value.Span);
}
break;
default:
throw new ArgumentException($"Unexpected JsonTokenType: {jsonTokenType}");
}
}
}
break;
}
}
/// <summary>
/// Writes current token from a json reader to the internal buffer.
/// </summary>
/// <param name="jsonReader">The JsonReader to the get the current token from.</param>
public void WriteCurrentToken(IJsonReader jsonReader)
{
if (jsonReader == null)
{
throw new ArgumentNullException("jsonReader can not be null");
}
bool sameFormat = jsonReader.SerializationFormat == this.SerializationFormat;
JsonTokenType jsonTokenType = jsonReader.CurrentTokenType;
switch (jsonTokenType)
{
case JsonTokenType.NotStarted:
break;
case JsonTokenType.BeginArray:
this.WriteArrayStart();
break;
case JsonTokenType.EndArray:
this.WriteArrayEnd();
break;
case JsonTokenType.BeginObject:
this.WriteObjectStart();
break;
case JsonTokenType.EndObject:
this.WriteObjectEnd();
break;
case JsonTokenType.True:
this.WriteBoolValue(true);
break;
case JsonTokenType.False:
this.WriteBoolValue(false);
break;
case JsonTokenType.Null:
this.WriteNullValue();
break;
default:
{
if (sameFormat && jsonReader.TryGetBufferedRawJsonToken(out ReadOnlyMemory <byte> bufferedRawJsonToken))
{
this.WriteRawJsonToken(jsonTokenType, bufferedRawJsonToken.Span);
}
else
{
switch (jsonTokenType)
{
case JsonTokenType.String:
case JsonTokenType.FieldName:
{
string value = jsonReader.GetStringValue();
if (jsonTokenType == JsonTokenType.FieldName)
{
this.WriteFieldName(value);
}
else
{
this.WriteStringValue(value);
}
break;
}
case JsonTokenType.Number:
{
Number64 value = jsonReader.GetNumberValue();
this.WriteNumber64Value(value);
}
break;
case JsonTokenType.Int8:
{
sbyte value = jsonReader.GetInt8Value();
this.WriteInt8Value(value);
}
break;
case JsonTokenType.Int16:
{
short value = jsonReader.GetInt16Value();
this.WriteInt16Value(value);
}
break;
case JsonTokenType.Int32:
{
int value = jsonReader.GetInt32Value();
this.WriteInt32Value(value);
}
break;
case JsonTokenType.Int64:
{
long value = jsonReader.GetInt64Value();
this.WriteInt64Value(value);
}
break;
case JsonTokenType.UInt32:
{
uint value = jsonReader.GetUInt32Value();
this.WriteUInt32Value(value);
}
break;
case JsonTokenType.Float32:
{
float value = jsonReader.GetFloat32Value();
this.WriteFloat32Value(value);
}
break;
case JsonTokenType.Float64:
{
double value = jsonReader.GetFloat64Value();
this.WriteFloat64Value(value);
}
break;
case JsonTokenType.Guid:
{
Guid value = jsonReader.GetGuidValue();
this.WriteGuidValue(value);
}
break;
case JsonTokenType.Binary:
{
ReadOnlyMemory <byte> value = jsonReader.GetBinaryValue();
this.WriteBinaryValue(value.Span);
}
break;
default:
throw new ArgumentException($"Unexpected JsonTokenType: {jsonTokenType}");
}
}
}
break;
}
}
/// <summary>
/// Writes current token from a json reader to the internal buffer.
/// </summary>
/// <param name="jsonReader">The JsonReader to the get the current token from.</param>
public void WriteCurrentToken(IJsonReader jsonReader)
{
if (jsonReader == null)
{
throw new ArgumentNullException("jsonReader can not be null");
}
// For now short circuit this to false until we figure out how to optimize this.
bool sameFormat = jsonReader.SerializationFormat == this.SerializationFormat && false;
JsonTokenType jsonTokenType = jsonReader.CurrentTokenType;
switch (jsonTokenType)
{
case JsonTokenType.NotStarted:
break;
case JsonTokenType.BeginArray:
this.WriteArrayStart();
break;
case JsonTokenType.EndArray:
this.WriteArrayEnd();
break;
case JsonTokenType.BeginObject:
this.WriteObjectStart();
break;
case JsonTokenType.EndObject:
this.WriteObjectEnd();
break;
case JsonTokenType.True:
this.WriteBoolValue(true);
break;
case JsonTokenType.False:
this.WriteBoolValue(false);
break;
case JsonTokenType.Null:
this.WriteNullValue();
break;
case JsonTokenType.String:
case JsonTokenType.Number:
case JsonTokenType.FieldName:
{
if (sameFormat)
{
IReadOnlyList <byte> bufferedRawJson = jsonReader.GetBufferedRawJsonToken();
this.WriteRawJsonToken(jsonTokenType, bufferedRawJson);
}
else
{
if (jsonTokenType == JsonTokenType.Number)
{
double number = jsonReader.GetNumberValue();
this.WriteNumberValue(number);
}
else
{
string value = jsonReader.GetStringValue();
if (jsonTokenType == JsonTokenType.FieldName)
{
this.WriteFieldName(value);
}
else
{
this.WriteStringValue(value);
}
}
}
break;
}
default:
throw new ArgumentException($"Unexpected JsonTokenType: {jsonTokenType}");
}
}
