public static JsonToken[] Tokenize(IJsonReader jsonReader, string json) { List <JsonToken> tokensFromReader = new List <JsonToken>(); while (jsonReader.Read()) { switch (jsonReader.CurrentTokenType) { case JsonTokenType.NotStarted: throw new ArgumentException(string.Format("Got an unexpected JsonTokenType: {0} as an expected token type", jsonReader.CurrentTokenType)); case JsonTokenType.BeginArray: tokensFromReader.Add(JsonToken.ArrayStart()); break; case JsonTokenType.EndArray: tokensFromReader.Add(JsonToken.ArrayEnd()); break; case JsonTokenType.BeginObject: tokensFromReader.Add(JsonToken.ObjectStart()); break; case JsonTokenType.EndObject: tokensFromReader.Add(JsonToken.ObjectEnd()); break; case JsonTokenType.String: tokensFromReader.Add(JsonToken.String(jsonReader.GetStringValue())); break; case JsonTokenType.Number: tokensFromReader.Add(JsonToken.Number(jsonReader.GetNumberValue())); break; case JsonTokenType.True: tokensFromReader.Add(JsonToken.Boolean(true)); break; case JsonTokenType.False: tokensFromReader.Add(JsonToken.Boolean(false)); break; case JsonTokenType.Null: tokensFromReader.Add(JsonToken.Null()); break; case JsonTokenType.FieldName: tokensFromReader.Add(JsonToken.FieldName(jsonReader.GetStringValue())); break; default: break; } } return(tokensFromReader.ToArray()); }
internal static JsonTokenInfo[] GetTokensWithReader(IJsonReader jsonReader) { List <JsonTokenInfo> tokens = new List <JsonTokenInfo>(); while (jsonReader.Read()) { switch (jsonReader.CurrentTokenType) { case JsonTokenType.NotStarted: throw new InvalidOperationException(); case JsonTokenType.BeginArray: tokens.Add(JsonTokenInfo.ArrayStart()); break; case JsonTokenType.EndArray: tokens.Add(JsonTokenInfo.ArrayEnd()); break; case JsonTokenType.BeginObject: tokens.Add(JsonTokenInfo.ObjectStart()); break; case JsonTokenType.EndObject: tokens.Add(JsonTokenInfo.ObjectEnd()); break; case JsonTokenType.String: tokens.Add(JsonTokenInfo.String(jsonReader.GetStringValue())); break; case JsonTokenType.Number: tokens.Add(JsonTokenInfo.Number(jsonReader.GetNumberValue())); break; case JsonTokenType.True: tokens.Add(JsonTokenInfo.Boolean(true)); break; case JsonTokenType.False: tokens.Add(JsonTokenInfo.Boolean(false)); break; case JsonTokenType.Null: tokens.Add(JsonTokenInfo.Null()); break; case JsonTokenType.FieldName: tokens.Add(JsonTokenInfo.FieldName(jsonReader.GetStringValue())); break; default: break; } } return(tokens.ToArray()); }
static void TokenizeInternal(IJsonReader jsonReader, List <JsonToken> tokenList) { while (jsonReader.Read()) { switch (jsonReader.CurrentTokenType) { case JsonTokenType.NotStarted: throw new ArgumentException(string.Format("Got an unexpected JsonTokenType: {0} as an expected token type", jsonReader.CurrentTokenType)); case JsonTokenType.BeginArray: tokenList.Add(JsonToken.ArrayStart()); break; case JsonTokenType.EndArray: tokenList.Add(JsonToken.ArrayEnd()); break; case JsonTokenType.BeginObject: tokenList.Add(JsonToken.ObjectStart()); break; case JsonTokenType.EndObject: tokenList.Add(JsonToken.ObjectEnd()); break; case JsonTokenType.String: tokenList.Add(JsonToken.String(jsonReader.GetStringValue())); break; case JsonTokenType.Number: tokenList.Add(JsonToken.Number(jsonReader.GetNumberValue())); break; case JsonTokenType.True: tokenList.Add(JsonToken.Boolean(true)); break; case JsonTokenType.False: tokenList.Add(JsonToken.Boolean(false)); break; case JsonTokenType.Null: tokenList.Add(JsonToken.Null()); break; case JsonTokenType.FieldName: tokenList.Add(JsonToken.FieldName(jsonReader.GetStringValue())); break; default: throw new ArgumentOutOfRangeException($"Unknown {nameof(JsonTokenType)}: '{jsonReader.CurrentTokenType}'."); } } }
private static void RunReadBenchmark( Payload payload, bool useUtf8) { // Don't really need to test both serialization formats, since they are similiar. IJsonReader jsonReader = JsonReader.Create(payload.Binary); while (jsonReader.Read()) { // Materialize the value switch (jsonReader.CurrentTokenType) { case JsonTokenType.BeginArray: case JsonTokenType.EndArray: // Single byte tokens break; case JsonTokenType.String: UtfAnyString value = jsonReader.GetStringValue(); if (useUtf8) { Utf8String _ = value.ToUtf8String(); } else { string _ = value.ToString(); } break; default: throw new ArgumentException("$Unknown token type."); } } }
public void Utf8VsUtf16StringRead() { void RunPerf(string utf16String, JsonSerializationFormat jsonSerializationFormat, bool useUtf8) { Stopwatch stopWatch = new Stopwatch(); byte[] payload = JsonTestUtils.ConvertTextToBinary("\"" + utf16String + "\""); for (int i = 0; i < 1000000; i++) { IJsonReader reader = JsonReader.Create(payload); reader.Read(); stopWatch.Start(); if (useUtf8) { reader.TryGetBufferedUtf8StringValue(out ReadOnlyMemory <byte> bufferedUtf8StringValue); } else { string value = reader.GetStringValue(); } stopWatch.Stop(); } Console.WriteLine($"UTF {(useUtf8 ? 8 : 16)} {jsonSerializationFormat} reader + string length: {utf16String.Length} = {stopWatch.ElapsedMilliseconds} ms"); } foreach (int stringLength in new int[] { 8, 32, 256, 1024, 4096 }) { foreach (JsonSerializationFormat jsonSerializationFormat in new JsonSerializationFormat[] { JsonSerializationFormat.Text, JsonSerializationFormat.Binary }) { foreach (bool useUtf8 in new bool[] { false, true }) { RunPerf( utf16String: new string('a', stringLength), jsonSerializationFormat: jsonSerializationFormat, useUtf8: useUtf8); } } } }
private static void RunReadBenchmark( Payload payload, bool useUtf8) { // Don't really need to test both serialization formats, since they are similiar. IJsonReader jsonReader = JsonReader.Create(payload.Binary); while (jsonReader.Read()) { // Materialize the value switch (jsonReader.CurrentTokenType) { case JsonTokenType.BeginArray: case JsonTokenType.EndArray: // Single byte tokens break; case JsonTokenType.String: if (useUtf8) { if (!jsonReader.TryGetBufferedUtf8StringValue(out ReadOnlyMemory <byte> bufferedUtf8StringValue)) { throw new InvalidOperationException("Failed to get utf8 string."); } } else { string value = jsonReader.GetStringValue(); } break; default: throw new ArgumentException("$Unknown token type."); } } }
/// <summary> /// Writes current token from a json reader to the internal buffer. /// </summary> /// <param name="jsonReader">The JsonReader to the get the current token from.</param> public void WriteCurrentToken(IJsonReader jsonReader) { if (jsonReader == null) { throw new ArgumentNullException("jsonReader can not be null"); } // For now we don't optimize for text, since the reader could be UTF-8 and the writer could be UTF-16. // We need to add more enums for the different serialization formats. bool sameFormat = jsonReader.SerializationFormat == this.SerializationFormat && (this.SerializationFormat == JsonSerializationFormat.Binary || this.SerializationFormat == JsonSerializationFormat.HybridRow); JsonTokenType jsonTokenType = jsonReader.CurrentTokenType; switch (jsonTokenType) { case JsonTokenType.NotStarted: break; case JsonTokenType.BeginArray: this.WriteArrayStart(); break; case JsonTokenType.EndArray: this.WriteArrayEnd(); break; case JsonTokenType.BeginObject: this.WriteObjectStart(); break; case JsonTokenType.EndObject: this.WriteObjectEnd(); break; case JsonTokenType.True: this.WriteBoolValue(true); break; case JsonTokenType.False: this.WriteBoolValue(false); break; case JsonTokenType.Null: this.WriteNullValue(); break; case JsonTokenType.String: case JsonTokenType.Number: case JsonTokenType.FieldName: { if (sameFormat) { IReadOnlyList <byte> bufferedRawJson = jsonReader.GetBufferedRawJsonToken(); this.WriteRawJsonToken(jsonTokenType, bufferedRawJson); } else { if (jsonTokenType == JsonTokenType.Number) { double number = jsonReader.GetNumberValue(); this.WriteNumberValue(number); } else { string value = jsonReader.GetStringValue(); if (jsonTokenType == JsonTokenType.FieldName) { this.WriteFieldName(value); } else { this.WriteStringValue(value); } } } break; } default: throw new ArgumentException($"Unexpected JsonTokenType: {jsonTokenType}"); } }
private static JsonToken[] GetTokensWithReader(IJsonReader jsonReader) { List <JsonToken> tokens = new List <JsonToken>(); while (jsonReader.Read()) { JsonToken token; switch (jsonReader.CurrentTokenType) { case JsonTokenType.NotStarted: throw new InvalidOperationException(); case JsonTokenType.BeginArray: token = JsonToken.ArrayStart(); break; case JsonTokenType.EndArray: token = JsonToken.ArrayEnd(); break; case JsonTokenType.BeginObject: token = JsonToken.ObjectStart(); break; case JsonTokenType.EndObject: token = JsonToken.ObjectEnd(); break; case JsonTokenType.String: token = JsonToken.String(jsonReader.GetStringValue()); break; case JsonTokenType.Number: token = JsonToken.Number(jsonReader.GetNumberValue()); break; case JsonTokenType.True: token = JsonToken.Boolean(true); break; case JsonTokenType.False: token = JsonToken.Boolean(false); break; case JsonTokenType.Null: token = JsonToken.Null(); break; case JsonTokenType.FieldName: token = JsonToken.FieldName(jsonReader.GetStringValue()); break; case JsonTokenType.Int8: token = JsonToken.Int8(jsonReader.GetInt8Value()); break; case JsonTokenType.Int16: token = JsonToken.Int16(jsonReader.GetInt16Value()); break; case JsonTokenType.Int32: token = JsonToken.Int32(jsonReader.GetInt32Value()); break; case JsonTokenType.Int64: token = JsonToken.Int64(jsonReader.GetInt64Value()); break; case JsonTokenType.UInt32: token = JsonToken.UInt32(jsonReader.GetUInt32Value()); break; case JsonTokenType.Float32: token = JsonToken.Float32(jsonReader.GetFloat32Value()); break; case JsonTokenType.Float64: token = JsonToken.Float64(jsonReader.GetFloat64Value()); break; case JsonTokenType.Guid: token = JsonToken.Guid(jsonReader.GetGuidValue()); break; case JsonTokenType.Binary: token = JsonToken.Binary(jsonReader.GetBinaryValue()); break; default: throw new ArgumentException($"Unknown {nameof(JsonTokenType)}: {jsonReader.CurrentTokenType}"); } tokens.Add(token); } return(tokens.ToArray()); }
public static void DrainReader(IJsonReader jsonReader, bool materializeValue) { if (jsonReader == null) { throw new ArgumentNullException(nameof(jsonReader)); } while (jsonReader.Read()) { // Materialize the value switch (jsonReader.CurrentTokenType) { case JsonTokenType.BeginArray: case JsonTokenType.EndArray: case JsonTokenType.BeginObject: case JsonTokenType.EndObject: case JsonTokenType.Null: case JsonTokenType.True: case JsonTokenType.False: // Single byte tokens break; case JsonTokenType.String: case JsonTokenType.FieldName: if (materializeValue) { string stringValue = jsonReader.GetStringValue(); } break; case JsonTokenType.Number: if (materializeValue) { Number64 number64Value = jsonReader.GetNumberValue(); } break; case JsonTokenType.Int8: if (materializeValue) { sbyte int8Value = jsonReader.GetInt8Value(); } break; case JsonTokenType.Int16: if (materializeValue) { short int16Value = jsonReader.GetInt16Value(); } break; case JsonTokenType.Int32: if (materializeValue) { int int32Value = jsonReader.GetInt32Value(); } break; case JsonTokenType.Int64: if (materializeValue) { long int64Value = jsonReader.GetInt64Value(); } break; case JsonTokenType.UInt32: if (materializeValue) { uint uInt32Value = jsonReader.GetUInt32Value(); } break; case JsonTokenType.Float32: if (materializeValue) { float float32Value = jsonReader.GetFloat32Value(); } break; case JsonTokenType.Float64: if (materializeValue) { double doubleValue = jsonReader.GetFloat64Value(); } break; case JsonTokenType.Guid: if (materializeValue) { Guid guidValue = jsonReader.GetGuidValue(); } break; case JsonTokenType.Binary: if (materializeValue) { ReadOnlyMemory <byte> binaryValue = jsonReader.GetBinaryValue(); } break; default: throw new ArgumentException("$Unknown token type."); } } }
public static TimeSpan MeasureReadPerformance(IJsonReader jsonReader, int numberOfIterations = 1) { Stopwatch stopwatch = new Stopwatch(); for (int i = 0; i < numberOfIterations; i++) { GC.Collect(); GC.WaitForPendingFinalizers(); stopwatch.Start(); while (jsonReader.Read()) { // Materialize the value switch (jsonReader.CurrentTokenType) { case JsonTokenType.BeginArray: case JsonTokenType.EndArray: case JsonTokenType.BeginObject: case JsonTokenType.EndObject: case JsonTokenType.Null: case JsonTokenType.True: case JsonTokenType.False: // Single byte tokens break; case JsonTokenType.String: case JsonTokenType.FieldName: string stringValue = jsonReader.GetStringValue(); break; case JsonTokenType.Number: Number64 number64Value = jsonReader.GetNumberValue(); break; case JsonTokenType.Int8: sbyte int8Value = jsonReader.GetInt8Value(); break; case JsonTokenType.Int16: short int16Value = jsonReader.GetInt16Value(); break; case JsonTokenType.Int32: int int32Value = jsonReader.GetInt32Value(); break; case JsonTokenType.Int64: long int64Value = jsonReader.GetInt64Value(); break; case JsonTokenType.UInt32: uint uInt32Value = jsonReader.GetUInt32Value(); break; case JsonTokenType.Float32: float float32Value = jsonReader.GetFloat32Value(); break; case JsonTokenType.Float64: double doubleValue = jsonReader.GetFloat64Value(); break; case JsonTokenType.Guid: Guid guidValue = jsonReader.GetGuidValue(); break; case JsonTokenType.Binary: ReadOnlyMemory <byte> binaryValue = jsonReader.GetBinaryValue(); break; default: throw new ArgumentException("$Unknown token type."); } } stopwatch.Stop(); } return(stopwatch.Elapsed); }
/// <summary> /// Writes current token from a json reader to the internal buffer. /// </summary> /// <param name="jsonReader">The JsonReader to the get the current token from.</param> public void WriteCurrentToken(IJsonReader jsonReader) { if (jsonReader == null) { throw new ArgumentNullException("jsonReader can not be null"); } // For now we don't optimize for text, since the reader could be UTF-8 and the writer could be UTF-16. // We need to add more enums for the different serialization formats. bool sameFormat = jsonReader.SerializationFormat == this.SerializationFormat && (this.SerializationFormat == JsonSerializationFormat.Binary || this.SerializationFormat == JsonSerializationFormat.HybridRow); JsonTokenType jsonTokenType = jsonReader.CurrentTokenType; switch (jsonTokenType) { case JsonTokenType.NotStarted: break; case JsonTokenType.BeginArray: this.WriteArrayStart(); break; case JsonTokenType.EndArray: this.WriteArrayEnd(); break; case JsonTokenType.BeginObject: this.WriteObjectStart(); break; case JsonTokenType.EndObject: this.WriteObjectEnd(); break; case JsonTokenType.True: this.WriteBoolValue(true); break; case JsonTokenType.False: this.WriteBoolValue(false); break; case JsonTokenType.Null: this.WriteNullValue(); break; default: { if (sameFormat) { ReadOnlySpan <byte> bufferedRawJson = jsonReader.GetBufferedRawJsonToken().Span; this.WriteRawJsonToken(jsonTokenType, bufferedRawJson); } else { switch (jsonTokenType) { case JsonTokenType.String: case JsonTokenType.FieldName: { string value = jsonReader.GetStringValue(); if (jsonTokenType == JsonTokenType.FieldName) { this.WriteFieldName(value); } else { this.WriteStringValue(value); } break; } case JsonTokenType.Number: { Number64 value = jsonReader.GetNumberValue(); this.WriteNumberValue(value); } break; case JsonTokenType.Int8: { sbyte value = jsonReader.GetInt8Value(); this.WriteInt8Value(value); } break; case JsonTokenType.Int16: { short value = jsonReader.GetInt16Value(); this.WriteInt16Value(value); } break; case JsonTokenType.Int32: { int value = jsonReader.GetInt32Value(); this.WriteInt32Value(value); } break; case JsonTokenType.Int64: { long value = jsonReader.GetInt64Value(); this.WriteInt64Value(value); } break; case JsonTokenType.UInt32: { uint value = jsonReader.GetUInt32Value(); this.WriteUInt32Value(value); } break; case JsonTokenType.Float32: { float value = jsonReader.GetFloat32Value(); this.WriteFloat32Value(value); } break; case JsonTokenType.Float64: { double value = jsonReader.GetFloat64Value(); this.WriteFloat64Value(value); } break; case JsonTokenType.Guid: { Guid value = jsonReader.GetGuidValue(); this.WriteGuidValue(value); } break; case JsonTokenType.Binary: { ReadOnlyMemory <byte> value = jsonReader.GetBinaryValue(); this.WriteBinaryValue(value.Span); } break; default: throw new ArgumentException($"Unexpected JsonTokenType: {jsonTokenType}"); } } } break; } }
/// <summary> /// Writes current token from a json reader to the internal buffer. /// </summary> /// <param name="jsonReader">The JsonReader to the get the current token from.</param> public void WriteCurrentToken(IJsonReader jsonReader) { if (jsonReader == null) { throw new ArgumentNullException("jsonReader can not be null"); } bool sameFormat = jsonReader.SerializationFormat == this.SerializationFormat; JsonTokenType jsonTokenType = jsonReader.CurrentTokenType; switch (jsonTokenType) { case JsonTokenType.NotStarted: break; case JsonTokenType.BeginArray: this.WriteArrayStart(); break; case JsonTokenType.EndArray: this.WriteArrayEnd(); break; case JsonTokenType.BeginObject: this.WriteObjectStart(); break; case JsonTokenType.EndObject: this.WriteObjectEnd(); break; case JsonTokenType.True: this.WriteBoolValue(true); break; case JsonTokenType.False: this.WriteBoolValue(false); break; case JsonTokenType.Null: this.WriteNullValue(); break; default: { if (sameFormat && jsonReader.TryGetBufferedRawJsonToken(out ReadOnlyMemory <byte> bufferedRawJsonToken)) { this.WriteRawJsonToken(jsonTokenType, bufferedRawJsonToken.Span); } else { switch (jsonTokenType) { case JsonTokenType.String: case JsonTokenType.FieldName: { string value = jsonReader.GetStringValue(); if (jsonTokenType == JsonTokenType.FieldName) { this.WriteFieldName(value); } else { this.WriteStringValue(value); } break; } case JsonTokenType.Number: { Number64 value = jsonReader.GetNumberValue(); this.WriteNumber64Value(value); } break; case JsonTokenType.Int8: { sbyte value = jsonReader.GetInt8Value(); this.WriteInt8Value(value); } break; case JsonTokenType.Int16: { short value = jsonReader.GetInt16Value(); this.WriteInt16Value(value); } break; case JsonTokenType.Int32: { int value = jsonReader.GetInt32Value(); this.WriteInt32Value(value); } break; case JsonTokenType.Int64: { long value = jsonReader.GetInt64Value(); this.WriteInt64Value(value); } break; case JsonTokenType.UInt32: { uint value = jsonReader.GetUInt32Value(); this.WriteUInt32Value(value); } break; case JsonTokenType.Float32: { float value = jsonReader.GetFloat32Value(); this.WriteFloat32Value(value); } break; case JsonTokenType.Float64: { double value = jsonReader.GetFloat64Value(); this.WriteFloat64Value(value); } break; case JsonTokenType.Guid: { Guid value = jsonReader.GetGuidValue(); this.WriteGuidValue(value); } break; case JsonTokenType.Binary: { ReadOnlyMemory <byte> value = jsonReader.GetBinaryValue(); this.WriteBinaryValue(value.Span); } break; default: throw new ArgumentException($"Unexpected JsonTokenType: {jsonTokenType}"); } } } break; } }
/// <summary> /// Writes current token from a json reader to the internal buffer. /// </summary> /// <param name="jsonReader">The JsonReader to the get the current token from.</param> public void WriteCurrentToken(IJsonReader jsonReader) { if (jsonReader == null) { throw new ArgumentNullException("jsonReader can not be null"); } // For now short circuit this to false until we figure out how to optimize this. bool sameFormat = jsonReader.SerializationFormat == this.SerializationFormat && false; JsonTokenType jsonTokenType = jsonReader.CurrentTokenType; switch (jsonTokenType) { case JsonTokenType.NotStarted: break; case JsonTokenType.BeginArray: this.WriteArrayStart(); break; case JsonTokenType.EndArray: this.WriteArrayEnd(); break; case JsonTokenType.BeginObject: this.WriteObjectStart(); break; case JsonTokenType.EndObject: this.WriteObjectEnd(); break; case JsonTokenType.True: this.WriteBoolValue(true); break; case JsonTokenType.False: this.WriteBoolValue(false); break; case JsonTokenType.Null: this.WriteNullValue(); break; case JsonTokenType.String: case JsonTokenType.Number: case JsonTokenType.FieldName: { if (sameFormat) { IReadOnlyList <byte> bufferedRawJson = jsonReader.GetBufferedRawJsonToken(); this.WriteRawJsonToken(jsonTokenType, bufferedRawJson); } else { if (jsonTokenType == JsonTokenType.Number) { double number = jsonReader.GetNumberValue(); this.WriteNumberValue(number); } else { string value = jsonReader.GetStringValue(); if (jsonTokenType == JsonTokenType.FieldName) { this.WriteFieldName(value); } else { this.WriteStringValue(value); } } } break; } default: throw new ArgumentException($"Unexpected JsonTokenType: {jsonTokenType}"); } }