public static void TestSingleStringsMultiSegmentByOne()
{
string jsonString = "\"Hello, \\u0041hson!\"";
string expectedString = "Hello, \\u0041hson!, ";
byte[] dataUtf8 = Encoding.UTF8.GetBytes(jsonString);
ReadOnlySequence <byte> sequence = JsonTestHelper.GetSequence(dataUtf8, 1);
for (int j = 0; j < dataUtf8.Length; j++)
{
var utf8JsonReader = new Utf8JsonReader(sequence.Slice(0, j), isFinalBlock: false, default);
byte[] resultSequence = JsonTestHelper.ReaderLoop(dataUtf8.Length, out int length, ref utf8JsonReader);
string actualStrSequence = Encoding.UTF8.GetString(resultSequence, 0, length);
long consumed = utf8JsonReader.BytesConsumed;
Assert.Equal(consumed, utf8JsonReader.CurrentState.BytesConsumed);
utf8JsonReader = new Utf8JsonReader(sequence.Slice(consumed), isFinalBlock: true, utf8JsonReader.CurrentState);
resultSequence = JsonTestHelper.ReaderLoop(dataUtf8.Length, out length, ref utf8JsonReader);
actualStrSequence += Encoding.UTF8.GetString(resultSequence, 0, length);
string message = $"Expected consumed: {dataUtf8.Length - consumed}, Actual consumed: {utf8JsonReader.BytesConsumed}, Index: {j}";
Assert.Equal(utf8JsonReader.BytesConsumed, utf8JsonReader.CurrentState.BytesConsumed);
Assert.True(dataUtf8.Length - consumed == utf8JsonReader.BytesConsumed, message);
Assert.Equal(expectedString, actualStrSequence);
}
}
public static void InvalidJsonMultiSegmentByOne(string jsonString, int expectedlineNumber, int expectedBytePosition, int maxDepth = 64)
{
foreach (JsonCommentHandling commentHandling in Enum.GetValues(typeof(JsonCommentHandling)))
{
byte[] dataUtf8 = Encoding.UTF8.GetBytes(jsonString);
ReadOnlySequence <byte> sequence = JsonTestHelper.GetSequence(dataUtf8, 1);
SpanSequenceStatesAreEqualInvalidJson(dataUtf8, sequence, maxDepth, commentHandling);
var state = new JsonReaderState(new JsonReaderOptions {
CommentHandling = commentHandling, MaxDepth = maxDepth
});
var json = new Utf8JsonReader(sequence, isFinalBlock: true, state);
try
{
while (json.Read())
{
;
}
Assert.True(false, "Expected JsonReaderException for multi-segment data was not thrown.");
}
catch (JsonReaderException ex)
{
Assert.Equal(expectedlineNumber, ex.LineNumber);
Assert.Equal(expectedBytePosition, ex.BytePositionInLine);
}
}
}
private static void ReadPartialSegmentSizeOne(bool compactData, TestCaseType type, string jsonString)
{
// Remove all formatting/indendation
if (compactData)
{
jsonString = JsonTestHelper.GetCompactString(jsonString);
}
byte[] dataUtf8 = Encoding.UTF8.GetBytes(jsonString);
Stream stream = new MemoryStream(dataUtf8);
TextReader reader = new StreamReader(stream, Encoding.UTF8, false, 1024, true);
string expectedStr = JsonTestHelper.NewtonsoftReturnStringHelper(reader);
ReadOnlySequence <byte> sequence = JsonTestHelper.GetSequence(dataUtf8, 1);
for (int j = 0; j < dataUtf8.Length; j++)
{
var utf8JsonReader = new Utf8JsonReader(sequence.Slice(0, j), isFinalBlock: false, default);
byte[] resultSequence = JsonTestHelper.ReaderLoop(dataUtf8.Length, out int length, ref utf8JsonReader);
string actualStrSequence = Encoding.UTF8.GetString(resultSequence, 0, length);
long consumed = utf8JsonReader.BytesConsumed;
Assert.Equal(consumed, utf8JsonReader.CurrentState.BytesConsumed);
utf8JsonReader = new Utf8JsonReader(sequence.Slice(consumed), isFinalBlock: true, utf8JsonReader.CurrentState);
resultSequence = JsonTestHelper.ReaderLoop(dataUtf8.Length, out length, ref utf8JsonReader);
actualStrSequence += Encoding.UTF8.GetString(resultSequence, 0, length);
string message = $"Expected consumed: {dataUtf8.Length - consumed}, Actual consumed: {utf8JsonReader.BytesConsumed}, Index: {j}";
Assert.Equal(utf8JsonReader.BytesConsumed, utf8JsonReader.CurrentState.BytesConsumed);
Assert.True(dataUtf8.Length - consumed == utf8JsonReader.BytesConsumed, message);
Assert.Equal(expectedStr, actualStrSequence);
}
}
public static void CheckOnlyOneOfValueSpanOrSequenceIsSetSingleValue(string jsonString)
{
byte[] dataUtf8 = Encoding.UTF8.GetBytes(jsonString);
ReadOnlySequence <byte> sequence = JsonTestHelper.GetSequence(dataUtf8, 1);
foreach (JsonCommentHandling commentHandling in Enum.GetValues(typeof(JsonCommentHandling)))
{
var state = new JsonReaderState(options: new JsonReaderOptions {
CommentHandling = commentHandling
});
var json = new Utf8JsonReader(sequence, isFinalBlock: true, state);
Assert.False(json.HasValueSequence);
Assert.True(json.ValueSpan == default);
Assert.True(json.ValueSequence.IsEmpty);
Assert.True(json.Read());
Assert.True(json.HasValueSequence);
Assert.True(json.ValueSpan == default);
Assert.False(json.ValueSequence.IsEmpty);
// Subsequent calls to Read clears the value properties since Read returned false.
Assert.False(json.Read());
Assert.False(json.HasValueSequence);
Assert.True(json.ValueSpan == default);
Assert.True(json.ValueSequence.IsEmpty);
}
}
public static void TextEqualDefault(string jsonString, bool expectedFound)
{
byte[] utf8Data = Encoding.UTF8.GetBytes(jsonString);
var json = new Utf8JsonReader(utf8Data, isFinalBlock: true, state: default);
while (json.Read())
{
if (json.TokenType == JsonTokenType.String)
{
Assert.Equal(expectedFound, json.ValueTextEquals(default(ReadOnlySpan <byte>)));
Assert.Equal(expectedFound, json.ValueTextEquals(default(ReadOnlySpan <char>)));
Assert.Equal(expectedFound, json.ValueTextEquals(default(string)));
break;
}
}
ReadOnlySequence <byte> sequence = JsonTestHelper.GetSequence(utf8Data, 1);
json = new Utf8JsonReader(sequence, isFinalBlock: true, state: default);
while (json.Read())
{
if (json.TokenType == JsonTokenType.String)
{
Assert.Equal(expectedFound, json.ValueTextEquals(default(ReadOnlySpan <byte>)));
Assert.Equal(expectedFound, json.ValueTextEquals(default(ReadOnlySpan <char>)));
Assert.Equal(expectedFound, json.ValueTextEquals(default(string)));
break;
}
}
}
public static void ReadJsonTokenWithExtraValueMultiSegment(string jsonString)
{
byte[] utf8 = Encoding.UTF8.GetBytes(jsonString);
ReadOnlySequence <byte> sequence = JsonTestHelper.GetSequence(utf8, 1);
foreach (JsonCommentHandling commentHandling in Enum.GetValues(typeof(JsonCommentHandling)))
{
TestReadTokenWithExtra(sequence, commentHandling, isFinalBlock: false);
TestReadTokenWithExtra(sequence, commentHandling, isFinalBlock: true);
}
}
public static void TestJsonReaderUtf8SegmentSizeOne(bool compactData, TestCaseType type, string jsonString)
{
// Remove all formatting/indendation
if (compactData)
{
using (JsonTextReader jsonReader = new JsonTextReader(new StringReader(jsonString)))
{
jsonReader.FloatParseHandling = FloatParseHandling.Decimal;
JToken jtoken = JToken.ReadFrom(jsonReader);
var stringWriter = new StringWriter();
using (JsonTextWriter jsonWriter = new JsonTextWriter(stringWriter))
{
jtoken.WriteTo(jsonWriter);
jsonString = stringWriter.ToString();
}
}
}
byte[] dataUtf8 = Encoding.UTF8.GetBytes(jsonString);
Stream stream = new MemoryStream(dataUtf8);
TextReader reader = new StreamReader(stream, Encoding.UTF8, false, 1024, true);
string expectedStr = JsonTestHelper.NewtonsoftReturnStringHelper(reader);
ReadOnlySequence <byte> sequence = JsonTestHelper.GetSequence(dataUtf8, 1);
// Skipping really large JSON since slicing them (O(n^2)) is too slow.
if (type == TestCaseType.Json40KB || type == TestCaseType.Json400KB || type == TestCaseType.ProjectLockJson)
{
var utf8JsonReader = new Utf8JsonReader(sequence, isFinalBlock: true, default);
byte[] resultSequence = JsonTestHelper.ReaderLoop(dataUtf8.Length, out int length, ref utf8JsonReader);
string actualStrSequence = Encoding.UTF8.GetString(resultSequence, 0, length);
Assert.Equal(expectedStr, actualStrSequence);
return;
}
for (int j = 0; j < dataUtf8.Length; j++)
{
var utf8JsonReader = new Utf8JsonReader(sequence.Slice(0, j), isFinalBlock: false, default);
byte[] resultSequence = JsonTestHelper.ReaderLoop(dataUtf8.Length, out int length, ref utf8JsonReader);
string actualStrSequence = Encoding.UTF8.GetString(resultSequence, 0, length);
long consumed = utf8JsonReader.BytesConsumed;
Assert.Equal(consumed, utf8JsonReader.CurrentState.BytesConsumed);
utf8JsonReader = new Utf8JsonReader(sequence.Slice(consumed), isFinalBlock: true, utf8JsonReader.CurrentState);
resultSequence = JsonTestHelper.ReaderLoop(dataUtf8.Length, out length, ref utf8JsonReader);
actualStrSequence += Encoding.UTF8.GetString(resultSequence, 0, length);
string message = $"Expected consumed: {dataUtf8.Length - consumed}, Actual consumed: {utf8JsonReader.BytesConsumed}, Index: {j}";
Assert.Equal(utf8JsonReader.BytesConsumed, utf8JsonReader.CurrentState.BytesConsumed);
Assert.True(dataUtf8.Length - consumed == utf8JsonReader.BytesConsumed, message);
Assert.Equal(expectedStr, actualStrSequence);
}
}
public static void JsonWithTrailingCommasMultiSegment_Valid(string jsonString)
{
byte[] utf8 = Encoding.UTF8.GetBytes(jsonString);
ReadOnlySequence <byte> sequence = JsonTestHelper.GetSequence(utf8, 1);
{
JsonReaderState state = default;
TrailingCommasHelper(sequence, state, allow: false, expectThrow: true);
}
{
var state = new JsonReaderState(options: default);
public static void AllowCommentStackMismatchMultiSegment(string jsonString, string expectedWithoutComments, string expectedWithComments)
{
byte[] data = Encoding.UTF8.GetBytes(jsonString);
ReadOnlySequence <byte> sequence = JsonTestHelper.GetSequence(data, 1);
TestReadingJsonWithComments(sequence, expectedWithoutComments, expectedWithComments);
var firstSegment = new BufferSegment <byte>(ReadOnlyMemory <byte> .Empty);
ReadOnlyMemory <byte> secondMem = data;
BufferSegment <byte> secondSegment = firstSegment.Append(secondMem);
sequence = new ReadOnlySequence <byte>(firstSegment, 0, secondSegment, secondMem.Length);
TestReadingJsonWithComments(sequence, expectedWithoutComments, expectedWithComments);
}
public static void SingleJsonValueMultiSegment(string jsonString, string expectedString)
{
byte[] dataUtf8 = Encoding.UTF8.GetBytes(jsonString);
ReadOnlySequence <byte> sequence = JsonTestHelper.GetSequence(dataUtf8, 1);
TestReadingSingleValueJson(sequence, expectedString);
var firstSegment = new BufferSegment <byte>(ReadOnlyMemory <byte> .Empty);
ReadOnlyMemory <byte> secondMem = dataUtf8;
BufferSegment <byte> secondSegment = firstSegment.Append(secondMem);
sequence = new ReadOnlySequence <byte>(firstSegment, 0, secondSegment, secondMem.Length);
TestReadingSingleValueJson(sequence, expectedString);
}
public static void TestTextEqualsTooLargeToMatch(string jsonString)
{
byte[] utf8Data = Encoding.UTF8.GetBytes(jsonString);
var lookupString = new string('a', 13);
bool found = false;
var json = new Utf8JsonReader(utf8Data, isFinalBlock: true, state: default);
while (json.Read())
{
if (json.TokenType == JsonTokenType.String)
{
if (json.ValueTextEquals(Encoding.UTF8.GetBytes(lookupString)) ||
json.ValueTextEquals(lookupString.AsSpan()) ||
json.ValueTextEquals(lookupString))
{
found = true;
break;
}
}
}
Assert.False(found);
ReadOnlySequence <byte> sequence = JsonTestHelper.GetSequence(utf8Data, 1);
found = false;
json = new Utf8JsonReader(sequence, isFinalBlock: true, state: default);
while (json.Read())
{
if (json.TokenType == JsonTokenType.String)
{
if (json.ValueTextEquals(Encoding.UTF8.GetBytes(lookupString)) ||
json.ValueTextEquals(lookupString.AsSpan()) ||
json.ValueTextEquals(lookupString))
{
found = true;
break;
}
}
}
Assert.False(found);
}
public static void ReadJsonTokenWithExtraValueAndCommentsAppendedMultiSegment(string jsonString)
{
jsonString = " /* comment */ /* comment */ " + jsonString;
byte[] utf8 = Encoding.UTF8.GetBytes(jsonString);
ReadOnlySequence <byte> sequence = JsonTestHelper.GetSequence(utf8, 1);
foreach (JsonCommentHandling commentHandling in Enum.GetValues(typeof(JsonCommentHandling)))
{
if (commentHandling == JsonCommentHandling.Disallow)
{
continue;
}
TestReadTokenWithExtra(sequence, commentHandling, isFinalBlock: false, commentsAppended: true);
TestReadTokenWithExtra(sequence, commentHandling, isFinalBlock: true, commentsAppended: true);
}
}
public static void TestTextEqualsValue(string jsonString, string lookUpString, bool expectedFound)
{
byte[] lookup = Encoding.UTF8.GetBytes(lookUpString);
byte[] utf8Data = Encoding.UTF8.GetBytes(jsonString);
bool found = false;
var json = new Utf8JsonReader(utf8Data, isFinalBlock: true, state: default);
while (json.Read())
{
if (json.TokenType == JsonTokenType.String)
{
if (json.ValueTextEquals(lookup) &&
json.ValueTextEquals(lookUpString) &&
json.ValueTextEquals(lookUpString.AsSpan()))
{
found = true;
break;
}
}
}
Assert.Equal(expectedFound, found);
ReadOnlySequence <byte> sequence = JsonTestHelper.GetSequence(utf8Data, 1);
found = false;
json = new Utf8JsonReader(sequence, isFinalBlock: true, state: default);
while (json.Read())
{
if (json.TokenType == JsonTokenType.String)
{
if (json.ValueTextEquals(lookup) &&
json.ValueTextEquals(lookUpString) &&
json.ValueTextEquals(lookUpString.AsSpan()))
{
found = true;
break;
}
}
}
Assert.Equal(expectedFound, found);
}
public static void TestTextEqualsTooSmallToMatch(string jsonString)
{
byte[] utf8Data = Encoding.UTF8.GetBytes(jsonString);
bool found = false;
var json = new Utf8JsonReader(utf8Data, isFinalBlock: true, state: default);
while (json.Read())
{
if (json.TokenType == JsonTokenType.String)
{
if (json.ValueTextEquals(new byte[] { (byte)'a' }) ||
json.ValueTextEquals(new char[] { 'a' }) ||
json.ValueTextEquals("a"))
{
found = true;
break;
}
}
}
Assert.False(found);
ReadOnlySequence <byte> sequence = JsonTestHelper.GetSequence(utf8Data, 1);
found = false;
json = new Utf8JsonReader(sequence, isFinalBlock: true, state: default);
while (json.Read())
{
if (json.TokenType == JsonTokenType.String)
{
if (json.ValueTextEquals(new byte[] { (byte)'a' }) ||
json.ValueTextEquals(new char[] { 'a' }) ||
json.ValueTextEquals("a"))
{
found = true;
break;
}
}
}
Assert.False(found);
}
public static void EmptyJsonWithinSequenceIsInvalid()
{
ReadOnlySequence <byte> sequence = JsonTestHelper.GetSequence(new byte[0], 1);
var json = new Utf8JsonReader(sequence, isFinalBlock: true, state: default);
try
{
while (json.Read())
{
;
}
Assert.True(false, "Expected JsonReaderException was not thrown with single-segment data.");
}
catch (JsonReaderException ex)
{
Assert.Equal(0, ex.LineNumber);
Assert.Equal(0, ex.BytePositionInLine);
}
}
public static void SkipSingleLineCommentMultiSpanTest(string expected)
{
string jsonData = "{" + expected + "}";
byte[] dataUtf8 = Encoding.UTF8.GetBytes(jsonData);
ReadOnlySequence <byte> sequence = JsonTestHelper.GetSequence(dataUtf8, 1);
for (int i = 0; i < jsonData.Length; i++)
{
var state = new JsonReaderState(options: new JsonReaderOptions {
CommentHandling = JsonCommentHandling.Skip
});
var json = new Utf8JsonReader(sequence.Slice(0, i), isFinalBlock: false, state);
VerifyReadLoop(ref json, null);
json = new Utf8JsonReader(sequence.Slice(state.BytesConsumed), isFinalBlock: true, state);
VerifyReadLoop(ref json, null);
}
}
private static void ReadFullySegmentSizeOne(bool compactData, TestCaseType type, string jsonString)
{
// Remove all formatting/indendation
if (compactData)
{
jsonString = JsonTestHelper.GetCompactString(jsonString);
}
byte[] dataUtf8 = Encoding.UTF8.GetBytes(jsonString);
Stream stream = new MemoryStream(dataUtf8);
TextReader reader = new StreamReader(stream, Encoding.UTF8, false, 1024, true);
string expectedStr = JsonTestHelper.NewtonsoftReturnStringHelper(reader);
ReadOnlySequence <byte> sequence = JsonTestHelper.GetSequence(dataUtf8, 1);
var utf8JsonReader = new Utf8JsonReader(sequence, isFinalBlock: true, default);
byte[] resultSequence = JsonTestHelper.ReaderLoop(dataUtf8.Length, out int length, ref utf8JsonReader);
string actualStrSequence = Encoding.UTF8.GetString(resultSequence, 0, length);
Assert.Equal(expectedStr, actualStrSequence);
}
public static void TestTextEqualsLargeMatch()
{
var jsonChars = new char[320]; // Some value larger than 256 (stack threshold)
jsonChars.AsSpan().Fill('a');
byte[] lookup = Encoding.UTF8.GetBytes(jsonChars);
ReadOnlySpan <char> escapedA = new char[6] {
'\\', 'u', '0', '0', '6', '1'
};
ReadOnlySpan <byte> lookupSpan = lookup.AsSpan(0, lookup.Length - escapedA.Length + 1); // remove extra characters that were replaced by escaped bytes
Span <char> lookupChars = new char[jsonChars.Length];
jsonChars.CopyTo(lookupChars);
lookupChars = lookupChars.Slice(0, lookupChars.Length - escapedA.Length + 1);
// Replacing 'a' with '\u0061', so a net change of 5.
// escapedA.Length - 1 = 6 - 1 = 5
for (int i = 0; i < jsonChars.Length - escapedA.Length + 1; i++)
{
jsonChars.AsSpan().Fill('a');
escapedA.CopyTo(jsonChars.AsSpan(i));
string jsonString = "\"" + new string(jsonChars) + "\"";
byte[] utf8Data = Encoding.UTF8.GetBytes(jsonString);
bool found = false;
var json = new Utf8JsonReader(utf8Data, isFinalBlock: true, state: default);
while (json.Read())
{
if (json.TokenType == JsonTokenType.String)
{
if (json.ValueTextEquals(lookupSpan) &&
json.ValueTextEquals(lookupChars) &&
json.ValueTextEquals(new string(lookupChars.ToArray())))
{
found = true;
break;
}
}
}
Assert.True(found, $"Json String: {jsonString}");
ReadOnlySequence <byte> sequence = JsonTestHelper.GetSequence(utf8Data, 1);
found = false;
json = new Utf8JsonReader(sequence, isFinalBlock: true, state: default);
while (json.Read())
{
if (json.TokenType == JsonTokenType.String)
{
if (json.ValueTextEquals(lookupSpan) &&
json.ValueTextEquals(lookupChars) &&
json.ValueTextEquals(new string(lookupChars.ToArray())))
{
found = true;
break;
}
}
}
Assert.True(found, $"Json String: {jsonString} | Look up: {Encoding.UTF8.GetString(lookupSpan.ToArray())}");
}
}
public static void TestTextEqualsLargeMismatch()
{
var jsonChars = new char[320]; // Some value larger than 256 (stack threshold)
jsonChars.AsSpan().Fill('a');
ReadOnlySpan <char> escapedA = new char[6] {
'\\', 'u', '0', '0', '6', '1'
};
byte[] originalLookup = Encoding.UTF8.GetBytes(jsonChars);
char[] originalLookupChars = new char[jsonChars.Length];
Array.Copy(jsonChars, originalLookupChars, jsonChars.Length);
for (int i = 1; i < jsonChars.Length - 6; i++)
{
jsonChars.AsSpan().Fill('a');
escapedA.CopyTo(jsonChars.AsSpan(i));
string jsonString = "\"" + new string(jsonChars) + "\"";
byte[] utf8Data = Encoding.UTF8.GetBytes(jsonString);
for (int j = 0; j < 3; j++)
{
Span <byte> lookup = new byte[originalLookup.Length];
originalLookup.CopyTo(lookup);
lookup = lookup.Slice(0, lookup.Length - escapedA.Length + 1); // remove extra characters that were replaced by escaped bytes
Span <char> lookupChars = new char[originalLookupChars.Length];
originalLookupChars.CopyTo(lookupChars);
lookupChars = lookupChars.Slice(0, lookupChars.Length - escapedA.Length + 1); // remove extra characters that were replaced by escaped bytes
switch (j)
{
case 0:
lookup[i] = (byte)'b';
lookupChars[i] = 'b';
break;
case 1:
lookup[i + 1] = (byte)'b';
lookupChars[i + 1] = 'b';
break;
case 2:
lookup[i - 1] = (byte)'b';
lookupChars[i - 1] = 'b';
break;
}
bool found = false;
var json = new Utf8JsonReader(utf8Data, isFinalBlock: true, state: default);
while (json.Read())
{
if (json.TokenType == JsonTokenType.String)
{
if (json.ValueTextEquals(lookup) ||
json.ValueTextEquals(lookupChars) ||
json.ValueTextEquals(new string(lookupChars.ToArray())))
{
found = true;
break;
}
}
}
Assert.False(found, $"Json String: {jsonString}");
ReadOnlySequence <byte> sequence = JsonTestHelper.GetSequence(utf8Data, 1);
found = false;
json = new Utf8JsonReader(sequence, isFinalBlock: true, state: default);
while (json.Read())
{
if (json.TokenType == JsonTokenType.String)
{
if (json.ValueTextEquals(lookup) ||
json.ValueTextEquals(lookupChars) ||
json.ValueTextEquals(new string(lookupChars.ToArray())))
{
found = true;
break;
}
}
}
Assert.False(found);
}
}
}