private static CorpusZipMetadata ReadCorpusMetadata(ref Utf8JsonStreamReader reader)
{
IReadOnlyList <DocumentId> firstDocumentIdInBlock = Array.Empty <DocumentId>();
H.ReadToken(ref reader, JsonTokenType.StartObject);
while (reader.Read() && reader.TokenType == JsonTokenType.PropertyName)
{
var propertyName = reader.GetString();
switch (propertyName)
{
case FirstDocumentIdInBlockPropertyName:
firstDocumentIdInBlock = ReadFirstDocumentIdInBlock(ref reader);
break;
default:
throw new InvalidDataException($"Unexpected property at position {reader.Position}: {propertyName}");
}
}
if (firstDocumentIdInBlock.Count == 0)
{
throw new InvalidDataException("Mandatory field ReadFirstDocumentIdInBlock was not found");
}
return(new CorpusZipMetadata(firstDocumentIdInBlock));
}
public async Task DeserializeFromStreamSequencingTest(
string value,
int bufferSize,
int expectedRentCalls,
string getBuffersMethodName)
{
MethodInfo?getBuffersMethod = typeof(Utf8JsonStreamReaderTests).GetMethod(getBuffersMethodName, BindingFlags.Static | BindingFlags.NonPublic);
if (getBuffersMethod == null)
{
throw new InvalidOperationException($"getBuffersMethodName [{getBuffersMethodName}] could not be found reflectively.");
}
int rentCalls = 0;
int returnCalls = 0;
Utf8JsonStreamReader.RentBufferFunc = (size) =>
{
rentCalls++;
return(new byte[size]);
};
Utf8JsonStreamReader.ReturnBufferAction = (buffer) => returnCalls++;
using TestStream stream = new((Memory <byte>[])getBuffersMethod.Invoke(null, null) !);
TestClass instance = new();
await Utf8JsonStreamReader.DeserializeAsync(stream, instance, ReadMethod, bufferSize).ConfigureAwait(false);
Assert.AreEqual(1, instance.PropertyName);
Assert.AreEqual(value, instance.PropertyName2);
Assert.AreEqual(expectedRentCalls, rentCalls);
Assert.AreEqual(expectedRentCalls, returnCalls);
}
private static DocumentMetadata ReadDocumentMetadata(ref Utf8JsonStreamReader reader)
{
var id = DocumentId.Zero;
var title = string.Empty;
while (reader.Read() && reader.TokenType == JsonTokenType.PropertyName)
{
var propertyName = reader.GetString();
switch (propertyName)
{
case DocumentIdPropertyName:
id = DocumentId.Parse(JsonReaderHelper.ReadString(ref reader).AsSpan());
break;
case TitlePropertyName:
title = JsonReaderHelper.ReadString(ref reader);
break;
default:
throw new InvalidDataException($"Unexpected property at position {reader.Position}: {propertyName}");
}
}
return(new DocumentMetadata(id, title));
}
private static BlockMetadata ReadBlockMetadata(ref Utf8JsonStreamReader reader)
{
ushort id = 0;
IList <DocumentMetadata> docs = Array.Empty <DocumentMetadata>();
JsonReaderHelper.ReadToken(ref reader, JsonTokenType.StartObject);
while (reader.Read() && reader.TokenType != JsonTokenType.EndObject)
{
var propertyName = reader.GetString();
switch (propertyName)
{
case BlockIdPropertyName:
id = ParseId(JsonReaderHelper.ReadString(ref reader).AsSpan());
break;
case DocumentMetadataListPropertyName:
docs = ReadDocumentsMetadata(ref reader);
break;
default:
throw new InvalidDataException($"Unexpected property at position {reader.Position}: {propertyName}");
}
}
return(new BlockMetadata(id, docs));
}
/// <summary>
/// Read the start date from the stream and consume to the end of the payload.
/// </summary>
/// <param name="streamReader">The reader from which to read the start date.</param>
/// <returns>The start date read from the payload.</returns>
public static DateTimeOffset ReadStartDate(ref Utf8JsonStreamReader streamReader)
{
if (streamReader.TokenType != JsonTokenType.StartObject)
{
throw new JsonException("Expected to be at the start of the event payload.");
}
// Read the ID
streamReader.Read();
if (streamReader.TokenType != JsonTokenType.PropertyName || !streamReader.Match(StartDatePropertyString))
{
throw new JsonException($"Expected to find the {StartDatePropertyString}.");
}
streamReader.Read();
if (streamReader.TokenType != JsonTokenType.String)
{
throw new JsonException("Expected to find a string-encoded DateTimeOffset property.");
}
DateTimeOffset startDate = streamReader.GetDateTimeOffset();
// Read to the end of the object
streamReader.Read();
if (streamReader.TokenType != JsonTokenType.EndObject)
{
throw new JsonException("Expected to be at the end of the event payload.");
}
return(startDate);
}
/// <summary>
/// Read the ToDoItemId from the stream and consume to the end of the payload.
/// </summary>
/// <param name="streamReader">The reader from which to read the id.</param>
/// <returns>The ID read from the payload.</returns>
public static Guid ReadToDoItemId(ref Utf8JsonStreamReader streamReader)
{
if (streamReader.TokenType != JsonTokenType.StartObject)
{
throw new JsonException("Expected to be at the start of the event payload.");
}
// Read the ID
streamReader.Read();
if (streamReader.TokenType != JsonTokenType.PropertyName || !streamReader.Match(IdPropertyString))
{
throw new JsonException($"Expected to find the {IdPropertyString}.");
}
streamReader.Read();
if (streamReader.TokenType != JsonTokenType.String)
{
throw new JsonException("Expected to find a string-encoded GUID property.");
}
Guid id = streamReader.GetGuid();
// Read to the end of the object
streamReader.Read();
if (streamReader.TokenType != JsonTokenType.EndObject)
{
throw new JsonException("Expected to be at the end of the event payload.");
}
return(id);
}
/// <summary>
/// Read the owner from the stream and consume to the end of the payload.
/// </summary>
/// <param name="streamReader">The reader from which to read the id.</param>
/// <returns>The ID read from the payload.</returns>
public static string ReadOwner(ref Utf8JsonStreamReader streamReader)
{
if (streamReader.TokenType != JsonTokenType.StartObject)
{
throw new JsonException("Expected to be at the start of the event payload.");
}
// Read the ID
streamReader.Read();
if (streamReader.TokenType != JsonTokenType.PropertyName || !streamReader.Match(OwnerPropertyString))
{
throw new JsonException($"Expected to find the {OwnerPropertyString}.");
}
streamReader.Read();
if (streamReader.TokenType != JsonTokenType.String)
{
throw new JsonException("Expected to find a string property.");
}
string owner = streamReader.GetString() ?? throw new JsonException("Expected to find an owner.");
// Read to the end of the object
streamReader.Read();
if (streamReader.TokenType != JsonTokenType.EndObject)
{
throw new JsonException("Expected to be at the end of the event payload.");
}
return(owner);
}
private static void ConsumeToken(ref Utf8JsonStreamReader reader, JsonTokenType expected)
{
if (reader.Read() && expected == reader.TokenType)
{
return;
}
ThrowUnexpectedTokenException(ref reader, expected);
}
private static void ConsumeToken(ref Utf8JsonStreamReader reader, JsonTokenType expected)
{
if (reader.Read() && expected == reader.TokenType)
{
return;
}
throw new WorkloadManifestFormatException(Strings.ExpectedTokenAtOffset, expected, reader.TokenStartIndex);
}
public static WorkloadManifest ReadWorkloadManifest(string manifestId, Stream manifestStream)
{
using var textReader = new StreamReader(manifestStream, System.Text.Encoding.UTF8, true);
using var jsonReader = new JsonTextReader(textReader);
var reader = new Utf8JsonStreamReader(jsonReader);
return(ReadWorkloadManifest(manifestId, ref reader));
}
private static string ReadString(ref Utf8JsonStreamReader reader)
{
if (reader.Read() && reader.TokenType == JsonTokenType.String)
{
return(reader.GetString());
}
throw new WorkloadManifestFormatException(Strings.ExpectedStringAtOffset, reader.TokenStartIndex);
}
/// <summary>
/// Read to the end of the event, having read to the end of the payload.
/// </summary>
/// <param name="streamReader">The <see cref="Utf8JsonStreamReader"/> from which to read the payload.</param>
public static void ReadToEndOfEvent(ref Utf8JsonStreamReader streamReader)
{
// Skip to the end of the event
streamReader.Read();
if (streamReader.TokenType != JsonTokenType.EndObject)
{
throw new JsonException("Expected to see the end of the Event object.");
}
}
private static bool ReadBool(ref Utf8JsonStreamReader reader)
{
if (reader.Read() && reader.TokenType.IsBool())
{
return(reader.GetBool());
}
throw new WorkloadManifestFormatException(Strings.ExpectedBoolAtOffset, reader.TokenStartIndex);
}
public static WorkloadManifest ReadWorkloadManifest(string manifestId, Stream manifestStream, Stream?localizationStream, string?informationalPath = null)
{
using var textReader = new StreamReader(manifestStream, System.Text.Encoding.UTF8, true);
using var jsonReader = new JsonTextReader(textReader);
var manifestReader = new Utf8JsonStreamReader(jsonReader);
return(ReadWorkloadManifest(manifestId, informationalPath, ReadLocalizationCatalog(localizationStream), ref manifestReader));;
}
/// <summary>
/// Find the value of the payload property in the event.
/// </summary>
/// <param name="streamReader">The <see cref="Utf8JsonStreamReader"/> from which we are reading the event.</param>
public static void FindPayload(ref Utf8JsonStreamReader streamReader)
{
streamReader.Read();
if (streamReader.TokenType != JsonTokenType.PropertyName || !streamReader.Match(EventPayloadPropertyNameString))
{
throw new JsonException($"Expected to find the {EventPayloadPropertyNameString} property.");
}
// position at the start of the payload
streamReader.Read();
}
private static LocalizationCatalog?ReadLocalizationCatalog(Stream?localizationStream, JsonReaderOptions readerOptions)
{
if (localizationStream == null)
{
return(null);
}
var localizationReader = new Utf8JsonStreamReader(localizationStream, readerOptions);
return(ReadLocalizationCatalog(ref localizationReader));
}
public static WorkloadManifest ReadWorkloadManifest(Stream manifestStream)
{
var readerOptions = new JsonReaderOptions
{
AllowTrailingCommas = true,
CommentHandling = JsonCommentHandling.Skip
};
var reader = new Utf8JsonStreamReader(manifestStream, readerOptions);
return(ReadWorkloadManifest(ref reader));
}
public static WorkloadManifest ReadWorkloadManifest(string manifestId, Stream manifestStream, string?informationalPath = null)
{
var readerOptions = new JsonReaderOptions
{
AllowTrailingCommas = true,
CommentHandling = JsonCommentHandling.Skip
};
var reader = new Utf8JsonStreamReader(manifestStream, readerOptions);
return(ReadWorkloadManifest(manifestId, informationalPath, ref reader));
}
private static IReadOnlyList <DocumentId> ReadFirstDocumentIdInBlock(ref Utf8JsonStreamReader reader)
{
var firstDocumentIdInBlock = new List <DocumentId>();
H.ReadToken(ref reader, JsonTokenType.StartArray);
while (reader.Read() && reader.TokenType != JsonTokenType.EndArray)
{
firstDocumentIdInBlock.Add(new DocumentId(reader.GetUInt32()));
}
return(firstDocumentIdInBlock);
}
private static long ReadInt64(ref Utf8JsonStreamReader reader)
{
if (reader.Read() && reader.TokenType.IsInt())
{
if (reader.TryGetInt64(out long value))
{
return(value);
}
}
throw new WorkloadManifestFormatException(Strings.ExpectedIntegerAtOffset, reader.TokenStartIndex);
}
private static bool FindDocumentsProperty(ref Utf8JsonStreamReader streamReader)
{
while (streamReader.TokenType != JsonTokenType.PropertyName || !streamReader.Match(DocumentsName))
{
if (!streamReader.Read())
{
return(false);
}
}
return(true);
}
private static IList <DocumentMetadata> ReadDocumentsMetadata(ref Utf8JsonStreamReader reader)
{
var docs = new List <DocumentMetadata>();
JsonReaderHelper.ReadToken(ref reader, JsonTokenType.StartArray);
while (reader.Read() && reader.TokenType != JsonTokenType.EndArray)
{
var doc = ReadDocumentMetadata(ref reader);
docs.Add(doc);
}
return(docs);
}
public static WorkloadManifest ReadWorkloadManifest(string manifestId, Stream manifestStream, Stream?localizationStream, string manifestPath)
{
var readerOptions = new JsonReaderOptions
{
AllowTrailingCommas = true,
CommentHandling = JsonCommentHandling.Skip
};
var localizationCatalog = ReadLocalizationCatalog(localizationStream, readerOptions);
var manifestReader = new Utf8JsonStreamReader(manifestStream, readerOptions);
return(ReadWorkloadManifest(manifestId, manifestPath, localizationCatalog, ref manifestReader));
}
/// <summary>
/// Find the event type value in the stream reader.
/// </summary>
/// <param name="streamReader">The stream reader in which to find the event type.</param>
public static void FindEventType(ref Utf8JsonStreamReader streamReader)
{
streamReader.Read();
if (streamReader.TokenType != JsonTokenType.PropertyName || !streamReader.Match(EventTypePropertyNameString))
{
throw new JsonException($"Expected to find the {EventTypePropertyNameString} property.");
}
streamReader.Read();
if (streamReader.TokenType != JsonTokenType.String)
{
throw new JsonException($"Expected the {EventTypePropertyNameString} property to be a string.");
}
}
private static LocalizationCatalog?ReadLocalizationCatalog(Stream?localizationStream)
{
if (localizationStream == null)
{
return(null);
}
using var textReader = new StreamReader(localizationStream, System.Text.Encoding.UTF8, true);
using var jsonReader = new JsonTextReader(textReader);
var localizationReader = new Utf8JsonStreamReader(jsonReader);
return(ReadLocalizationCatalog(ref localizationReader));
}
/// <summary>
/// Read the event sequence number from the <see cref="Utf8JsonStreamReader"/>.
/// </summary>
/// <param name="streamReader">The stream reader from which to read the event sequence number.</param>
/// <returns>The event sequence number.</returns>
public static long ReadEventSequenceNumber(ref Utf8JsonStreamReader streamReader)
{
streamReader.Read();
if (streamReader.TokenType != JsonTokenType.PropertyName || !streamReader.Match(EventSequenceNumberPropertyNameString))
{
throw new JsonException($"Expected to find the {EventSequenceNumberPropertyNameString} property.");
}
streamReader.Read();
if (streamReader.TokenType != JsonTokenType.Number)
{
throw new JsonException($"Expected the {EventSequenceNumberPropertyNameString} property to be a number.");
}
return(streamReader.GetInt64());
}
private static void ProcessEventFeedStream <TEventHandler>(TEventHandler eventHandler, Stream content, int defaultBufferSize, CancellationToken cancellationToken)
where TEventHandler : IJsonEventFeedHandler
{
var streamReader = new Utf8JsonStreamReader(content, defaultBufferSize);
if (!FindDocumentsProperty(ref streamReader))
{
throw new JsonException("Unable to find the Documents property in the output stream.");
}
if (!FindDocumentsArray(ref streamReader))
{
throw new JsonException("The Documents propery was expected to be a JSON array.");
}
// We are now at the start of an array of commits
JsonEventFeed.ProcessCommits(eventHandler, ref streamReader, cancellationToken);
}
private static void ThrowUnexpectedTokenException(ref Utf8JsonStreamReader reader, JsonTokenType expected)
{
string key;
if (expected.IsBool())
{
key = Strings.ExpectedBoolAtOffset;
}
else if (expected.IsInt())
{
key = Strings.ExpectedIntegerAtOffset;
}
else if (expected == JsonTokenType.String)
{
key = Strings.ExpectedStringAtOffset;
}
else
{
throw new WorkloadManifestFormatException(Strings.ExpectedTokenAtOffset, expected, reader.TokenStartIndex);
}
throw new WorkloadManifestFormatException(key, reader.TokenStartIndex);
}
public async Task DeserializeFromStreamTest()
{
int rentCalls = 0;
int returnCalls = 0;
Utf8JsonStreamReader.RentBufferFunc = (size) =>
{
rentCalls++;
return(new byte[size]);
};
Utf8JsonStreamReader.ReturnBufferAction = (buffer) => returnCalls++;
using MemoryStream stream = new(Encoding.UTF8.GetBytes("{\"PropertyName\":1, \"PropertyName2\":\"value\"}"));
TestClass instance = new();
await Utf8JsonStreamReader.DeserializeAsync(stream, instance, ReadMethod, bufferSize : 8192).ConfigureAwait(false);
Assert.AreEqual(1, instance.PropertyName);
Assert.AreEqual("value", instance.PropertyName2);
Assert.AreEqual(1, rentCalls);
Assert.AreEqual(1, returnCalls);
}
private static string?ReadStringLocalized(ref Utf8JsonStreamReader reader, LocalizationCatalog?localizationCatalog, string id)
{
ConsumeToken(ref reader, JsonTokenType.String);
return(localizationCatalog?.Localize(id) ?? reader.GetString());
}
