private static CorpusZipMetadata ReadCorpusMetadata(ref Utf8JsonStreamReader reader) { IReadOnlyList <DocumentId> firstDocumentIdInBlock = Array.Empty <DocumentId>(); H.ReadToken(ref reader, JsonTokenType.StartObject); while (reader.Read() && reader.TokenType == JsonTokenType.PropertyName) { var propertyName = reader.GetString(); switch (propertyName) { case FirstDocumentIdInBlockPropertyName: firstDocumentIdInBlock = ReadFirstDocumentIdInBlock(ref reader); break; default: throw new InvalidDataException($"Unexpected property at position {reader.Position}: {propertyName}"); } } if (firstDocumentIdInBlock.Count == 0) { throw new InvalidDataException("Mandatory field ReadFirstDocumentIdInBlock was not found"); } return(new CorpusZipMetadata(firstDocumentIdInBlock)); }
public async Task DeserializeFromStreamSequencingTest( string value, int bufferSize, int expectedRentCalls, string getBuffersMethodName) { MethodInfo?getBuffersMethod = typeof(Utf8JsonStreamReaderTests).GetMethod(getBuffersMethodName, BindingFlags.Static | BindingFlags.NonPublic); if (getBuffersMethod == null) { throw new InvalidOperationException($"getBuffersMethodName [{getBuffersMethodName}] could not be found reflectively."); } int rentCalls = 0; int returnCalls = 0; Utf8JsonStreamReader.RentBufferFunc = (size) => { rentCalls++; return(new byte[size]); }; Utf8JsonStreamReader.ReturnBufferAction = (buffer) => returnCalls++; using TestStream stream = new((Memory <byte>[])getBuffersMethod.Invoke(null, null) !); TestClass instance = new(); await Utf8JsonStreamReader.DeserializeAsync(stream, instance, ReadMethod, bufferSize).ConfigureAwait(false); Assert.AreEqual(1, instance.PropertyName); Assert.AreEqual(value, instance.PropertyName2); Assert.AreEqual(expectedRentCalls, rentCalls); Assert.AreEqual(expectedRentCalls, returnCalls); }
private static DocumentMetadata ReadDocumentMetadata(ref Utf8JsonStreamReader reader) { var id = DocumentId.Zero; var title = string.Empty; while (reader.Read() && reader.TokenType == JsonTokenType.PropertyName) { var propertyName = reader.GetString(); switch (propertyName) { case DocumentIdPropertyName: id = DocumentId.Parse(JsonReaderHelper.ReadString(ref reader).AsSpan()); break; case TitlePropertyName: title = JsonReaderHelper.ReadString(ref reader); break; default: throw new InvalidDataException($"Unexpected property at position {reader.Position}: {propertyName}"); } } return(new DocumentMetadata(id, title)); }
private static BlockMetadata ReadBlockMetadata(ref Utf8JsonStreamReader reader) { ushort id = 0; IList <DocumentMetadata> docs = Array.Empty <DocumentMetadata>(); JsonReaderHelper.ReadToken(ref reader, JsonTokenType.StartObject); while (reader.Read() && reader.TokenType != JsonTokenType.EndObject) { var propertyName = reader.GetString(); switch (propertyName) { case BlockIdPropertyName: id = ParseId(JsonReaderHelper.ReadString(ref reader).AsSpan()); break; case DocumentMetadataListPropertyName: docs = ReadDocumentsMetadata(ref reader); break; default: throw new InvalidDataException($"Unexpected property at position {reader.Position}: {propertyName}"); } } return(new BlockMetadata(id, docs)); }
/// <summary> /// Read the start date from the stream and consume to the end of the payload. /// </summary> /// <param name="streamReader">The reader from which to read the start date.</param> /// <returns>The start date read from the payload.</returns> public static DateTimeOffset ReadStartDate(ref Utf8JsonStreamReader streamReader) { if (streamReader.TokenType != JsonTokenType.StartObject) { throw new JsonException("Expected to be at the start of the event payload."); } // Read the ID streamReader.Read(); if (streamReader.TokenType != JsonTokenType.PropertyName || !streamReader.Match(StartDatePropertyString)) { throw new JsonException($"Expected to find the {StartDatePropertyString}."); } streamReader.Read(); if (streamReader.TokenType != JsonTokenType.String) { throw new JsonException("Expected to find a string-encoded DateTimeOffset property."); } DateTimeOffset startDate = streamReader.GetDateTimeOffset(); // Read to the end of the object streamReader.Read(); if (streamReader.TokenType != JsonTokenType.EndObject) { throw new JsonException("Expected to be at the end of the event payload."); } return(startDate); }
/// <summary> /// Read the ToDoItemId from the stream and consume to the end of the payload. /// </summary> /// <param name="streamReader">The reader from which to read the id.</param> /// <returns>The ID read from the payload.</returns> public static Guid ReadToDoItemId(ref Utf8JsonStreamReader streamReader) { if (streamReader.TokenType != JsonTokenType.StartObject) { throw new JsonException("Expected to be at the start of the event payload."); } // Read the ID streamReader.Read(); if (streamReader.TokenType != JsonTokenType.PropertyName || !streamReader.Match(IdPropertyString)) { throw new JsonException($"Expected to find the {IdPropertyString}."); } streamReader.Read(); if (streamReader.TokenType != JsonTokenType.String) { throw new JsonException("Expected to find a string-encoded GUID property."); } Guid id = streamReader.GetGuid(); // Read to the end of the object streamReader.Read(); if (streamReader.TokenType != JsonTokenType.EndObject) { throw new JsonException("Expected to be at the end of the event payload."); } return(id); }
/// <summary> /// Read the owner from the stream and consume to the end of the payload. /// </summary> /// <param name="streamReader">The reader from which to read the id.</param> /// <returns>The ID read from the payload.</returns> public static string ReadOwner(ref Utf8JsonStreamReader streamReader) { if (streamReader.TokenType != JsonTokenType.StartObject) { throw new JsonException("Expected to be at the start of the event payload."); } // Read the ID streamReader.Read(); if (streamReader.TokenType != JsonTokenType.PropertyName || !streamReader.Match(OwnerPropertyString)) { throw new JsonException($"Expected to find the {OwnerPropertyString}."); } streamReader.Read(); if (streamReader.TokenType != JsonTokenType.String) { throw new JsonException("Expected to find a string property."); } string owner = streamReader.GetString() ?? throw new JsonException("Expected to find an owner."); // Read to the end of the object streamReader.Read(); if (streamReader.TokenType != JsonTokenType.EndObject) { throw new JsonException("Expected to be at the end of the event payload."); } return(owner); }
private static void ConsumeToken(ref Utf8JsonStreamReader reader, JsonTokenType expected) { if (reader.Read() && expected == reader.TokenType) { return; } ThrowUnexpectedTokenException(ref reader, expected); }
private static void ConsumeToken(ref Utf8JsonStreamReader reader, JsonTokenType expected) { if (reader.Read() && expected == reader.TokenType) { return; } throw new WorkloadManifestFormatException(Strings.ExpectedTokenAtOffset, expected, reader.TokenStartIndex); }
public static WorkloadManifest ReadWorkloadManifest(string manifestId, Stream manifestStream) { using var textReader = new StreamReader(manifestStream, System.Text.Encoding.UTF8, true); using var jsonReader = new JsonTextReader(textReader); var reader = new Utf8JsonStreamReader(jsonReader); return(ReadWorkloadManifest(manifestId, ref reader)); }
private static string ReadString(ref Utf8JsonStreamReader reader) { if (reader.Read() && reader.TokenType == JsonTokenType.String) { return(reader.GetString()); } throw new WorkloadManifestFormatException(Strings.ExpectedStringAtOffset, reader.TokenStartIndex); }
/// <summary> /// Read to the end of the event, having read to the end of the payload. /// </summary> /// <param name="streamReader">The <see cref="Utf8JsonStreamReader"/> from which to read the payload.</param> public static void ReadToEndOfEvent(ref Utf8JsonStreamReader streamReader) { // Skip to the end of the event streamReader.Read(); if (streamReader.TokenType != JsonTokenType.EndObject) { throw new JsonException("Expected to see the end of the Event object."); } }
private static bool ReadBool(ref Utf8JsonStreamReader reader) { if (reader.Read() && reader.TokenType.IsBool()) { return(reader.GetBool()); } throw new WorkloadManifestFormatException(Strings.ExpectedBoolAtOffset, reader.TokenStartIndex); }
public static WorkloadManifest ReadWorkloadManifest(string manifestId, Stream manifestStream, Stream?localizationStream, string?informationalPath = null) { using var textReader = new StreamReader(manifestStream, System.Text.Encoding.UTF8, true); using var jsonReader = new JsonTextReader(textReader); var manifestReader = new Utf8JsonStreamReader(jsonReader); return(ReadWorkloadManifest(manifestId, informationalPath, ReadLocalizationCatalog(localizationStream), ref manifestReader));; }
/// <summary> /// Find the value of the payload property in the event. /// </summary> /// <param name="streamReader">The <see cref="Utf8JsonStreamReader"/> from which we are reading the event.</param> public static void FindPayload(ref Utf8JsonStreamReader streamReader) { streamReader.Read(); if (streamReader.TokenType != JsonTokenType.PropertyName || !streamReader.Match(EventPayloadPropertyNameString)) { throw new JsonException($"Expected to find the {EventPayloadPropertyNameString} property."); } // position at the start of the payload streamReader.Read(); }
private static LocalizationCatalog?ReadLocalizationCatalog(Stream?localizationStream, JsonReaderOptions readerOptions) { if (localizationStream == null) { return(null); } var localizationReader = new Utf8JsonStreamReader(localizationStream, readerOptions); return(ReadLocalizationCatalog(ref localizationReader)); }
public static WorkloadManifest ReadWorkloadManifest(Stream manifestStream) { var readerOptions = new JsonReaderOptions { AllowTrailingCommas = true, CommentHandling = JsonCommentHandling.Skip }; var reader = new Utf8JsonStreamReader(manifestStream, readerOptions); return(ReadWorkloadManifest(ref reader)); }
public static WorkloadManifest ReadWorkloadManifest(string manifestId, Stream manifestStream, string?informationalPath = null) { var readerOptions = new JsonReaderOptions { AllowTrailingCommas = true, CommentHandling = JsonCommentHandling.Skip }; var reader = new Utf8JsonStreamReader(manifestStream, readerOptions); return(ReadWorkloadManifest(manifestId, informationalPath, ref reader)); }
private static IReadOnlyList <DocumentId> ReadFirstDocumentIdInBlock(ref Utf8JsonStreamReader reader) { var firstDocumentIdInBlock = new List <DocumentId>(); H.ReadToken(ref reader, JsonTokenType.StartArray); while (reader.Read() && reader.TokenType != JsonTokenType.EndArray) { firstDocumentIdInBlock.Add(new DocumentId(reader.GetUInt32())); } return(firstDocumentIdInBlock); }
private static long ReadInt64(ref Utf8JsonStreamReader reader) { if (reader.Read() && reader.TokenType.IsInt()) { if (reader.TryGetInt64(out long value)) { return(value); } } throw new WorkloadManifestFormatException(Strings.ExpectedIntegerAtOffset, reader.TokenStartIndex); }
private static bool FindDocumentsProperty(ref Utf8JsonStreamReader streamReader) { while (streamReader.TokenType != JsonTokenType.PropertyName || !streamReader.Match(DocumentsName)) { if (!streamReader.Read()) { return(false); } } return(true); }
private static IList <DocumentMetadata> ReadDocumentsMetadata(ref Utf8JsonStreamReader reader) { var docs = new List <DocumentMetadata>(); JsonReaderHelper.ReadToken(ref reader, JsonTokenType.StartArray); while (reader.Read() && reader.TokenType != JsonTokenType.EndArray) { var doc = ReadDocumentMetadata(ref reader); docs.Add(doc); } return(docs); }
public static WorkloadManifest ReadWorkloadManifest(string manifestId, Stream manifestStream, Stream?localizationStream, string manifestPath) { var readerOptions = new JsonReaderOptions { AllowTrailingCommas = true, CommentHandling = JsonCommentHandling.Skip }; var localizationCatalog = ReadLocalizationCatalog(localizationStream, readerOptions); var manifestReader = new Utf8JsonStreamReader(manifestStream, readerOptions); return(ReadWorkloadManifest(manifestId, manifestPath, localizationCatalog, ref manifestReader)); }
/// <summary> /// Find the event type value in the stream reader. /// </summary> /// <param name="streamReader">The stream reader in which to find the event type.</param> public static void FindEventType(ref Utf8JsonStreamReader streamReader) { streamReader.Read(); if (streamReader.TokenType != JsonTokenType.PropertyName || !streamReader.Match(EventTypePropertyNameString)) { throw new JsonException($"Expected to find the {EventTypePropertyNameString} property."); } streamReader.Read(); if (streamReader.TokenType != JsonTokenType.String) { throw new JsonException($"Expected the {EventTypePropertyNameString} property to be a string."); } }
private static LocalizationCatalog?ReadLocalizationCatalog(Stream?localizationStream) { if (localizationStream == null) { return(null); } using var textReader = new StreamReader(localizationStream, System.Text.Encoding.UTF8, true); using var jsonReader = new JsonTextReader(textReader); var localizationReader = new Utf8JsonStreamReader(jsonReader); return(ReadLocalizationCatalog(ref localizationReader)); }
/// <summary> /// Read the event sequence number from the <see cref="Utf8JsonStreamReader"/>. /// </summary> /// <param name="streamReader">The stream reader from which to read the event sequence number.</param> /// <returns>The event sequence number.</returns> public static long ReadEventSequenceNumber(ref Utf8JsonStreamReader streamReader) { streamReader.Read(); if (streamReader.TokenType != JsonTokenType.PropertyName || !streamReader.Match(EventSequenceNumberPropertyNameString)) { throw new JsonException($"Expected to find the {EventSequenceNumberPropertyNameString} property."); } streamReader.Read(); if (streamReader.TokenType != JsonTokenType.Number) { throw new JsonException($"Expected the {EventSequenceNumberPropertyNameString} property to be a number."); } return(streamReader.GetInt64()); }
private static void ProcessEventFeedStream <TEventHandler>(TEventHandler eventHandler, Stream content, int defaultBufferSize, CancellationToken cancellationToken) where TEventHandler : IJsonEventFeedHandler { var streamReader = new Utf8JsonStreamReader(content, defaultBufferSize); if (!FindDocumentsProperty(ref streamReader)) { throw new JsonException("Unable to find the Documents property in the output stream."); } if (!FindDocumentsArray(ref streamReader)) { throw new JsonException("The Documents propery was expected to be a JSON array."); } // We are now at the start of an array of commits JsonEventFeed.ProcessCommits(eventHandler, ref streamReader, cancellationToken); }
private static void ThrowUnexpectedTokenException(ref Utf8JsonStreamReader reader, JsonTokenType expected) { string key; if (expected.IsBool()) { key = Strings.ExpectedBoolAtOffset; } else if (expected.IsInt()) { key = Strings.ExpectedIntegerAtOffset; } else if (expected == JsonTokenType.String) { key = Strings.ExpectedStringAtOffset; } else { throw new WorkloadManifestFormatException(Strings.ExpectedTokenAtOffset, expected, reader.TokenStartIndex); } throw new WorkloadManifestFormatException(key, reader.TokenStartIndex); }
public async Task DeserializeFromStreamTest() { int rentCalls = 0; int returnCalls = 0; Utf8JsonStreamReader.RentBufferFunc = (size) => { rentCalls++; return(new byte[size]); }; Utf8JsonStreamReader.ReturnBufferAction = (buffer) => returnCalls++; using MemoryStream stream = new(Encoding.UTF8.GetBytes("{\"PropertyName\":1, \"PropertyName2\":\"value\"}")); TestClass instance = new(); await Utf8JsonStreamReader.DeserializeAsync(stream, instance, ReadMethod, bufferSize : 8192).ConfigureAwait(false); Assert.AreEqual(1, instance.PropertyName); Assert.AreEqual("value", instance.PropertyName2); Assert.AreEqual(1, rentCalls); Assert.AreEqual(1, returnCalls); }
private static string?ReadStringLocalized(ref Utf8JsonStreamReader reader, LocalizationCatalog?localizationCatalog, string id) { ConsumeToken(ref reader, JsonTokenType.String); return(localizationCatalog?.Localize(id) ?? reader.GetString()); }