private static IJsonNavigator CreateNavigatorFromContent(ReadOnlyMemory <byte> content, JsonSerializationFormatOptions jsonSerializationFormatOptions) { IJsonNavigator jsonNavigator; if (jsonSerializationFormatOptions != null) { if (jsonSerializationFormatOptions is JsonSerializationFormatOptions.CustomJsonSerializationFormatOptions customOptions) { jsonNavigator = customOptions.createNavigator(content); if (jsonNavigator == null) { throw new InvalidOperationException("The CosmosSerializationOptions did not return a JSON navigator."); } } else if (jsonSerializationFormatOptions is JsonSerializationFormatOptions.NativelySupportedJsonSerializationFormatOptions) { jsonNavigator = JsonNavigator.Create(content); } else { throw new ArgumentOutOfRangeException($"Unknown {nameof(JsonSerializationFormatOptions)} type: {jsonSerializationFormatOptions.GetType()}"); } } else { jsonNavigator = JsonNavigator.Create(content); } return(jsonNavigator); }
public static CosmosElement CreateFromBuffer(ReadOnlyMemory <byte> buffer) { IJsonNavigator jsonNavigator = JsonNavigator.Create(buffer); IJsonNavigatorNode jsonNavigatorNode = jsonNavigator.GetRootNode(); return(CosmosElement.Dispatch(jsonNavigator, jsonNavigatorNode)); }
private async Task RunQueryAsync(Container container) { QueryRequestOptions requestOptions = new QueryRequestOptions() { MaxConcurrency = this.MaxConcurrency, MaxItemCount = this.MaxItemCount, CosmosSerializationFormatOptions = new CosmosSerializationFormatOptions( contentSerializationFormat: this.contentSerialization, createCustomNavigator: (content) => JsonNavigator.Create(content), createCustomWriter: () => JsonWriter.Create(this.contentSerialization == "JsonText" ? JsonSerializationFormat.Text : JsonSerializationFormat.Binary)) }; if (this.useStronglyTypedIterator) { using (FeedIterator <States> iterator = container.GetItemQueryIterator <States>( queryText: this.query, requestOptions: requestOptions)) { await this.GetIteratorResponse(iterator); } } else { using (FeedIterator <dynamic> distinctQueryIterator = container.GetItemQueryIterator <dynamic>( queryText: this.query, requestOptions: requestOptions)) { await this.GetIteratorResponse(distinctQueryIterator); } } }
public static (QueryResponseCore queryResponse, IList <ToDoItem> items) Create( string itemIdPrefix, string continuationToken, string collectionRid, int itemCount = 50) { // Use -1 to represent a split response if (itemCount == QueryResponseMessageFactory.SPLIT) { return(CreateSplitResponse(collectionRid), new List <ToDoItem>().AsReadOnly()); } IList <ToDoItem> items = ToDoItem.CreateItems(itemCount, itemIdPrefix); MemoryStream memoryStream = (MemoryStream)cosmosSerializer.ToStream <IList <ToDoItem> >(items); long responseLengthBytes = memoryStream.Length; IJsonNavigator jsonNavigator = JsonNavigator.Create(memoryStream.ToArray()); IJsonNavigatorNode jsonNavigatorNode = jsonNavigator.GetRootNode(); CosmosArray cosmosArray = CosmosArray.Create(jsonNavigator, jsonNavigatorNode); QueryResponseCore message = QueryResponseCore.CreateSuccess( result: cosmosArray, continuationToken: continuationToken, disallowContinuationTokenMessage: null, activityId: Guid.NewGuid().ToString(), requestCharge: 42, queryMetricsText: null, queryMetrics: new Dictionary <string, QueryMetrics>(), requestStatistics: null, responseLengthBytes: responseLengthBytes); return(message, items); }
public static QueryResponseCore CreateQueryResponse( IList <ToDoItem> items, bool isOrderByQuery, string continuationToken, string collectionRid) { MemoryStream memoryStream; string json; if (isOrderByQuery) { memoryStream = SerializeForOrderByQuery(items); using (StreamReader sr = new StreamReader(SerializeForOrderByQuery(items))) { json = sr.ReadToEnd(); } } else { memoryStream = (MemoryStream)MockCosmosUtil.Serializer.ToStream <IList <ToDoItem> >(items); } long responseLengthBytes = memoryStream.Length; IJsonNavigator jsonNavigator = JsonNavigator.Create(memoryStream.ToArray()); IJsonNavigatorNode jsonNavigatorNode = jsonNavigator.GetRootNode(); CosmosArray cosmosArray = CosmosArray.Create(jsonNavigator, jsonNavigatorNode); CosmosDiagnosticsContext diagnosticsContext = new CosmosDiagnosticsContextCore(); diagnosticsContext.AddDiagnosticsInternal(new PointOperationStatistics( activityId: Guid.NewGuid().ToString(), statusCode: HttpStatusCode.OK, subStatusCode: SubStatusCodes.Unknown, responseTimeUtc: DateTime.UtcNow, requestCharge: 4, errorMessage: null, method: HttpMethod.Post, requestUri: "http://localhost.com", requestSessionToken: null, responseSessionToken: null)); IReadOnlyCollection <QueryPageDiagnostics> diagnostics = new List <QueryPageDiagnostics>() { new QueryPageDiagnostics( Guid.NewGuid(), "0", "SomeQueryMetricText", "SomeIndexUtilText", diagnosticsContext) }; QueryResponseCore message = QueryResponseCore.CreateSuccess( result: cosmosArray, requestCharge: 4, activityId: Guid.NewGuid().ToString(), responseLengthBytes: responseLengthBytes, disallowContinuationTokenMessage: null, continuationToken: continuationToken); return(message); }
public static CosmosElement Create(byte[] buffer) { IJsonNavigator jsonNavigator = JsonNavigator.Create(buffer); IJsonNavigatorNode jsonNavigatorNode = jsonNavigator.GetRootNode(); return(CosmosElement.Dispatch(jsonNavigator, jsonNavigatorNode)); }
public void LargeArrayBinaryJsonTest() { StringBuilder builder = new StringBuilder((1 << 24) + 50); builder.Append('['); for (int x = 1 << 24; x < (1 << 24) + 3355450; ++x) { builder.Append(x); builder.Append(','); } builder.Append("\"string_one\""); builder.Append(','); builder.Append("\"string_two\""); builder.Append(','); builder.Append("\"string_two\""); builder.Append(']'); string json = builder.ToString(); byte[] binaryJson = JsonTestUtils.ConvertTextToBinary(json); IJsonNavigator navigator = JsonNavigator.Create(binaryJson); int count = navigator.GetArrayItemCount(navigator.GetRootNode()); IJsonNavigatorNode node = navigator.GetArrayItemAt(navigator.GetRootNode(), count - 1); string stringValue = navigator.GetStringValue(node); Assert.AreEqual("string_two", stringValue); }
public static (QueryResponse queryResponse, IList <ToDoItem> items) Create( string itemIdPrefix, string continuationToken, string collectionRid, int itemCount = 50) { // Use -1 to represent a split response if (itemCount == QueryResponseMessageFactory.SPLIT) { return(CreateSplitResponse(collectionRid), new List <ToDoItem>().AsReadOnly()); } IList <ToDoItem> items = ToDoItem.CreateItems(itemCount, itemIdPrefix); MemoryStream memoryStream = (MemoryStream)cosmosSerializer.ToStream <IList <ToDoItem> >(items); long responseLengthBytes = memoryStream.Length; IJsonNavigator jsonNavigator = JsonNavigator.Create(memoryStream.ToArray()); IJsonNavigatorNode jsonNavigatorNode = jsonNavigator.GetRootNode(); CosmosArray cosmosArray = CosmosArray.Create(jsonNavigator, jsonNavigatorNode); Headers headers = new Headers(); headers.ContinuationToken = continuationToken; headers.ActivityId = Guid.NewGuid().ToString(); QueryResponse message = QueryResponse.CreateSuccess( result: cosmosArray, count: itemCount, responseHeaders: CosmosQueryResponseMessageHeaders.ConvertToQueryHeaders(headers, ResourceType.Document, collectionRid), responseLengthBytes: responseLengthBytes); return(message, items); }
public static TryCatch <TCosmosElement> CreateFromBuffer <TCosmosElement>(ReadOnlyMemory <byte> buffer) where TCosmosElement : CosmosElement { if (buffer.IsEmpty) { TryCatch <TCosmosElement> .FromException( new ArgumentException($"{nameof(buffer)} must not be empty.")); } CosmosElement unTypedCosmosElement; try { IJsonNavigator jsonNavigator = JsonNavigator.Create(buffer); IJsonNavigatorNode jsonNavigatorNode = jsonNavigator.GetRootNode(); unTypedCosmosElement = CosmosElement.Dispatch(jsonNavigator, jsonNavigatorNode); } catch (JsonParseException jpe) { return(TryCatch <TCosmosElement> .FromException(jpe)); } if (!(unTypedCosmosElement is TCosmosElement typedCosmosElement)) { return(TryCatch <TCosmosElement> .FromException( new CosmosElementWrongTypeException( message : $"buffer was incorrect cosmos element type: {unTypedCosmosElement.GetType()} when {typeof(TCosmosElement)} was requested."))); } return(TryCatch <TCosmosElement> .FromResult(typedCosmosElement)); }
private static void VerifyNavigator( string input, bool performExtraChecks = true) { CultureInfo defaultCultureInfo = System.Threading.Thread.CurrentThread.CurrentCulture; CultureInfo[] cultureInfoList = new CultureInfo[] { defaultCultureInfo, System.Globalization.CultureInfo.GetCultureInfo("fr-FR") }; try { foreach (CultureInfo cultureInfo in cultureInfoList) { System.Threading.Thread.CurrentThread.CurrentCulture = cultureInfo; IJsonReader jsonReader = JsonReader.Create(Encoding.UTF8.GetBytes(input)); JsonToken[] tokensFromReader = JsonNavigatorTests.GetTokensWithReader(jsonReader); // Test text IJsonNavigator textNavigator = JsonNavigator.Create(Encoding.UTF8.GetBytes(input)); IJsonNavigatorNode textRootNode = textNavigator.GetRootNode(); JsonToken[] tokensFromTextNavigator = JsonNavigatorTests.GetTokensFromNode(textRootNode, textNavigator, performExtraChecks); Assert.IsTrue(tokensFromTextNavigator.SequenceEqual(tokensFromReader)); // Test binary byte[] binaryInput = JsonTestUtils.ConvertTextToBinary(input); IJsonNavigator binaryNavigator = JsonNavigator.Create(binaryInput); IJsonNavigatorNode binaryRootNode = binaryNavigator.GetRootNode(); JsonToken[] tokensFromBinaryNavigator = JsonNavigatorTests.GetTokensFromNode(binaryRootNode, binaryNavigator, performExtraChecks); Assert.IsTrue(tokensFromBinaryNavigator.SequenceEqual(tokensFromReader)); // Test binary + user string encoding JsonStringDictionary jsonStringDictionary = new JsonStringDictionary(capacity: 4096); byte[] binaryWithUserStringEncodingInput = JsonTestUtils.ConvertTextToBinary(input, jsonStringDictionary); if (jsonStringDictionary.TryGetStringAtIndex(index: 0, value: out _)) { Assert.IsFalse(binaryWithUserStringEncodingInput.SequenceEqual(binaryInput), "Binary should be different with user string encoding"); } IJsonNavigator binaryNavigatorWithUserStringEncoding = JsonNavigator.Create(binaryInput, jsonStringDictionary); IJsonNavigatorNode binaryRootNodeWithUserStringEncoding = binaryNavigatorWithUserStringEncoding.GetRootNode(); JsonToken[] tokensFromBinaryNavigatorWithUserStringEncoding = JsonNavigatorTests.GetTokensFromNode(binaryRootNodeWithUserStringEncoding, binaryNavigatorWithUserStringEncoding, performExtraChecks); Assert.IsTrue(tokensFromBinaryNavigatorWithUserStringEncoding.SequenceEqual(tokensFromReader)); } } finally { System.Threading.Thread.CurrentThread.CurrentCulture = defaultCultureInfo; } }
public static (QueryResponseCore queryResponse, IList <ToDoItem> items) Create( string itemIdPrefix, string continuationToken, string collectionRid, int itemCount = 50) { // Use -1 to represent a split response if (itemCount == QueryResponseMessageFactory.SPLIT) { return(CreateSplitResponse(collectionRid), new List <ToDoItem>().AsReadOnly()); } IList <ToDoItem> items = ToDoItem.CreateItems(itemCount, itemIdPrefix); MemoryStream memoryStream = (MemoryStream)MockCosmosUtil.Serializer.ToStream <IList <ToDoItem> >(items); long responseLengthBytes = memoryStream.Length; IJsonNavigator jsonNavigator = JsonNavigator.Create(memoryStream.ToArray()); IJsonNavigatorNode jsonNavigatorNode = jsonNavigator.GetRootNode(); CosmosArray cosmosArray = CosmosArray.Create(jsonNavigator, jsonNavigatorNode); double requestCharge = 42; string activityId = Guid.NewGuid().ToString(); CosmosDiagnosticsContext diagnosticsContext = new CosmosDiagnosticsContextCore(); diagnosticsContext.AddDiagnosticsInternal(new PointOperationStatistics( activityId: Guid.NewGuid().ToString(), statusCode: HttpStatusCode.OK, subStatusCode: SubStatusCodes.Unknown, responseTimeUtc: DateTime.UtcNow, requestCharge: requestCharge, errorMessage: null, method: HttpMethod.Post, requestUri: "http://localhost.com", requestSessionToken: null, responseSessionToken: null)); IReadOnlyCollection <QueryPageDiagnostics> diagnostics = new List <QueryPageDiagnostics>() { new QueryPageDiagnostics( Guid.NewGuid(), "0", "SomeQueryMetricText", "SomeIndexUtilText", diagnosticsContext) }; QueryResponseCore message = QueryResponseCore.CreateSuccess( result: cosmosArray, continuationToken: continuationToken, disallowContinuationTokenMessage: null, activityId: activityId, requestCharge: requestCharge, responseLengthBytes: responseLengthBytes); return(message, items); }
private static void ExecuteNavigateMicroBenchmark( BenchmarkPayload payload, BenchmarkSerializationFormat benchmarkSerializationFormat) { IJsonNavigator jsonNavigator = benchmarkSerializationFormat switch { BenchmarkSerializationFormat.Text => JsonNavigator.Create(payload.Text), BenchmarkSerializationFormat.Binary => JsonNavigator.Create(payload.Binary), _ => throw new ArgumentOutOfRangeException($"Unknown {nameof(BenchmarkSerializationFormat)}: '{benchmarkSerializationFormat}'."), }; Utils.DrainNavigator(jsonNavigator); }
internal async Task <PartitionKey> GetPartitionKeyValueFromStreamAsync( Stream stream, CancellationToken cancellation = default(CancellationToken)) { if (!stream.CanSeek) { throw new ArgumentException("Stream is needs to be seekable", nameof(stream)); } try { stream.Position = 0; MemoryStream memoryStream = stream as MemoryStream; if (memoryStream == null) { memoryStream = new MemoryStream(); stream.CopyTo(memoryStream); } // TODO: Avoid copy IJsonNavigator jsonNavigator = JsonNavigator.Create(memoryStream.ToArray()); IJsonNavigatorNode jsonNavigatorNode = jsonNavigator.GetRootNode(); CosmosObject pathTraversal = CosmosObject.Create(jsonNavigator, jsonNavigatorNode); string[] tokens = await this.GetPartitionKeyPathTokensAsync(cancellation); for (int i = 0; i < tokens.Length - 1; i++) { pathTraversal = pathTraversal[tokens[i]] as CosmosObject; if (pathTraversal == null) { return(PartitionKey.None); } } CosmosElement partitionKeyValue = pathTraversal[tokens[tokens.Length - 1]]; if (partitionKeyValue == null) { return(PartitionKey.None); } return(this.CosmosElementToPartitionKeyObject(partitionKeyValue)); } finally { // MemoryStream casting leverage might change position stream.Position = 0; } }
public override async Task <PartitionKey> GetPartitionKeyValueFromStreamAsync( Stream stream, CancellationToken cancellation = default(CancellationToken)) { if (!stream.CanSeek) { throw new ArgumentException("Stream needs to be seekable", nameof(stream)); } try { stream.Position = 0; if (!(stream is MemoryStream memoryStream)) { memoryStream = new MemoryStream(); stream.CopyTo(memoryStream); } // TODO: Avoid copy IJsonNavigator jsonNavigator = JsonNavigator.Create(memoryStream.ToArray()); IJsonNavigatorNode jsonNavigatorNode = jsonNavigator.GetRootNode(); CosmosObject pathTraversal = CosmosObject.Create(jsonNavigator, jsonNavigatorNode); IReadOnlyList <IReadOnlyList <string> > tokenslist = await this.GetPartitionKeyPathTokensAsync(cancellation); List <CosmosElement> cosmosElementList = new List <CosmosElement>(tokenslist.Count); foreach (IReadOnlyList <string> tokenList in tokenslist) { CosmosElement element; if (ContainerCore.TryParseTokenListForElement(pathTraversal, tokenList, out element)) { cosmosElementList.Add(element); } else { cosmosElementList.Add(null); } } return(ContainerCore.CosmosElementToPartitionKeyObject(cosmosElementList)); } finally { // MemoryStream casting leverage might change position stream.Position = 0; } }
public async Task ItemQueryStreamSerializationSetting() { IList <ToDoActivity> deleteList = await this.CreateRandomItems(101, randomPartitionKey : true); CosmosSqlQueryDefinition sql = new CosmosSqlQueryDefinition("SELECT * FROM toDoActivity t ORDER BY t.taskNum"); CosmosSerializationOptions options = new CosmosSerializationOptions( ContentSerializationFormat.CosmosBinary.ToString(), (content) => JsonNavigator.Create(content), () => JsonWriter.Create(JsonSerializationFormat.Binary)); CosmosQueryRequestOptions requestOptions = new CosmosQueryRequestOptions() { CosmosSerializationOptions = options }; List <ToDoActivity> resultList = new List <ToDoActivity>(); double totalRequstCharge = 0; CosmosResultSetIterator setIterator = this.Container.Items.CreateItemQueryAsStream(sql, maxConcurrency: 5, maxItemCount: 5, requestOptions: requestOptions); while (setIterator.HasMoreResults) { using (CosmosQueryResponse iter = await setIterator.FetchNextSetAsync()) { Assert.IsTrue(iter.IsSuccess); Assert.IsNull(iter.ErrorMessage); Assert.IsTrue(iter.Count <= 5); totalRequstCharge += iter.RequestCharge; IJsonReader reader = JsonReader.Create(iter.Content); IJsonWriter textWriter = JsonWriter.Create(JsonSerializationFormat.Text); textWriter.WriteAll(reader); string json = Encoding.UTF8.GetString(textWriter.GetResult()); Assert.IsNotNull(json); ToDoActivity[] responseActivities = JsonConvert.DeserializeObject <ToDoActivity[]>(json); resultList.AddRange(responseActivities); } } Assert.AreEqual(deleteList.Count, resultList.Count); Assert.IsTrue(totalRequstCharge > 0); List <ToDoActivity> verifiedOrderBy = deleteList.OrderBy(x => x.taskNum).ToList(); for (int i = 0; i < verifiedOrderBy.Count(); i++) { Assert.AreEqual(verifiedOrderBy[i].taskNum, resultList[i].taskNum); Assert.AreEqual(verifiedOrderBy[i].id, resultList[i].id); } }
public async Task DecryptQueryBinaryResponse() { TestDoc testDoc = await EncryptionTests.CreateItemAsync(EncryptionTests.containerCore, EncryptionTests.dekId, TestDoc.PathsToEncrypt); CosmosSerializationFormatOptions options = new CosmosSerializationFormatOptions( Documents.ContentSerializationFormat.CosmosBinary.ToString(), (content) => JsonNavigator.Create(content), () => JsonWriter.Create(JsonSerializationFormat.Binary)); QueryRequestOptions requestOptions = new QueryRequestOptions() { CosmosSerializationFormatOptions = options }; TestDoc expectedDoc = new TestDoc(testDoc); string query = "SELECT * FROM c"; FeedIterator feedIterator = EncryptionTests.containerCore.GetItemQueryStreamIterator( query, requestOptions: requestOptions); while (feedIterator.HasMoreResults) { ResponseMessage response = await feedIterator.ReadNextAsync(); Assert.IsTrue(response.IsSuccessStatusCode); Assert.IsNull(response.ErrorMessage); // Copy the stream and check that the first byte is the correct value MemoryStream memoryStream = new MemoryStream(); response.Content.CopyTo(memoryStream); byte[] content = memoryStream.ToArray(); response.Content.Position = 0; // Examine the first buffer byte to determine the serialization format byte firstByte = content[0]; Assert.AreEqual(128, firstByte); Assert.AreEqual(JsonSerializationFormat.Binary, (JsonSerializationFormat)firstByte); IJsonReader reader = JsonReader.Create(content); IJsonWriter textWriter = JsonWriter.Create(JsonSerializationFormat.Text); textWriter.WriteAll(reader); string json = Encoding.UTF8.GetString(textWriter.GetResult().ToArray()); Assert.IsNotNull(json); Assert.IsTrue(json.Contains(testDoc.Sensitive)); } }
public async Task TestContentSerializationOptions() { { // Native format IAsyncEnumerable <TryCatch <ChangeFeedPage> > asyncEnumerable = this.Container.GetChangeFeedAsyncEnumerable( ChangeFeedCrossFeedRangeState.CreateFromBeginning(), ChangeFeedMode.Incremental, new ChangeFeedRequestOptions() { JsonSerializationFormatOptions = JsonSerializationFormatOptions.Create(JsonSerializationFormat.Binary) }); await foreach (TryCatch <ChangeFeedPage> monadicPage in asyncEnumerable) { monadicPage.ThrowIfFailed(); if (monadicPage.Result.NotModified) { break; } } } { // Custom format IAsyncEnumerable <TryCatch <ChangeFeedPage> > asyncEnumerable = this.Container.GetChangeFeedAsyncEnumerable( ChangeFeedCrossFeedRangeState.CreateFromBeginning(), ChangeFeedMode.Incremental, new ChangeFeedRequestOptions() { JsonSerializationFormatOptions = JsonSerializationFormatOptions.Create( JsonSerializationFormat.Binary, (content) => JsonNavigator.Create(content)) }); await foreach (TryCatch <ChangeFeedPage> monadicPage in asyncEnumerable) { monadicPage.ThrowIfFailed(); if (monadicPage.Result.NotModified) { break; } } } }
private static void SetSerializationFormat( QueryRequestOptions queryRequestOptions, JsonSerializationFormat jsonSerializationFormat) { string contentSerializationFormat = jsonSerializationFormat switch { JsonSerializationFormat.Text => "JsonText", JsonSerializationFormat.Binary => "CosmosBinary", JsonSerializationFormat.HybridRow => "HybridRow", _ => throw new Exception(), }; CosmosSerializationFormatOptions formatOptions = new CosmosSerializationFormatOptions( contentSerializationFormat, (content) => JsonNavigator.Create(content), () => JsonWriter.Create(JsonSerializationFormat.Text)); queryRequestOptions.CosmosSerializationFormatOptions = formatOptions; }
private async Task TestCurratedDocs(CosmosClient cosmosClient, Container container, List <JToken> items) { HashSet <JToken> inputItems = new HashSet <JToken>(items, JsonTokenEqualityComparer.Value); async Task AssertQueryDrainsCorrectlyAsync(FeedIterator <JToken> feedIterator) { while (feedIterator.HasMoreResults) { FeedResponse <JToken> feedResponse = await feedIterator.ReadNextAsync(); foreach (JToken item in feedResponse) { Assert.IsTrue(inputItems.Contains(item), "Documents differ from input documents"); } } } FeedIterator <JToken> textFeedIterator = container.GetItemQueryIterator <JToken>( queryDefinition: new QueryDefinition("SELECT * FROM c ORDER BY c._ts"), requestOptions: new QueryRequestOptions() { CosmosSerializationFormatOptions = new CosmosSerializationFormatOptions( "JsonText", (content) => JsonNavigator.Create(content), () => Cosmos.Json.JsonWriter.Create(JsonSerializationFormat.Text)), }); await AssertQueryDrainsCorrectlyAsync(textFeedIterator); FeedIterator <JToken> binaryFeedIterator = container.GetItemQueryIterator <JToken>( queryDefinition: new QueryDefinition("SELECT * FROM c ORDER BY c._ts"), requestOptions: new QueryRequestOptions() { CosmosSerializationFormatOptions = new CosmosSerializationFormatOptions( "CosmosBinary", (content) => JsonNavigator.Create(content), () => Cosmos.Json.JsonWriter.Create(JsonSerializationFormat.Text)), }); await AssertQueryDrainsCorrectlyAsync(binaryFeedIterator); }
public static QueryResponse CreateQueryResponse( IList <ToDoItem> items, bool isOrderByQuery, string continuationToken, string collectionRid) { MemoryStream memoryStream; string json; if (isOrderByQuery) { memoryStream = SerializeForOrderByQuery(items); using (StreamReader sr = new StreamReader(SerializeForOrderByQuery(items))) { json = sr.ReadToEnd(); } } else { memoryStream = (MemoryStream)cosmosSerializer.ToStream <IList <ToDoItem> >(items); } long responseLengthBytes = memoryStream.Length; IJsonNavigator jsonNavigator = JsonNavigator.Create(memoryStream.ToArray()); IJsonNavigatorNode jsonNavigatorNode = jsonNavigator.GetRootNode(); CosmosArray cosmosArray = CosmosArray.Create(jsonNavigator, jsonNavigatorNode); Headers headers = new Headers(); headers.ContinuationToken = continuationToken; headers.ActivityId = Guid.NewGuid().ToString(); QueryResponse message = QueryResponse.CreateSuccess( result: cosmosArray, count: items.Count, responseHeaders: CosmosQueryResponseMessageHeaders.ConvertToQueryHeaders(headers, ResourceType.Document, collectionRid), responseLengthBytes: responseLengthBytes); return(message); }
public static QueryResponseCore CreateQueryResponse( IList <ToDoItem> items, bool isOrderByQuery, string continuationToken, string collectionRid) { MemoryStream memoryStream; string json; if (isOrderByQuery) { memoryStream = SerializeForOrderByQuery(items); using (StreamReader sr = new StreamReader(SerializeForOrderByQuery(items))) { json = sr.ReadToEnd(); } } else { memoryStream = (MemoryStream)cosmosSerializer.ToStream <IList <ToDoItem> >(items); } long responseLengthBytes = memoryStream.Length; IJsonNavigator jsonNavigator = JsonNavigator.Create(memoryStream.ToArray()); IJsonNavigatorNode jsonNavigatorNode = jsonNavigator.GetRootNode(); CosmosArray cosmosArray = CosmosArray.Create(jsonNavigator, jsonNavigatorNode); QueryResponseCore message = QueryResponseCore.CreateSuccess( result: cosmosArray, requestCharge: 4, activityId: Guid.NewGuid().ToString(), queryMetricsText: null, queryMetrics: null, requestStatistics: null, responseLengthBytes: responseLengthBytes, disallowContinuationTokenMessage: null, continuationToken: continuationToken); return(message); }
private static void ExecuteNavigatorBenchmark( CurratedDocsPayload payload, SerializationFormat sourceFormat, SerializationFormat destinationFormat) { IJsonNavigator navigator = sourceFormat switch { SerializationFormat.Text => JsonNavigator.Create(payload.Text), SerializationFormat.Binary => JsonNavigator.Create(payload.Binary), _ => throw new ArgumentException($"Unexpected {nameof(sourceFormat)} of type: '{sourceFormat}'"), }; IJsonWriter writer = destinationFormat switch { SerializationFormat.Text => JsonWriter.Create(JsonSerializationFormat.Text), SerializationFormat.Binary => JsonWriter.Create(JsonSerializationFormat.Binary), _ => throw new ArgumentException($"Unexpected {nameof(destinationFormat)} of type: {destinationFormat}"), }; writer.WriteJsonNode(navigator, navigator.GetRootNode()); }
private void VerifyNavigator(string input, Exception expectedException, bool performExtraChecks = true) { IJsonReader jsonReader = JsonReader.Create(Encoding.UTF8.GetBytes(input)); JsonTokenInfo[] tokensFromReader = JsonNavigatorTests.GetTokensWithReader(jsonReader); // Test text IJsonNavigator textNavigator = JsonNavigator.Create(Encoding.UTF8.GetBytes(input)); IJsonNavigatorNode textRootNode = textNavigator.GetRootNode(); JsonTokenInfo[] tokensFromTextNavigator = JsonNavigatorTests.GetTokensFromNode(textRootNode, textNavigator, performExtraChecks); Assert.IsTrue(tokensFromTextNavigator.SequenceEqual(tokensFromReader)); // Test binary byte[] binaryInput = JsonTestUtils.ConvertTextToBinary(input); IJsonNavigator binaryNavigator = JsonNavigator.Create(binaryInput); IJsonNavigatorNode binaryRootNode = binaryNavigator.GetRootNode(); JsonTokenInfo[] tokensFromBinaryNavigator = JsonNavigatorTests.GetTokensFromNode(binaryRootNode, binaryNavigator, performExtraChecks); Assert.IsTrue(tokensFromBinaryNavigator.SequenceEqual(tokensFromReader)); // Test binary + user string encoding JsonStringDictionary jsonStringDictionary = new JsonStringDictionary(capacity: 4096); byte[] binaryWithUserStringEncodingInput = JsonTestUtils.ConvertTextToBinary(input, jsonStringDictionary); if (jsonStringDictionary.TryGetStringAtIndex(index: 0, value: out string temp)) { Assert.IsFalse(binaryWithUserStringEncodingInput.SequenceEqual(binaryInput), "Binary should be different with user string encoding"); } IJsonNavigator binaryNavigatorWithUserStringEncoding = JsonNavigator.Create(binaryInput, jsonStringDictionary); IJsonNavigatorNode binaryRootNodeWithUserStringEncoding = binaryNavigatorWithUserStringEncoding.GetRootNode(); JsonTokenInfo[] tokensFromBinaryNavigatorWithUserStringEncoding = JsonNavigatorTests.GetTokensFromNode(binaryRootNode, binaryNavigator, performExtraChecks); Assert.IsTrue(tokensFromBinaryNavigatorWithUserStringEncoding.SequenceEqual(tokensFromReader)); }
private void VerifyNavigator(string input, Exception expectedException, bool performExtraChecks = true) { IJsonReader jsonReader = JsonReader.Create(Encoding.UTF8.GetBytes(input)); JsonTokenInfo[] tokensFromReader = JsonNavigatorTests.GetTokensWithReader(jsonReader); // Test text IJsonNavigator textNavigator = JsonNavigator.Create(Encoding.UTF8.GetBytes(input)); IJsonNavigatorNode textRootNode = textNavigator.GetRootNode(); JsonTokenInfo[] tokensFromTextNavigator = JsonNavigatorTests.GetTokensFromNode(textRootNode, textNavigator, performExtraChecks); Assert.IsTrue(tokensFromTextNavigator.SequenceEqual(tokensFromReader)); // Test binary byte[] binaryInput = JsonTestUtils.ConvertTextToBinary(input); IJsonNavigator binaryNavigator = JsonNavigator.Create(binaryInput); IJsonNavigatorNode binaryRootNode = binaryNavigator.GetRootNode(); JsonTokenInfo[] tokensFromBinaryNavigator = JsonNavigatorTests.GetTokensFromNode(binaryRootNode, binaryNavigator, performExtraChecks); Assert.IsTrue(tokensFromBinaryNavigator.SequenceEqual(tokensFromReader)); }
/// <summary> /// Converts a list of CosmosElements into a memory stream. /// </summary> /// <param name="stream">The memory stream response for the query REST response Azure Cosmos</param> /// <param name="resourceType">The resource type</param> /// <param name="cosmosSerializationOptions">The custom serialization options. This allows custom serialization types like BSON, JSON, or other formats</param> /// <returns>An array of CosmosElements parsed from the response body.</returns> public static CosmosArray ParseElementsFromRestStream( Stream stream, ResourceType resourceType, CosmosSerializationFormatOptions cosmosSerializationOptions) { if (!(stream is MemoryStream memoryStream)) { memoryStream = new MemoryStream(); stream.CopyTo(memoryStream); } if (!memoryStream.CanRead) { throw new InvalidDataException("Stream can not be read"); } // Parse out the document from the REST response like this: // { // "_rid": "qHVdAImeKAQ=", // "Documents": [{ // "id": "03230", // "_rid": "qHVdAImeKAQBAAAAAAAAAA==", // "_self": "dbs\/qHVdAA==\/colls\/qHVdAImeKAQ=\/docs\/qHVdAImeKAQBAAAAAAAAAA==\/", // "_etag": "\"410000b0-0000-0000-0000-597916b00000\"", // "_attachments": "attachments\/", // "_ts": 1501107886 // }], // "_count": 1 // } // You want to create a CosmosElement for each document in "Documents". ReadOnlyMemory <byte> content; if (memoryStream.TryGetBuffer(out ArraySegment <byte> buffer)) { content = buffer; } else { content = memoryStream.ToArray(); } IJsonNavigator jsonNavigator; if (cosmosSerializationOptions != null) { // Use the users custom navigator jsonNavigator = cosmosSerializationOptions.CreateCustomNavigatorCallback(content); if (jsonNavigator == null) { throw new InvalidOperationException("The CosmosSerializationOptions did not return a JSON navigator."); } } else { jsonNavigator = JsonNavigator.Create(content); } string resourceName = resourceType switch { ResourceType.Collection => "DocumentCollections", _ => resourceType.ToResourceTypeString() + "s", }; if (!jsonNavigator.TryGetObjectProperty( jsonNavigator.GetRootNode(), resourceName, out ObjectProperty objectProperty)) { throw new InvalidOperationException($"Response Body Contract was violated. QueryResponse did not have property: {resourceName}"); } if (!(CosmosElement.Dispatch( jsonNavigator, objectProperty.ValueNode) is CosmosArray cosmosArray)) { throw new InvalidOperationException($"QueryResponse did not have an array of : {resourceName}"); } return(cosmosArray); }
private DocumentFeedResponse <CosmosElement> GetFeedResponse( DocumentServiceRequest documentServiceRequest, DocumentServiceResponse documentServiceResponse) { // Execute the callback an each element of the page // For example just could get a response like this // { // "_rid": "qHVdAImeKAQ=", // "Documents": [{ // "id": "03230", // "_rid": "qHVdAImeKAQBAAAAAAAAAA==", // "_self": "dbs\/qHVdAA==\/colls\/qHVdAImeKAQ=\/docs\/qHVdAImeKAQBAAAAAAAAAA==\/", // "_etag": "\"410000b0-0000-0000-0000-597916b00000\"", // "_attachments": "attachments\/", // "_ts": 1501107886 // }], // "_count": 1 // } // And you should execute the callback on each document in "Documents". MemoryStream memoryStream = new MemoryStream(); documentServiceResponse.ResponseBody.CopyTo(memoryStream); long responseLengthBytes = memoryStream.Length; ReadOnlyMemory <byte> content; if (memoryStream.TryGetBuffer(out ArraySegment <byte> buffer)) { content = buffer; } else { content = memoryStream.ToArray(); } IJsonNavigator jsonNavigator = null; // Use the users custom navigator first. If it returns null back try the // internal navigator. if (this.feedOptions.CosmosSerializationFormatOptions != null) { jsonNavigator = this.feedOptions.CosmosSerializationFormatOptions.CreateCustomNavigatorCallback(content); if (jsonNavigator == null) { throw new InvalidOperationException("The CosmosSerializationOptions did not return a JSON navigator."); } } else { jsonNavigator = JsonNavigator.Create(content); } string resourceName = this.GetRootNodeName(documentServiceRequest.ResourceType); if (!jsonNavigator.TryGetObjectProperty( jsonNavigator.GetRootNode(), resourceName, out ObjectProperty objectProperty)) { throw new InvalidOperationException($"Response Body Contract was violated. QueryResponse did not have property: {resourceName}"); } IJsonNavigatorNode cosmosElements = objectProperty.ValueNode; if (!(CosmosElement.Dispatch( jsonNavigator, cosmosElements) is CosmosArray cosmosArray)) { throw new InvalidOperationException($"QueryResponse did not have an array of : {resourceName}"); } int itemCount = cosmosArray.Count; return(new DocumentFeedResponse <CosmosElement>( cosmosArray, itemCount, documentServiceResponse.Headers, documentServiceResponse.RequestStats, responseLengthBytes)); }
public static async Task RewriteStreamAsTextAsync(ResponseMessage responseMessage, QueryRequestOptions requestOptions) { // Rewrite the payload to be in the specified format. // If it's already in the correct format, then the following will be a memcpy. MemoryStream memoryStream; if (responseMessage.Content is MemoryStream responseContentAsMemoryStream) { memoryStream = responseContentAsMemoryStream; } else { memoryStream = new MemoryStream(); await responseMessage.Content.CopyToAsync(memoryStream); } ReadOnlyMemory <byte> buffer; if (memoryStream.TryGetBuffer(out ArraySegment <byte> segment)) { buffer = segment.Array.AsMemory().Slice(start: segment.Offset, length: segment.Count); } else { buffer = memoryStream.ToArray(); } IJsonNavigator jsonNavigator = JsonNavigator.Create(buffer); if (jsonNavigator.SerializationFormat == JsonSerializationFormat.Text) { // Exit to avoid the memory allocation. return; } IJsonWriter jsonWriter; if (requestOptions?.CosmosSerializationFormatOptions != null) { jsonWriter = requestOptions.CosmosSerializationFormatOptions.CreateCustomWriterCallback(); } else { jsonWriter = JsonWriter.Create(JsonSerializationFormat.Text); } jsonNavigator.WriteTo(jsonNavigator.GetRootNode(), jsonWriter); ReadOnlyMemory <byte> result = jsonWriter.GetResult(); MemoryStream rewrittenMemoryStream; if (MemoryMarshal.TryGetArray(result, out ArraySegment <byte> rewrittenSegment)) { rewrittenMemoryStream = new MemoryStream(rewrittenSegment.Array, index: rewrittenSegment.Offset, count: rewrittenSegment.Count, writable: false, publiclyVisible: true); } else { byte[] toArray = result.ToArray(); rewrittenMemoryStream = new MemoryStream(toArray, index: 0, count: toArray.Length, writable: false, publiclyVisible: true); } responseMessage.Content = rewrittenMemoryStream; }
private static void PerformRoundTrip( SerializationFormat sourceFormat, SerializationFormat destinationFormat, string json, bool writeAsRootNode) { IJsonReader reader = sourceFormat switch { SerializationFormat.Text => JsonReader.Create(Encoding.UTF8.GetBytes(json)), SerializationFormat.Binary => JsonReader.Create(JsonTestUtils.ConvertTextToBinary(json)), SerializationFormat.NewtonsoftText => NewtonsoftToCosmosDBReader.CreateFromString(json), _ => throw new ArgumentException($"Unexpected {nameof(sourceFormat)} of type: {sourceFormat}"), }; IJsonNavigator navigator = sourceFormat switch { SerializationFormat.Text => JsonNavigator.Create(Encoding.UTF8.GetBytes(json)), SerializationFormat.Binary => JsonNavigator.Create(JsonTestUtils.ConvertTextToBinary(json)), SerializationFormat.NewtonsoftText => new JsonNewtonsoftNavigator(json), _ => throw new ArgumentException($"Unexpected {nameof(sourceFormat)} of type: {sourceFormat}"), }; foreach (object source in new object[] { reader, navigator }) { IJsonWriter writer = destinationFormat switch { SerializationFormat.Text => JsonWriter.Create(JsonSerializationFormat.Text), SerializationFormat.Binary => JsonWriter.Create(JsonSerializationFormat.Binary), SerializationFormat.NewtonsoftText => NewtonsoftToCosmosDBWriter.CreateTextWriter(), _ => throw new ArgumentException($"Unexpected {nameof(destinationFormat)} of type: {destinationFormat}"), }; switch (source) { case IJsonReader sourceReader: sourceReader.WriteAll(writer); break; case IJsonNavigator sourceNavigator: if (writeAsRootNode) { sourceNavigator.WriteNode(sourceNavigator.GetRootNode(), writer); } else { IJsonNavigatorNode rootNode = sourceNavigator.GetRootNode(); JsonNodeType jsonNodeType = sourceNavigator.GetNodeType(rootNode); switch (jsonNodeType) { case JsonNodeType.Array: writer.WriteArrayStart(); foreach (IJsonNavigatorNode arrayItem in sourceNavigator.GetArrayItems(rootNode)) { sourceNavigator.WriteNode(arrayItem, writer); } writer.WriteArrayEnd(); break; case JsonNodeType.Object: writer.WriteObjectStart(); foreach (ObjectProperty objectProperty in sourceNavigator.GetObjectProperties(rootNode)) { sourceNavigator.WriteNode(objectProperty.NameNode, writer); sourceNavigator.WriteNode(objectProperty.ValueNode, writer); } writer.WriteObjectEnd(); break; default: sourceNavigator.WriteNode(sourceNavigator.GetRootNode(), writer); break; } } break; default: Assert.Fail("Failed to downcast source type."); break; } string result = writer.SerializationFormat switch { JsonSerializationFormat.Text => Utf8String.UnsafeFromUtf8BytesNoValidation(writer.GetResult()).ToString(), JsonSerializationFormat.Binary => JsonTestUtils.ConvertBinaryToText(writer.GetResult()), _ => throw new ArgumentException(), }; string normalizedResult = JsonRoundTripsTests.NewtonsoftFormat(result); string normalizedJson = JsonRoundTripsTests.NewtonsoftFormat(json); Assert.AreEqual(normalizedJson, normalizedResult); } }
/// <summary> /// Converts a list of CosmosElements into a memory stream. /// </summary> /// <param name="memoryStream">The memory stream response from Azure Cosmos</param> /// <param name="resourceType">The resource type</param> /// <param name="cosmosSerializationOptions">The custom serialization options. This allows custom serialization types like BSON, JSON, or other formats</param> /// <returns>Returns a memory stream of cosmos elements. By default the memory stream will contain JSON.</returns> internal static CosmosArray ToCosmosElements( MemoryStream memoryStream, ResourceType resourceType, CosmosSerializationFormatOptions cosmosSerializationOptions = null) { if (!memoryStream.CanRead) { throw new InvalidDataException("Stream can not be read"); } // Execute the callback an each element of the page // For example just could get a response like this // { // "_rid": "qHVdAImeKAQ=", // "Documents": [{ // "id": "03230", // "_rid": "qHVdAImeKAQBAAAAAAAAAA==", // "_self": "dbs\/qHVdAA==\/colls\/qHVdAImeKAQ=\/docs\/qHVdAImeKAQBAAAAAAAAAA==\/", // "_etag": "\"410000b0-0000-0000-0000-597916b00000\"", // "_attachments": "attachments\/", // "_ts": 1501107886 // }], // "_count": 1 // } // And you should execute the callback on each document in "Documents". long responseLengthBytes = memoryStream.Length; ReadOnlyMemory <byte> content; if (memoryStream.TryGetBuffer(out ArraySegment <byte> buffer)) { content = buffer; } else { content = memoryStream.ToArray(); } IJsonNavigator jsonNavigator; // Use the users custom navigator if (cosmosSerializationOptions != null) { jsonNavigator = cosmosSerializationOptions.CreateCustomNavigatorCallback(content); if (jsonNavigator == null) { throw new InvalidOperationException("The CosmosSerializationOptions did not return a JSON navigator."); } } else { jsonNavigator = JsonNavigator.Create(content); } string resourceName = CosmosElementSerializer.GetRootNodeName(resourceType); CosmosArray documents; if ((jsonNavigator.SerializationFormat == JsonSerializationFormat.Binary) && jsonNavigator.TryGetObjectProperty( jsonNavigator.GetRootNode(), "stringDictionary", out ObjectProperty stringDictionaryProperty)) { // Payload is string dictionary encode so we have to decode using the string dictionary. IJsonNavigatorNode stringDictionaryNode = stringDictionaryProperty.ValueNode; JsonStringDictionary jsonStringDictionary = JsonStringDictionary.CreateFromStringArray( jsonNavigator .GetArrayItems(stringDictionaryNode) .Select(item => jsonNavigator.GetStringValue(item)) .ToList()); if (!jsonNavigator.TryGetObjectProperty( jsonNavigator.GetRootNode(), resourceName, out ObjectProperty resourceProperty)) { throw new InvalidOperationException($"Response Body Contract was violated. QueryResponse did not have property: {resourceName}"); } IJsonNavigatorNode resources = resourceProperty.ValueNode; if (!jsonNavigator.TryGetBufferedBinaryValue(resources, out ReadOnlyMemory <byte> resourceBinary)) { resourceBinary = jsonNavigator.GetBinaryValue(resources); } IJsonNavigator navigatorWithStringDictionary = JsonNavigator.Create(resourceBinary, jsonStringDictionary); if (!(CosmosElement.Dispatch( navigatorWithStringDictionary, navigatorWithStringDictionary.GetRootNode()) is CosmosArray cosmosArray)) { throw new InvalidOperationException($"QueryResponse did not have an array of : {resourceName}"); } documents = cosmosArray; } else { // Payload is not string dictionary encoded so we can just do for the documents as is. if (!jsonNavigator.TryGetObjectProperty( jsonNavigator.GetRootNode(), resourceName, out ObjectProperty objectProperty)) { throw new InvalidOperationException($"Response Body Contract was violated. QueryResponse did not have property: {resourceName}"); } if (!(CosmosElement.Dispatch( jsonNavigator, objectProperty.ValueNode) is CosmosArray cosmosArray)) { throw new InvalidOperationException($"QueryResponse did not have an array of : {resourceName}"); } documents = cosmosArray; } return(documents); }
private void MultiSerializationRoundTrip(string json) { // Normalize the json to get rid of any formatting issues json = this.NewtonsoftFormat(json); foreach (SerializationFormat sourceFormat in Enum.GetValues(typeof(SerializationFormat))) { foreach (SerializationFormat destinationFormat in Enum.GetValues(typeof(SerializationFormat))) { IJsonReader reader; switch (sourceFormat) { case SerializationFormat.Text: reader = JsonReader.Create(Encoding.UTF8.GetBytes(json)); break; case SerializationFormat.Binary: reader = JsonReader.Create(JsonTestUtils.ConvertTextToBinary(json)); break; case SerializationFormat.NewtonsoftText: reader = new JsonNewtonsoftNewtonsoftTextReader(json); break; default: throw new ArgumentException($"Unexpected {nameof(sourceFormat)} of type: {sourceFormat}"); } IJsonNavigator navigator; switch (sourceFormat) { case SerializationFormat.Text: navigator = JsonNavigator.Create(Encoding.UTF8.GetBytes(json)); break; case SerializationFormat.Binary: navigator = JsonNavigator.Create(JsonTestUtils.ConvertTextToBinary(json)); break; case SerializationFormat.NewtonsoftText: navigator = new JsonNewtonsoftNavigator(json); break; default: throw new ArgumentException($"Unexpected {nameof(sourceFormat)} of type: {sourceFormat}"); } object[] sources = new object[] { reader, navigator }; foreach (object source in sources) { IJsonWriter writer; switch (destinationFormat) { case SerializationFormat.Text: writer = JsonWriter.Create(JsonSerializationFormat.Text); break; case SerializationFormat.Binary: writer = JsonWriter.Create(JsonSerializationFormat.Binary); break; case SerializationFormat.NewtonsoftText: writer = new JsonNewtonsoftNewtonsoftTextWriter(); break; default: throw new ArgumentException($"Unexpected {nameof(destinationFormat)} of type: {destinationFormat}"); } Stopwatch stopwatch = Stopwatch.StartNew(); if (source is IJsonReader) { IJsonReader sourceReader = source as IJsonReader; writer.WriteAll(sourceReader); } else if (source is IJsonNavigator) { IJsonNavigator sourceNavigator = source as IJsonNavigator; writer.WriteJsonNode(sourceNavigator, sourceNavigator.GetRootNode()); } stopwatch.Stop(); string result; switch (writer.SerializationFormat) { case JsonSerializationFormat.Text: result = Encoding.UTF8.GetString(writer.GetResult()); break; case JsonSerializationFormat.Binary: result = JsonTestUtils.ConvertBinaryToText(writer.GetResult()); break; default: throw new ArgumentException(); } result = this.NewtonsoftFormat(result); Assert.AreEqual(json, result); string sourceType = (source is IJsonReader) ? "Reader" : "Navigator"; Console.WriteLine($"{sourceFormat} {sourceType} to {destinationFormat} Writer took {stopwatch.ElapsedMilliseconds}ms"); } } } }