/// <summary> /// Converts a list of CosmosElements into a memory stream. /// </summary> /// <param name="cosmosElements">The cosmos elements</param> /// <param name="cosmosSerializationOptions">The custom serialization options. This allows custom serialization types like BSON, JSON, or other formats</param> /// <returns>Returns a memory stream of cosmos elements. By default the memory stream will contain JSON.</returns> internal static MemoryStream ElementsToMemoryStream( IReadOnlyList <CosmosElement> cosmosElements, CosmosSerializationFormatOptions cosmosSerializationOptions = null) { IJsonWriter jsonWriter; if (cosmosSerializationOptions != null) { jsonWriter = cosmosSerializationOptions.CreateCustomWriterCallback(); } else { jsonWriter = JsonWriter.Create(JsonSerializationFormat.Text); } jsonWriter.WriteArrayStart(); foreach (CosmosElement element in cosmosElements) { element.WriteTo(jsonWriter); } jsonWriter.WriteArrayEnd(); return(GetMemoryStreamFromJsonWriter(jsonWriter)); }
/// <summary> /// Converts a list of CosmosElements into a memory stream. /// </summary> /// <param name="cosmosElement">The cosmos elements</param> /// <param name="cosmosSerializationOptions">The custom serialization options. This allows custom serialization types like BSON, JSON, or other formats</param> /// <returns>Returns a memory stream of cosmos elements. By default the memory stream will contain JSON.</returns> private static MemoryStream ElementToMemoryStream( CosmosElement cosmosElement, CosmosSerializationFormatOptions cosmosSerializationOptions = null) { IJsonWriter jsonWriter; if (cosmosSerializationOptions != null) { jsonWriter = cosmosSerializationOptions.CreateCustomWriterCallback(); } else { jsonWriter = JsonWriter.Create(JsonSerializationFormat.Text); } cosmosElement.WriteTo(jsonWriter); ReadOnlyMemory <byte> result = jsonWriter.GetResult(); if (!MemoryMarshal.TryGetArray(result, out ArraySegment <byte> resultAsArray)) { resultAsArray = new ArraySegment <byte>(result.ToArray()); } return(new MemoryStream(resultAsArray.Array, resultAsArray.Offset, resultAsArray.Count)); }
private QueryIterator( CosmosQueryContext cosmosQueryContext, CosmosQueryExecutionContext cosmosQueryExecutionContext, CosmosSerializationFormatOptions cosmosSerializationFormatOptions) { this.cosmosQueryContext = cosmosQueryContext ?? throw new ArgumentNullException(nameof(cosmosQueryContext)); this.cosmosQueryExecutionContext = cosmosQueryExecutionContext ?? throw new ArgumentNullException(nameof(cosmosQueryExecutionContext)); this.cosmosSerializationFormatOptions = cosmosSerializationFormatOptions; }
internal static T[] GetResourcesHelper <T>( IReadOnlyList <CosmosElement> cosmosElements, CosmosSerializerCore serializerCore, CosmosSerializationFormatOptions cosmosSerializationOptions = null) { using (MemoryStream memoryStream = ElementsToMemoryStream( cosmosElements, cosmosSerializationOptions)) { return(serializerCore.FromFeedStream <T>(memoryStream)); } }
private QueryIterator( CosmosQueryContextCore cosmosQueryContext, CosmosQueryExecutionContext cosmosQueryExecutionContext, CosmosSerializationFormatOptions cosmosSerializationFormatOptions, RequestOptions requestOptions, CosmosClientContext clientContext) { this.cosmosQueryContext = cosmosQueryContext ?? throw new ArgumentNullException(nameof(cosmosQueryContext)); this.cosmosQueryExecutionContext = cosmosQueryExecutionContext ?? throw new ArgumentNullException(nameof(cosmosQueryExecutionContext)); this.cosmosSerializationFormatOptions = cosmosSerializationFormatOptions; this.requestOptions = requestOptions; this.clientContext = clientContext ?? throw new ArgumentNullException(nameof(clientContext)); }
private QueryIterator( CosmosQueryContextCore cosmosQueryContext, IQueryPipelineStage cosmosQueryExecutionContext, CosmosSerializationFormatOptions cosmosSerializationFormatOptions, RequestOptions requestOptions, CosmosClientContext clientContext) { this.cosmosQueryContext = cosmosQueryContext ?? throw new ArgumentNullException(nameof(cosmosQueryContext)); this.queryPipelineStage = cosmosQueryExecutionContext ?? throw new ArgumentNullException(nameof(cosmosQueryExecutionContext)); this.cosmosSerializationFormatOptions = cosmosSerializationFormatOptions; this.requestOptions = requestOptions; this.clientContext = clientContext ?? throw new ArgumentNullException(nameof(clientContext)); this.hasMoreResults = true; }
public async Task DecryptQueryBinaryResponse() { TestDoc testDoc = await EncryptionTests.CreateItemAsync(EncryptionTests.containerCore, EncryptionTests.dekId, TestDoc.PathsToEncrypt); CosmosSerializationFormatOptions options = new CosmosSerializationFormatOptions( Documents.ContentSerializationFormat.CosmosBinary.ToString(), (content) => JsonNavigator.Create(content), () => JsonWriter.Create(JsonSerializationFormat.Binary)); QueryRequestOptions requestOptions = new QueryRequestOptions() { CosmosSerializationFormatOptions = options }; TestDoc expectedDoc = new TestDoc(testDoc); string query = "SELECT * FROM c"; FeedIterator feedIterator = EncryptionTests.containerCore.GetItemQueryStreamIterator( query, requestOptions: requestOptions); while (feedIterator.HasMoreResults) { ResponseMessage response = await feedIterator.ReadNextAsync(); Assert.IsTrue(response.IsSuccessStatusCode); Assert.IsNull(response.ErrorMessage); // Copy the stream and check that the first byte is the correct value MemoryStream memoryStream = new MemoryStream(); response.Content.CopyTo(memoryStream); byte[] content = memoryStream.ToArray(); response.Content.Position = 0; // Examine the first buffer byte to determine the serialization format byte firstByte = content[0]; Assert.AreEqual(128, firstByte); Assert.AreEqual(JsonSerializationFormat.Binary, (JsonSerializationFormat)firstByte); IJsonReader reader = JsonReader.Create(content); IJsonWriter textWriter = JsonWriter.Create(JsonSerializationFormat.Text); textWriter.WriteAll(reader); string json = Encoding.UTF8.GetString(textWriter.GetResult().ToArray()); Assert.IsNotNull(json); Assert.IsTrue(json.Contains(testDoc.Sensitive)); } }
/// <summary> /// Converts a list of CosmosElements into a memory stream. /// </summary> /// <param name="stream">The stream response from Azure Cosmos</param> /// <param name="resourceType">The resource type</param> /// <param name="cosmosSerializationOptions">The custom serialization options. This allows custom serialization types like BSON, JSON, or other formats</param> /// <returns>Returns a memory stream of cosmos elements. By default the memory stream will contain JSON.</returns> internal static CosmosArray ToCosmosElements( Stream stream, ResourceType resourceType, CosmosSerializationFormatOptions cosmosSerializationOptions = null) { MemoryStream memoryStream = stream as MemoryStream; if (memoryStream == null) { memoryStream = new MemoryStream(); stream.CopyTo(memoryStream); } return(CosmosElementSerializer.ToCosmosElements( memoryStream, resourceType, cosmosSerializationOptions)); }
private static void SetSerializationFormat( QueryRequestOptions queryRequestOptions, JsonSerializationFormat jsonSerializationFormat) { string contentSerializationFormat = jsonSerializationFormat switch { JsonSerializationFormat.Text => "JsonText", JsonSerializationFormat.Binary => "CosmosBinary", JsonSerializationFormat.HybridRow => "HybridRow", _ => throw new Exception(), }; CosmosSerializationFormatOptions formatOptions = new CosmosSerializationFormatOptions( contentSerializationFormat, (content) => JsonNavigator.Create(content), () => JsonWriter.Create(JsonSerializationFormat.Text)); queryRequestOptions.CosmosSerializationFormatOptions = formatOptions; }
internal QueryIterator( CosmosQueryClient client, SqlQuerySpec sqlQuerySpec, string continuationToken, QueryRequestOptions queryRequestOptions, Uri resourceLink, bool isContinuationExpected, bool allowNonValueAggregateQuery) { if (queryRequestOptions == null) { queryRequestOptions = new QueryRequestOptions(); } CosmosQueryContext context = new CosmosQueryContextCore( client: client, queryRequestOptions: queryRequestOptions, resourceTypeEnum: Documents.ResourceType.Document, operationType: Documents.OperationType.Query, resourceType: typeof(QueryResponseCore), resourceLink: resourceLink, isContinuationExpected: isContinuationExpected, allowNonValueAggregateQuery: allowNonValueAggregateQuery, correlatedActivityId: Guid.NewGuid()); CosmosQueryExecutionContextFactory.InputParameters inputParams = new CosmosQueryExecutionContextFactory.InputParameters() { SqlQuerySpec = sqlQuerySpec, InitialUserContinuationToken = continuationToken, MaxBufferedItemCount = queryRequestOptions.MaxBufferedItemCount, MaxConcurrency = queryRequestOptions.MaxConcurrency, MaxItemCount = queryRequestOptions.MaxItemCount, PartitionKey = queryRequestOptions.PartitionKey, Properties = queryRequestOptions.Properties }; this.cosmosSerializationFormatOptions = queryRequestOptions.CosmosSerializationFormatOptions; this.cosmosQueryExecutionContext = new CosmosQueryExecutionContextFactory( cosmosQueryContext: context, inputParameters: inputParams); }
/// <summary> /// Converts a list of CosmosElements into a list of objects. /// </summary> /// <param name="containerRid">Container Rid</param> /// <param name="cosmosElements">The cosmos elements</param> /// <param name="resourceType">The resource type</param> /// <param name="jsonSerializer">The JSON </param> /// <param name="cosmosSerializationOptions">The custom serialization options. This allows custom serialization types like BSON, JSON, or other formats</param> /// <returns>Returns a list of deserialized objects</returns> internal static IEnumerable <T> Deserialize <T>( string containerRid, IEnumerable <CosmosElement> cosmosElements, ResourceType resourceType, CosmosSerializer jsonSerializer, CosmosSerializationFormatOptions cosmosSerializationOptions = null) { if (!cosmosElements.Any()) { return(Enumerable.Empty <T>()); } Stream stream = CosmosElementSerializer.ToStream( containerRid, cosmosElements, resourceType, cosmosSerializationOptions); IEnumerable <T> typedResults = jsonSerializer.FromStream <CosmosFeedResponseUtil <T> >(stream).Data; return(typedResults); }
/// <summary> /// Converts a list of CosmosElements into a memory stream. /// </summary> /// <param name="memoryStream">The memory stream response from Azure Cosmos</param> /// <param name="resourceType">The resource type</param> /// <param name="cosmosSerializationOptions">The custom serialization options. This allows custom serialization types like BSON, JSON, or other formats</param> /// <returns>Returns a memory stream of cosmos elements. By default the memory stream will contain JSON.</returns> internal static CosmosArray ToCosmosElements( MemoryStream memoryStream, ResourceType resourceType, CosmosSerializationFormatOptions cosmosSerializationOptions = null) { if (!memoryStream.CanRead) { throw new InvalidDataException("Stream can not be read"); } // Execute the callback an each element of the page // For example just could get a response like this // { // "_rid": "qHVdAImeKAQ=", // "Documents": [{ // "id": "03230", // "_rid": "qHVdAImeKAQBAAAAAAAAAA==", // "_self": "dbs\/qHVdAA==\/colls\/qHVdAImeKAQ=\/docs\/qHVdAImeKAQBAAAAAAAAAA==\/", // "_etag": "\"410000b0-0000-0000-0000-597916b00000\"", // "_attachments": "attachments\/", // "_ts": 1501107886 // }], // "_count": 1 // } // And you should execute the callback on each document in "Documents". long responseLengthBytes = memoryStream.Length; ReadOnlyMemory <byte> content; if (memoryStream.TryGetBuffer(out ArraySegment <byte> buffer)) { content = buffer; } else { content = memoryStream.ToArray(); } IJsonNavigator jsonNavigator; // Use the users custom navigator if (cosmosSerializationOptions != null) { jsonNavigator = cosmosSerializationOptions.CreateCustomNavigatorCallback(content); if (jsonNavigator == null) { throw new InvalidOperationException("The CosmosSerializationOptions did not return a JSON navigator."); } } else { jsonNavigator = JsonNavigator.Create(content); } string resourceName = CosmosElementSerializer.GetRootNodeName(resourceType); CosmosArray documents; if ((jsonNavigator.SerializationFormat == JsonSerializationFormat.Binary) && jsonNavigator.TryGetObjectProperty( jsonNavigator.GetRootNode(), "stringDictionary", out ObjectProperty stringDictionaryProperty)) { // Payload is string dictionary encode so we have to decode using the string dictionary. IJsonNavigatorNode stringDictionaryNode = stringDictionaryProperty.ValueNode; JsonStringDictionary jsonStringDictionary = JsonStringDictionary.CreateFromStringArray( jsonNavigator .GetArrayItems(stringDictionaryNode) .Select(item => jsonNavigator.GetStringValue(item)) .ToList()); if (!jsonNavigator.TryGetObjectProperty( jsonNavigator.GetRootNode(), resourceName, out ObjectProperty resourceProperty)) { throw new InvalidOperationException($"Response Body Contract was violated. QueryResponse did not have property: {resourceName}"); } IJsonNavigatorNode resources = resourceProperty.ValueNode; if (!jsonNavigator.TryGetBufferedBinaryValue(resources, out ReadOnlyMemory <byte> resourceBinary)) { resourceBinary = jsonNavigator.GetBinaryValue(resources); } IJsonNavigator navigatorWithStringDictionary = JsonNavigator.Create(resourceBinary, jsonStringDictionary); if (!(CosmosElement.Dispatch( navigatorWithStringDictionary, navigatorWithStringDictionary.GetRootNode()) is CosmosArray cosmosArray)) { throw new InvalidOperationException($"QueryResponse did not have an array of : {resourceName}"); } documents = cosmosArray; } else { // Payload is not string dictionary encoded so we can just do for the documents as is. if (!jsonNavigator.TryGetObjectProperty( jsonNavigator.GetRootNode(), resourceName, out ObjectProperty objectProperty)) { throw new InvalidOperationException($"Response Body Contract was violated. QueryResponse did not have property: {resourceName}"); } if (!(CosmosElement.Dispatch( jsonNavigator, objectProperty.ValueNode) is CosmosArray cosmosArray)) { throw new InvalidOperationException($"QueryResponse did not have an array of : {resourceName}"); } documents = cosmosArray; } return(documents); }
/// <summary> /// Converts a list of CosmosElements into a memory stream. /// </summary> /// <param name="containerRid">Container Rid</param> /// <param name="cosmosElements">The cosmos elements</param> /// <param name="resourceType">The resource type</param> /// <param name="cosmosSerializationOptions">The custom serialization options. This allows custom serialization types like BSON, JSON, or other formats</param> /// <returns>Returns a memory stream of cosmos elements. By default the memory stream will contain JSON.</returns> internal static MemoryStream ToStream( string containerRid, IEnumerable <CosmosElement> cosmosElements, ResourceType resourceType, CosmosSerializationFormatOptions cosmosSerializationOptions = null) { IJsonWriter jsonWriter; if (cosmosSerializationOptions != null) { jsonWriter = cosmosSerializationOptions.CreateCustomWriterCallback(); } else { jsonWriter = JsonWriter.Create(JsonSerializationFormat.Text); } // The stream contract should return the same contract as read feed. // { // "_rid": "qHVdAImeKAQ=", // "Documents": [{ // "id": "03230", // "_rid": "qHVdAImeKAQBAAAAAAAAAA==", // "_self": "dbs\/qHVdAA==\/colls\/qHVdAImeKAQ=\/docs\/qHVdAImeKAQBAAAAAAAAAA==\/", // "_etag": "\"410000b0-0000-0000-0000-597916b00000\"", // "_attachments": "attachments\/", // "_ts": 1501107886 // }], // "_count": 1 // } jsonWriter.WriteObjectStart(); // Write the rid field and value jsonWriter.WriteFieldName("_rid"); jsonWriter.WriteStringValue(containerRid); // Write the array of elements string rootName = CosmosElementSerializer.GetRootNodeName(resourceType); jsonWriter.WriteFieldName(rootName); int count = 0; jsonWriter.WriteArrayStart(); foreach (CosmosElement element in cosmosElements) { count++; element.WriteTo(jsonWriter); } jsonWriter.WriteArrayEnd(); // Write the count field and value jsonWriter.WriteFieldName("_count"); jsonWriter.WriteNumber64Value(count); jsonWriter.WriteObjectEnd(); ReadOnlyMemory <byte> result = jsonWriter.GetResult(); if (!MemoryMarshal.TryGetArray(result, out ArraySegment <byte> resultAsArray)) { resultAsArray = new ArraySegment <byte>(result.ToArray()); } return(new MemoryStream(resultAsArray.Array, resultAsArray.Offset, resultAsArray.Count)); }
/// <summary> /// Converts a list of CosmosElements into a memory stream. /// </summary> /// <param name="memoryStream">The memory stream response from Azure Cosmos</param> /// <param name="resourceType">The resource type</param> /// <param name="cosmosSerializationOptions">The custom serialization options. This allows custom serialization types like BSON, JSON, or other formats</param> /// <returns>Returns a memory stream of cosmos elements. By default the memory stream will contain JSON.</returns> internal static CosmosArray ToCosmosElements( MemoryStream memoryStream, ResourceType resourceType, CosmosSerializationFormatOptions cosmosSerializationOptions = null) { if (!memoryStream.CanRead) { throw new InvalidDataException("Stream can not be read"); } // Execute the callback an each element of the page // For example just could get a response like this // { // "_rid": "qHVdAImeKAQ=", // "Documents": [{ // "id": "03230", // "_rid": "qHVdAImeKAQBAAAAAAAAAA==", // "_self": "dbs\/qHVdAA==\/colls\/qHVdAImeKAQ=\/docs\/qHVdAImeKAQBAAAAAAAAAA==\/", // "_etag": "\"410000b0-0000-0000-0000-597916b00000\"", // "_attachments": "attachments\/", // "_ts": 1501107886 // }], // "_count": 1 // } // And you should execute the callback on each document in "Documents". long responseLengthBytes = memoryStream.Length; byte[] content = memoryStream.ToArray(); IJsonNavigator jsonNavigator = null; // Use the users custom navigator if (cosmosSerializationOptions != null) { jsonNavigator = cosmosSerializationOptions.CreateCustomNavigatorCallback(content); if (jsonNavigator == null) { throw new InvalidOperationException("The CosmosSerializationOptions did not return a JSON navigator."); } } else { jsonNavigator = JsonNavigator.Create(new ArraySegment <byte>(content)); } string resourceName = CosmosElementSerializer.GetRootNodeName(resourceType); if (!jsonNavigator.TryGetObjectProperty( jsonNavigator.GetRootNode(), resourceName, out ObjectProperty objectProperty)) { throw new InvalidOperationException($"Response Body Contract was violated. QueryResponse did not have property: {resourceName}"); } IJsonNavigatorNode cosmosElements = objectProperty.ValueNode; if (!(CosmosElement.Dispatch( jsonNavigator, cosmosElements) is CosmosArray cosmosArray)) { throw new InvalidOperationException($"QueryResponse did not have an array of : {resourceName}"); } return(cosmosArray); }