/// <summary>
/// Converts a list of CosmosElements into a memory stream.
/// </summary>
/// <param name="cosmosElements">The cosmos elements</param>
/// <param name="cosmosSerializationOptions">The custom serialization options. This allows custom serialization types like BSON, JSON, or other formats</param>
/// <returns>Returns a memory stream of cosmos elements. By default the memory stream will contain JSON.</returns>
internal static MemoryStream ElementsToMemoryStream(
IReadOnlyList <CosmosElement> cosmosElements,
CosmosSerializationFormatOptions cosmosSerializationOptions = null)
{
IJsonWriter jsonWriter;
if (cosmosSerializationOptions != null)
{
jsonWriter = cosmosSerializationOptions.CreateCustomWriterCallback();
}
else
{
jsonWriter = JsonWriter.Create(JsonSerializationFormat.Text);
}
jsonWriter.WriteArrayStart();
foreach (CosmosElement element in cosmosElements)
{
element.WriteTo(jsonWriter);
}
jsonWriter.WriteArrayEnd();
return(GetMemoryStreamFromJsonWriter(jsonWriter));
}
/// <summary>
/// Converts a list of CosmosElements into a memory stream.
/// </summary>
/// <param name="cosmosElement">The cosmos elements</param>
/// <param name="cosmosSerializationOptions">The custom serialization options. This allows custom serialization types like BSON, JSON, or other formats</param>
/// <returns>Returns a memory stream of cosmos elements. By default the memory stream will contain JSON.</returns>
private static MemoryStream ElementToMemoryStream(
CosmosElement cosmosElement,
CosmosSerializationFormatOptions cosmosSerializationOptions = null)
{
IJsonWriter jsonWriter;
if (cosmosSerializationOptions != null)
{
jsonWriter = cosmosSerializationOptions.CreateCustomWriterCallback();
}
else
{
jsonWriter = JsonWriter.Create(JsonSerializationFormat.Text);
}
cosmosElement.WriteTo(jsonWriter);
ReadOnlyMemory <byte> result = jsonWriter.GetResult();
if (!MemoryMarshal.TryGetArray(result, out ArraySegment <byte> resultAsArray))
{
resultAsArray = new ArraySegment <byte>(result.ToArray());
}
return(new MemoryStream(resultAsArray.Array, resultAsArray.Offset, resultAsArray.Count));
}
private QueryIterator(
CosmosQueryContext cosmosQueryContext,
CosmosQueryExecutionContext cosmosQueryExecutionContext,
CosmosSerializationFormatOptions cosmosSerializationFormatOptions)
{
this.cosmosQueryContext = cosmosQueryContext ?? throw new ArgumentNullException(nameof(cosmosQueryContext));
this.cosmosQueryExecutionContext = cosmosQueryExecutionContext ?? throw new ArgumentNullException(nameof(cosmosQueryExecutionContext));
this.cosmosSerializationFormatOptions = cosmosSerializationFormatOptions;
}
internal static T[] GetResourcesHelper <T>(
IReadOnlyList <CosmosElement> cosmosElements,
CosmosSerializerCore serializerCore,
CosmosSerializationFormatOptions cosmosSerializationOptions = null)
{
using (MemoryStream memoryStream = ElementsToMemoryStream(
cosmosElements,
cosmosSerializationOptions))
{
return(serializerCore.FromFeedStream <T>(memoryStream));
}
}
private QueryIterator(
CosmosQueryContextCore cosmosQueryContext,
CosmosQueryExecutionContext cosmosQueryExecutionContext,
CosmosSerializationFormatOptions cosmosSerializationFormatOptions,
RequestOptions requestOptions,
CosmosClientContext clientContext)
{
this.cosmosQueryContext = cosmosQueryContext ?? throw new ArgumentNullException(nameof(cosmosQueryContext));
this.cosmosQueryExecutionContext = cosmosQueryExecutionContext ?? throw new ArgumentNullException(nameof(cosmosQueryExecutionContext));
this.cosmosSerializationFormatOptions = cosmosSerializationFormatOptions;
this.requestOptions = requestOptions;
this.clientContext = clientContext ?? throw new ArgumentNullException(nameof(clientContext));
}
private QueryIterator(
CosmosQueryContextCore cosmosQueryContext,
IQueryPipelineStage cosmosQueryExecutionContext,
CosmosSerializationFormatOptions cosmosSerializationFormatOptions,
RequestOptions requestOptions,
CosmosClientContext clientContext)
{
this.cosmosQueryContext = cosmosQueryContext ?? throw new ArgumentNullException(nameof(cosmosQueryContext));
this.queryPipelineStage = cosmosQueryExecutionContext ?? throw new ArgumentNullException(nameof(cosmosQueryExecutionContext));
this.cosmosSerializationFormatOptions = cosmosSerializationFormatOptions;
this.requestOptions = requestOptions;
this.clientContext = clientContext ?? throw new ArgumentNullException(nameof(clientContext));
this.hasMoreResults = true;
}
public async Task DecryptQueryBinaryResponse()
{
TestDoc testDoc = await EncryptionTests.CreateItemAsync(EncryptionTests.containerCore, EncryptionTests.dekId, TestDoc.PathsToEncrypt);
CosmosSerializationFormatOptions options = new CosmosSerializationFormatOptions(
Documents.ContentSerializationFormat.CosmosBinary.ToString(),
(content) => JsonNavigator.Create(content),
() => JsonWriter.Create(JsonSerializationFormat.Binary));
QueryRequestOptions requestOptions = new QueryRequestOptions()
{
CosmosSerializationFormatOptions = options
};
TestDoc expectedDoc = new TestDoc(testDoc);
string query = "SELECT * FROM c";
FeedIterator feedIterator = EncryptionTests.containerCore.GetItemQueryStreamIterator(
query,
requestOptions: requestOptions);
while (feedIterator.HasMoreResults)
{
ResponseMessage response = await feedIterator.ReadNextAsync();
Assert.IsTrue(response.IsSuccessStatusCode);
Assert.IsNull(response.ErrorMessage);
// Copy the stream and check that the first byte is the correct value
MemoryStream memoryStream = new MemoryStream();
response.Content.CopyTo(memoryStream);
byte[] content = memoryStream.ToArray();
response.Content.Position = 0;
// Examine the first buffer byte to determine the serialization format
byte firstByte = content[0];
Assert.AreEqual(128, firstByte);
Assert.AreEqual(JsonSerializationFormat.Binary, (JsonSerializationFormat)firstByte);
IJsonReader reader = JsonReader.Create(content);
IJsonWriter textWriter = JsonWriter.Create(JsonSerializationFormat.Text);
textWriter.WriteAll(reader);
string json = Encoding.UTF8.GetString(textWriter.GetResult().ToArray());
Assert.IsNotNull(json);
Assert.IsTrue(json.Contains(testDoc.Sensitive));
}
}
/// <summary>
/// Converts a list of CosmosElements into a memory stream.
/// </summary>
/// <param name="stream">The stream response from Azure Cosmos</param>
/// <param name="resourceType">The resource type</param>
/// <param name="cosmosSerializationOptions">The custom serialization options. This allows custom serialization types like BSON, JSON, or other formats</param>
/// <returns>Returns a memory stream of cosmos elements. By default the memory stream will contain JSON.</returns>
internal static CosmosArray ToCosmosElements(
Stream stream,
ResourceType resourceType,
CosmosSerializationFormatOptions cosmosSerializationOptions = null)
{
MemoryStream memoryStream = stream as MemoryStream;
if (memoryStream == null)
{
memoryStream = new MemoryStream();
stream.CopyTo(memoryStream);
}
return(CosmosElementSerializer.ToCosmosElements(
memoryStream,
resourceType,
cosmosSerializationOptions));
}
private static void SetSerializationFormat(
QueryRequestOptions queryRequestOptions,
JsonSerializationFormat jsonSerializationFormat)
{
string contentSerializationFormat = jsonSerializationFormat switch
{
JsonSerializationFormat.Text => "JsonText",
JsonSerializationFormat.Binary => "CosmosBinary",
JsonSerializationFormat.HybridRow => "HybridRow",
_ => throw new Exception(),
};
CosmosSerializationFormatOptions formatOptions = new CosmosSerializationFormatOptions(
contentSerializationFormat,
(content) => JsonNavigator.Create(content),
() => JsonWriter.Create(JsonSerializationFormat.Text));
queryRequestOptions.CosmosSerializationFormatOptions = formatOptions;
}
internal QueryIterator(
CosmosQueryClient client,
SqlQuerySpec sqlQuerySpec,
string continuationToken,
QueryRequestOptions queryRequestOptions,
Uri resourceLink,
bool isContinuationExpected,
bool allowNonValueAggregateQuery)
{
if (queryRequestOptions == null)
{
queryRequestOptions = new QueryRequestOptions();
}
CosmosQueryContext context = new CosmosQueryContextCore(
client: client,
queryRequestOptions: queryRequestOptions,
resourceTypeEnum: Documents.ResourceType.Document,
operationType: Documents.OperationType.Query,
resourceType: typeof(QueryResponseCore),
resourceLink: resourceLink,
isContinuationExpected: isContinuationExpected,
allowNonValueAggregateQuery: allowNonValueAggregateQuery,
correlatedActivityId: Guid.NewGuid());
CosmosQueryExecutionContextFactory.InputParameters inputParams = new CosmosQueryExecutionContextFactory.InputParameters()
{
SqlQuerySpec = sqlQuerySpec,
InitialUserContinuationToken = continuationToken,
MaxBufferedItemCount = queryRequestOptions.MaxBufferedItemCount,
MaxConcurrency = queryRequestOptions.MaxConcurrency,
MaxItemCount = queryRequestOptions.MaxItemCount,
PartitionKey = queryRequestOptions.PartitionKey,
Properties = queryRequestOptions.Properties
};
this.cosmosSerializationFormatOptions = queryRequestOptions.CosmosSerializationFormatOptions;
this.cosmosQueryExecutionContext = new CosmosQueryExecutionContextFactory(
cosmosQueryContext: context,
inputParameters: inputParams);
}
/// <summary>
/// Converts a list of CosmosElements into a list of objects.
/// </summary>
/// <param name="containerRid">Container Rid</param>
/// <param name="cosmosElements">The cosmos elements</param>
/// <param name="resourceType">The resource type</param>
/// <param name="jsonSerializer">The JSON </param>
/// <param name="cosmosSerializationOptions">The custom serialization options. This allows custom serialization types like BSON, JSON, or other formats</param>
/// <returns>Returns a list of deserialized objects</returns>
internal static IEnumerable <T> Deserialize <T>(
string containerRid,
IEnumerable <CosmosElement> cosmosElements,
ResourceType resourceType,
CosmosSerializer jsonSerializer,
CosmosSerializationFormatOptions cosmosSerializationOptions = null)
{
if (!cosmosElements.Any())
{
return(Enumerable.Empty <T>());
}
Stream stream = CosmosElementSerializer.ToStream(
containerRid,
cosmosElements,
resourceType,
cosmosSerializationOptions);
IEnumerable <T> typedResults = jsonSerializer.FromStream <CosmosFeedResponseUtil <T> >(stream).Data;
return(typedResults);
}
/// <summary>
/// Converts a list of CosmosElements into a memory stream.
/// </summary>
/// <param name="memoryStream">The memory stream response from Azure Cosmos</param>
/// <param name="resourceType">The resource type</param>
/// <param name="cosmosSerializationOptions">The custom serialization options. This allows custom serialization types like BSON, JSON, or other formats</param>
/// <returns>Returns a memory stream of cosmos elements. By default the memory stream will contain JSON.</returns>
internal static CosmosArray ToCosmosElements(
MemoryStream memoryStream,
ResourceType resourceType,
CosmosSerializationFormatOptions cosmosSerializationOptions = null)
{
if (!memoryStream.CanRead)
{
throw new InvalidDataException("Stream can not be read");
}
// Execute the callback an each element of the page
// For example just could get a response like this
// {
// "_rid": "qHVdAImeKAQ=",
// "Documents": [{
// "id": "03230",
// "_rid": "qHVdAImeKAQBAAAAAAAAAA==",
// "_self": "dbs\/qHVdAA==\/colls\/qHVdAImeKAQ=\/docs\/qHVdAImeKAQBAAAAAAAAAA==\/",
// "_etag": "\"410000b0-0000-0000-0000-597916b00000\"",
// "_attachments": "attachments\/",
// "_ts": 1501107886
// }],
// "_count": 1
// }
// And you should execute the callback on each document in "Documents".
long responseLengthBytes = memoryStream.Length;
ReadOnlyMemory <byte> content;
if (memoryStream.TryGetBuffer(out ArraySegment <byte> buffer))
{
content = buffer;
}
else
{
content = memoryStream.ToArray();
}
IJsonNavigator jsonNavigator;
// Use the users custom navigator
if (cosmosSerializationOptions != null)
{
jsonNavigator = cosmosSerializationOptions.CreateCustomNavigatorCallback(content);
if (jsonNavigator == null)
{
throw new InvalidOperationException("The CosmosSerializationOptions did not return a JSON navigator.");
}
}
else
{
jsonNavigator = JsonNavigator.Create(content);
}
string resourceName = CosmosElementSerializer.GetRootNodeName(resourceType);
CosmosArray documents;
if ((jsonNavigator.SerializationFormat == JsonSerializationFormat.Binary) && jsonNavigator.TryGetObjectProperty(
jsonNavigator.GetRootNode(),
"stringDictionary",
out ObjectProperty stringDictionaryProperty))
{
// Payload is string dictionary encode so we have to decode using the string dictionary.
IJsonNavigatorNode stringDictionaryNode = stringDictionaryProperty.ValueNode;
JsonStringDictionary jsonStringDictionary = JsonStringDictionary.CreateFromStringArray(
jsonNavigator
.GetArrayItems(stringDictionaryNode)
.Select(item => jsonNavigator.GetStringValue(item))
.ToList());
if (!jsonNavigator.TryGetObjectProperty(
jsonNavigator.GetRootNode(),
resourceName,
out ObjectProperty resourceProperty))
{
throw new InvalidOperationException($"Response Body Contract was violated. QueryResponse did not have property: {resourceName}");
}
IJsonNavigatorNode resources = resourceProperty.ValueNode;
if (!jsonNavigator.TryGetBufferedBinaryValue(resources, out ReadOnlyMemory <byte> resourceBinary))
{
resourceBinary = jsonNavigator.GetBinaryValue(resources);
}
IJsonNavigator navigatorWithStringDictionary = JsonNavigator.Create(resourceBinary, jsonStringDictionary);
if (!(CosmosElement.Dispatch(
navigatorWithStringDictionary,
navigatorWithStringDictionary.GetRootNode()) is CosmosArray cosmosArray))
{
throw new InvalidOperationException($"QueryResponse did not have an array of : {resourceName}");
}
documents = cosmosArray;
}
else
{
// Payload is not string dictionary encoded so we can just do for the documents as is.
if (!jsonNavigator.TryGetObjectProperty(
jsonNavigator.GetRootNode(),
resourceName,
out ObjectProperty objectProperty))
{
throw new InvalidOperationException($"Response Body Contract was violated. QueryResponse did not have property: {resourceName}");
}
if (!(CosmosElement.Dispatch(
jsonNavigator,
objectProperty.ValueNode) is CosmosArray cosmosArray))
{
throw new InvalidOperationException($"QueryResponse did not have an array of : {resourceName}");
}
documents = cosmosArray;
}
return(documents);
}
/// <summary>
/// Converts a list of CosmosElements into a memory stream.
/// </summary>
/// <param name="containerRid">Container Rid</param>
/// <param name="cosmosElements">The cosmos elements</param>
/// <param name="resourceType">The resource type</param>
/// <param name="cosmosSerializationOptions">The custom serialization options. This allows custom serialization types like BSON, JSON, or other formats</param>
/// <returns>Returns a memory stream of cosmos elements. By default the memory stream will contain JSON.</returns>
internal static MemoryStream ToStream(
string containerRid,
IEnumerable <CosmosElement> cosmosElements,
ResourceType resourceType,
CosmosSerializationFormatOptions cosmosSerializationOptions = null)
{
IJsonWriter jsonWriter;
if (cosmosSerializationOptions != null)
{
jsonWriter = cosmosSerializationOptions.CreateCustomWriterCallback();
}
else
{
jsonWriter = JsonWriter.Create(JsonSerializationFormat.Text);
}
// The stream contract should return the same contract as read feed.
// {
// "_rid": "qHVdAImeKAQ=",
// "Documents": [{
// "id": "03230",
// "_rid": "qHVdAImeKAQBAAAAAAAAAA==",
// "_self": "dbs\/qHVdAA==\/colls\/qHVdAImeKAQ=\/docs\/qHVdAImeKAQBAAAAAAAAAA==\/",
// "_etag": "\"410000b0-0000-0000-0000-597916b00000\"",
// "_attachments": "attachments\/",
// "_ts": 1501107886
// }],
// "_count": 1
// }
jsonWriter.WriteObjectStart();
// Write the rid field and value
jsonWriter.WriteFieldName("_rid");
jsonWriter.WriteStringValue(containerRid);
// Write the array of elements
string rootName = CosmosElementSerializer.GetRootNodeName(resourceType);
jsonWriter.WriteFieldName(rootName);
int count = 0;
jsonWriter.WriteArrayStart();
foreach (CosmosElement element in cosmosElements)
{
count++;
element.WriteTo(jsonWriter);
}
jsonWriter.WriteArrayEnd();
// Write the count field and value
jsonWriter.WriteFieldName("_count");
jsonWriter.WriteNumber64Value(count);
jsonWriter.WriteObjectEnd();
ReadOnlyMemory <byte> result = jsonWriter.GetResult();
if (!MemoryMarshal.TryGetArray(result, out ArraySegment <byte> resultAsArray))
{
resultAsArray = new ArraySegment <byte>(result.ToArray());
}
return(new MemoryStream(resultAsArray.Array, resultAsArray.Offset, resultAsArray.Count));
}
/// <summary>
/// Converts a list of CosmosElements into a memory stream.
/// </summary>
/// <param name="memoryStream">The memory stream response from Azure Cosmos</param>
/// <param name="resourceType">The resource type</param>
/// <param name="cosmosSerializationOptions">The custom serialization options. This allows custom serialization types like BSON, JSON, or other formats</param>
/// <returns>Returns a memory stream of cosmos elements. By default the memory stream will contain JSON.</returns>
internal static CosmosArray ToCosmosElements(
MemoryStream memoryStream,
ResourceType resourceType,
CosmosSerializationFormatOptions cosmosSerializationOptions = null)
{
if (!memoryStream.CanRead)
{
throw new InvalidDataException("Stream can not be read");
}
// Execute the callback an each element of the page
// For example just could get a response like this
// {
// "_rid": "qHVdAImeKAQ=",
// "Documents": [{
// "id": "03230",
// "_rid": "qHVdAImeKAQBAAAAAAAAAA==",
// "_self": "dbs\/qHVdAA==\/colls\/qHVdAImeKAQ=\/docs\/qHVdAImeKAQBAAAAAAAAAA==\/",
// "_etag": "\"410000b0-0000-0000-0000-597916b00000\"",
// "_attachments": "attachments\/",
// "_ts": 1501107886
// }],
// "_count": 1
// }
// And you should execute the callback on each document in "Documents".
long responseLengthBytes = memoryStream.Length;
byte[] content = memoryStream.ToArray();
IJsonNavigator jsonNavigator = null;
// Use the users custom navigator
if (cosmosSerializationOptions != null)
{
jsonNavigator = cosmosSerializationOptions.CreateCustomNavigatorCallback(content);
if (jsonNavigator == null)
{
throw new InvalidOperationException("The CosmosSerializationOptions did not return a JSON navigator.");
}
}
else
{
jsonNavigator = JsonNavigator.Create(new ArraySegment <byte>(content));
}
string resourceName = CosmosElementSerializer.GetRootNodeName(resourceType);
if (!jsonNavigator.TryGetObjectProperty(
jsonNavigator.GetRootNode(),
resourceName,
out ObjectProperty objectProperty))
{
throw new InvalidOperationException($"Response Body Contract was violated. QueryResponse did not have property: {resourceName}");
}
IJsonNavigatorNode cosmosElements = objectProperty.ValueNode;
if (!(CosmosElement.Dispatch(
jsonNavigator,
cosmosElements) is CosmosArray cosmosArray))
{
throw new InvalidOperationException($"QueryResponse did not have an array of : {resourceName}");
}
return(cosmosArray);
}
