public void LargeArrayBinaryJsonTest()
{
StringBuilder builder = new StringBuilder((1 << 24) + 50);
builder.Append('[');
for (int x = 1 << 24; x < (1 << 24) + 3355450; ++x)
{
builder.Append(x);
builder.Append(',');
}
builder.Append("\"string_one\"");
builder.Append(',');
builder.Append("\"string_two\"");
builder.Append(',');
builder.Append("\"string_two\"");
builder.Append(']');
string json = builder.ToString();
byte[] binaryJson = JsonTestUtils.ConvertTextToBinary(json);
IJsonNavigator navigator = JsonNavigator.Create(binaryJson);
int count = navigator.GetArrayItemCount(navigator.GetRootNode());
IJsonNavigatorNode node = navigator.GetArrayItemAt(navigator.GetRootNode(), count - 1);
string stringValue = navigator.GetStringValue(node);
Assert.AreEqual("string_two", stringValue);
}
public static CosmosElement Create(byte[] buffer)
{
IJsonNavigator jsonNavigator = JsonNavigator.Create(buffer);
IJsonNavigatorNode jsonNavigatorNode = jsonNavigator.GetRootNode();
return(CosmosElement.Dispatch(jsonNavigator, jsonNavigatorNode));
}
public static CosmosElement CreateFromBuffer(ReadOnlyMemory <byte> buffer)
{
IJsonNavigator jsonNavigator = JsonNavigator.Create(buffer);
IJsonNavigatorNode jsonNavigatorNode = jsonNavigator.GetRootNode();
return(CosmosElement.Dispatch(jsonNavigator, jsonNavigatorNode));
}
public static TryCatch <TCosmosElement> CreateFromBuffer <TCosmosElement>(ReadOnlyMemory <byte> buffer)
where TCosmosElement : CosmosElement
{
if (buffer.IsEmpty)
{
TryCatch <TCosmosElement> .FromException(
new ArgumentException($"{nameof(buffer)} must not be empty."));
}
CosmosElement unTypedCosmosElement;
try
{
IJsonNavigator jsonNavigator = JsonNavigator.Create(buffer);
IJsonNavigatorNode jsonNavigatorNode = jsonNavigator.GetRootNode();
unTypedCosmosElement = CosmosElement.Dispatch(jsonNavigator, jsonNavigatorNode);
}
catch (JsonParseException jpe)
{
return(TryCatch <TCosmosElement> .FromException(jpe));
}
if (!(unTypedCosmosElement is TCosmosElement typedCosmosElement))
{
return(TryCatch <TCosmosElement> .FromException(
new CosmosElementWrongTypeException(
message : $"buffer was incorrect cosmos element type: {unTypedCosmosElement.GetType()} when {typeof(TCosmosElement)} was requested.")));
}
return(TryCatch <TCosmosElement> .FromResult(typedCosmosElement));
}
public static (QueryResponse queryResponse, IList <ToDoItem> items) Create(
string itemIdPrefix,
string continuationToken,
string collectionRid,
int itemCount = 50)
{
// Use -1 to represent a split response
if (itemCount == QueryResponseMessageFactory.SPLIT)
{
return(CreateSplitResponse(collectionRid), new List <ToDoItem>().AsReadOnly());
}
IList <ToDoItem> items = ToDoItem.CreateItems(itemCount, itemIdPrefix);
MemoryStream memoryStream = (MemoryStream)cosmosSerializer.ToStream <IList <ToDoItem> >(items);
long responseLengthBytes = memoryStream.Length;
IJsonNavigator jsonNavigator = JsonNavigator.Create(memoryStream.ToArray());
IJsonNavigatorNode jsonNavigatorNode = jsonNavigator.GetRootNode();
CosmosArray cosmosArray = CosmosArray.Create(jsonNavigator, jsonNavigatorNode);
Headers headers = new Headers();
headers.ContinuationToken = continuationToken;
headers.ActivityId = Guid.NewGuid().ToString();
QueryResponse message = QueryResponse.CreateSuccess(
result: cosmosArray,
count: itemCount,
responseHeaders: CosmosQueryResponseMessageHeaders.ConvertToQueryHeaders(headers, ResourceType.Document, collectionRid),
responseLengthBytes: responseLengthBytes);
return(message, items);
}
public void NavigatorToWriter(
CurratedDocsPayload payload,
SerializationFormat sourceFormat,
SerializationFormat destinationFormat)
{
IJsonNavigator navigator = sourceFormat switch
{
SerializationFormat.Text => JsonNavigator.Create(payload.Text),
SerializationFormat.BinaryWithDictionaryEncoding => JsonNavigator.Create(
payload.BinaryWithDictionaryEncoding.binary,
payload.BinaryWithDictionaryEncoding.dictionary),
SerializationFormat.Binary => JsonNavigator.Create(payload.Binary),
_ => throw new ArgumentException($"Unexpected {nameof(sourceFormat)} of type: '{sourceFormat}'"),
};
IJsonWriter writer = destinationFormat switch
{
SerializationFormat.Text => JsonWriter.Create(JsonSerializationFormat.Text),
SerializationFormat.Binary => JsonWriter.Create(JsonSerializationFormat.Binary),
SerializationFormat.BinaryWithDictionaryEncoding => JsonWriter.Create(
JsonSerializationFormat.Binary,
new JsonStringDictionary(capacity: 128)),
SerializationFormat.NewtonsoftText => NewtonsoftToCosmosDBWriter.CreateTextWriter(),
_ => throw new ArgumentException($"Unexpected {nameof(destinationFormat)} of type: {destinationFormat}"),
};
navigator.WriteTo(navigator.GetRootNode(), writer);
}
public static QueryResponseCore CreateQueryResponse(
IList <ToDoItem> items,
bool isOrderByQuery,
string continuationToken,
string collectionRid)
{
MemoryStream memoryStream;
string json;
if (isOrderByQuery)
{
memoryStream = SerializeForOrderByQuery(items);
using (StreamReader sr = new StreamReader(SerializeForOrderByQuery(items)))
{
json = sr.ReadToEnd();
}
}
else
{
memoryStream = (MemoryStream)MockCosmosUtil.Serializer.ToStream <IList <ToDoItem> >(items);
}
long responseLengthBytes = memoryStream.Length;
IJsonNavigator jsonNavigator = JsonNavigator.Create(memoryStream.ToArray());
IJsonNavigatorNode jsonNavigatorNode = jsonNavigator.GetRootNode();
CosmosArray cosmosArray = CosmosArray.Create(jsonNavigator, jsonNavigatorNode);
CosmosDiagnosticsContext diagnosticsContext = new CosmosDiagnosticsContextCore();
diagnosticsContext.AddDiagnosticsInternal(new PointOperationStatistics(
activityId: Guid.NewGuid().ToString(),
statusCode: HttpStatusCode.OK,
subStatusCode: SubStatusCodes.Unknown,
responseTimeUtc: DateTime.UtcNow,
requestCharge: 4,
errorMessage: null,
method: HttpMethod.Post,
requestUri: "http://localhost.com",
requestSessionToken: null,
responseSessionToken: null));
IReadOnlyCollection <QueryPageDiagnostics> diagnostics = new List <QueryPageDiagnostics>()
{
new QueryPageDiagnostics(
Guid.NewGuid(),
"0",
"SomeQueryMetricText",
"SomeIndexUtilText",
diagnosticsContext)
};
QueryResponseCore message = QueryResponseCore.CreateSuccess(
result: cosmosArray,
requestCharge: 4,
activityId: Guid.NewGuid().ToString(),
responseLengthBytes: responseLengthBytes,
disallowContinuationTokenMessage: null,
continuationToken: continuationToken);
return(message);
}
public static (QueryResponseCore queryResponse, IList <ToDoItem> items) Create(
string itemIdPrefix,
string continuationToken,
string collectionRid,
int itemCount = 50)
{
// Use -1 to represent a split response
if (itemCount == QueryResponseMessageFactory.SPLIT)
{
return(CreateSplitResponse(collectionRid), new List <ToDoItem>().AsReadOnly());
}
IList <ToDoItem> items = ToDoItem.CreateItems(itemCount, itemIdPrefix);
MemoryStream memoryStream = (MemoryStream)cosmosSerializer.ToStream <IList <ToDoItem> >(items);
long responseLengthBytes = memoryStream.Length;
IJsonNavigator jsonNavigator = JsonNavigator.Create(memoryStream.ToArray());
IJsonNavigatorNode jsonNavigatorNode = jsonNavigator.GetRootNode();
CosmosArray cosmosArray = CosmosArray.Create(jsonNavigator, jsonNavigatorNode);
QueryResponseCore message = QueryResponseCore.CreateSuccess(
result: cosmosArray,
continuationToken: continuationToken,
disallowContinuationTokenMessage: null,
activityId: Guid.NewGuid().ToString(),
requestCharge: 42,
queryMetricsText: null,
queryMetrics: new Dictionary <string, QueryMetrics>(),
requestStatistics: null,
responseLengthBytes: responseLengthBytes);
return(message, items);
}
private static void VerifyNavigator(
string input,
bool performExtraChecks = true)
{
CultureInfo defaultCultureInfo = System.Threading.Thread.CurrentThread.CurrentCulture;
CultureInfo[] cultureInfoList = new CultureInfo[]
{
defaultCultureInfo,
System.Globalization.CultureInfo.GetCultureInfo("fr-FR")
};
try
{
foreach (CultureInfo cultureInfo in cultureInfoList)
{
System.Threading.Thread.CurrentThread.CurrentCulture = cultureInfo;
IJsonReader jsonReader = JsonReader.Create(Encoding.UTF8.GetBytes(input));
JsonToken[] tokensFromReader = JsonNavigatorTests.GetTokensWithReader(jsonReader);
// Test text
IJsonNavigator textNavigator = JsonNavigator.Create(Encoding.UTF8.GetBytes(input));
IJsonNavigatorNode textRootNode = textNavigator.GetRootNode();
JsonToken[] tokensFromTextNavigator = JsonNavigatorTests.GetTokensFromNode(textRootNode, textNavigator, performExtraChecks);
Assert.IsTrue(tokensFromTextNavigator.SequenceEqual(tokensFromReader));
// Test binary
byte[] binaryInput = JsonTestUtils.ConvertTextToBinary(input);
IJsonNavigator binaryNavigator = JsonNavigator.Create(binaryInput);
IJsonNavigatorNode binaryRootNode = binaryNavigator.GetRootNode();
JsonToken[] tokensFromBinaryNavigator = JsonNavigatorTests.GetTokensFromNode(binaryRootNode, binaryNavigator, performExtraChecks);
Assert.IsTrue(tokensFromBinaryNavigator.SequenceEqual(tokensFromReader));
// Test binary + user string encoding
JsonStringDictionary jsonStringDictionary = new JsonStringDictionary(capacity: 4096);
byte[] binaryWithUserStringEncodingInput = JsonTestUtils.ConvertTextToBinary(input, jsonStringDictionary);
if (jsonStringDictionary.TryGetStringAtIndex(index: 0, value: out _))
{
Assert.IsFalse(binaryWithUserStringEncodingInput.SequenceEqual(binaryInput), "Binary should be different with user string encoding");
}
IJsonNavigator binaryNavigatorWithUserStringEncoding = JsonNavigator.Create(binaryInput, jsonStringDictionary);
IJsonNavigatorNode binaryRootNodeWithUserStringEncoding = binaryNavigatorWithUserStringEncoding.GetRootNode();
JsonToken[] tokensFromBinaryNavigatorWithUserStringEncoding = JsonNavigatorTests.GetTokensFromNode(binaryRootNodeWithUserStringEncoding, binaryNavigatorWithUserStringEncoding, performExtraChecks);
Assert.IsTrue(tokensFromBinaryNavigatorWithUserStringEncoding.SequenceEqual(tokensFromReader));
}
}
finally
{
System.Threading.Thread.CurrentThread.CurrentCulture = defaultCultureInfo;
}
}
public static (QueryResponseCore queryResponse, IList <ToDoItem> items) Create(
string itemIdPrefix,
string continuationToken,
string collectionRid,
int itemCount = 50)
{
// Use -1 to represent a split response
if (itemCount == QueryResponseMessageFactory.SPLIT)
{
return(CreateSplitResponse(collectionRid), new List <ToDoItem>().AsReadOnly());
}
IList <ToDoItem> items = ToDoItem.CreateItems(itemCount, itemIdPrefix);
MemoryStream memoryStream = (MemoryStream)MockCosmosUtil.Serializer.ToStream <IList <ToDoItem> >(items);
long responseLengthBytes = memoryStream.Length;
IJsonNavigator jsonNavigator = JsonNavigator.Create(memoryStream.ToArray());
IJsonNavigatorNode jsonNavigatorNode = jsonNavigator.GetRootNode();
CosmosArray cosmosArray = CosmosArray.Create(jsonNavigator, jsonNavigatorNode);
double requestCharge = 42;
string activityId = Guid.NewGuid().ToString();
CosmosDiagnosticsContext diagnosticsContext = new CosmosDiagnosticsContextCore();
diagnosticsContext.AddDiagnosticsInternal(new PointOperationStatistics(
activityId: Guid.NewGuid().ToString(),
statusCode: HttpStatusCode.OK,
subStatusCode: SubStatusCodes.Unknown,
responseTimeUtc: DateTime.UtcNow,
requestCharge: requestCharge,
errorMessage: null,
method: HttpMethod.Post,
requestUri: "http://localhost.com",
requestSessionToken: null,
responseSessionToken: null));
IReadOnlyCollection <QueryPageDiagnostics> diagnostics = new List <QueryPageDiagnostics>()
{
new QueryPageDiagnostics(
Guid.NewGuid(),
"0",
"SomeQueryMetricText",
"SomeIndexUtilText",
diagnosticsContext)
};
QueryResponseCore message = QueryResponseCore.CreateSuccess(
result: cosmosArray,
continuationToken: continuationToken,
disallowContinuationTokenMessage: null,
activityId: activityId,
requestCharge: requestCharge,
responseLengthBytes: responseLengthBytes);
return(message, items);
}
internal async Task <PartitionKey> GetPartitionKeyValueFromStreamAsync(
Stream stream,
CancellationToken cancellation = default(CancellationToken))
{
if (!stream.CanSeek)
{
throw new ArgumentException("Stream is needs to be seekable", nameof(stream));
}
try
{
stream.Position = 0;
MemoryStream memoryStream = stream as MemoryStream;
if (memoryStream == null)
{
memoryStream = new MemoryStream();
stream.CopyTo(memoryStream);
}
// TODO: Avoid copy
IJsonNavigator jsonNavigator = JsonNavigator.Create(memoryStream.ToArray());
IJsonNavigatorNode jsonNavigatorNode = jsonNavigator.GetRootNode();
CosmosObject pathTraversal = CosmosObject.Create(jsonNavigator, jsonNavigatorNode);
string[] tokens = await this.GetPartitionKeyPathTokensAsync(cancellation);
for (int i = 0; i < tokens.Length - 1; i++)
{
pathTraversal = pathTraversal[tokens[i]] as CosmosObject;
if (pathTraversal == null)
{
return(PartitionKey.None);
}
}
CosmosElement partitionKeyValue = pathTraversal[tokens[tokens.Length - 1]];
if (partitionKeyValue == null)
{
return(PartitionKey.None);
}
return(this.CosmosElementToPartitionKeyObject(partitionKeyValue));
}
finally
{
// MemoryStream casting leverage might change position
stream.Position = 0;
}
}
public override async Task <PartitionKey> GetPartitionKeyValueFromStreamAsync(
Stream stream,
CancellationToken cancellation = default(CancellationToken))
{
if (!stream.CanSeek)
{
throw new ArgumentException("Stream needs to be seekable", nameof(stream));
}
try
{
stream.Position = 0;
if (!(stream is MemoryStream memoryStream))
{
memoryStream = new MemoryStream();
stream.CopyTo(memoryStream);
}
// TODO: Avoid copy
IJsonNavigator jsonNavigator = JsonNavigator.Create(memoryStream.ToArray());
IJsonNavigatorNode jsonNavigatorNode = jsonNavigator.GetRootNode();
CosmosObject pathTraversal = CosmosObject.Create(jsonNavigator, jsonNavigatorNode);
IReadOnlyList <IReadOnlyList <string> > tokenslist = await this.GetPartitionKeyPathTokensAsync(cancellation);
List <CosmosElement> cosmosElementList = new List <CosmosElement>(tokenslist.Count);
foreach (IReadOnlyList <string> tokenList in tokenslist)
{
CosmosElement element;
if (ContainerCore.TryParseTokenListForElement(pathTraversal, tokenList, out element))
{
cosmosElementList.Add(element);
}
else
{
cosmosElementList.Add(null);
}
}
return(ContainerCore.CosmosElementToPartitionKeyObject(cosmosElementList));
}
finally
{
// MemoryStream casting leverage might change position
stream.Position = 0;
}
}
public static TimeSpan MeasureNavigationPerformance(IJsonNavigator jsonNavigator, int numberOfIterations = 1)
{
JsonToken[] tokensFromNode;
Stopwatch stopwatch = new Stopwatch();
for (int i = 0; i < numberOfIterations; i++)
{
stopwatch.Start();
tokensFromNode = JsonNavigatorTests.GetTokensFromNode(jsonNavigator.GetRootNode(), jsonNavigator, false);
stopwatch.Stop();
if (tokensFromNode.Length == 0)
{
throw new InvalidOperationException("got back zero tokens");
}
}
return(stopwatch.Elapsed);
}
public static QueryResponseCore CreateQueryResponse(
IList <ToDoItem> items,
bool isOrderByQuery,
string continuationToken,
string collectionRid)
{
MemoryStream memoryStream;
string json;
if (isOrderByQuery)
{
memoryStream = SerializeForOrderByQuery(items);
using (StreamReader sr = new StreamReader(SerializeForOrderByQuery(items)))
{
json = sr.ReadToEnd();
}
}
else
{
memoryStream = (MemoryStream)cosmosSerializer.ToStream <IList <ToDoItem> >(items);
}
long responseLengthBytes = memoryStream.Length;
IJsonNavigator jsonNavigator = JsonNavigator.Create(memoryStream.ToArray());
IJsonNavigatorNode jsonNavigatorNode = jsonNavigator.GetRootNode();
CosmosArray cosmosArray = CosmosArray.Create(jsonNavigator, jsonNavigatorNode);
QueryResponseCore message = QueryResponseCore.CreateSuccess(
result: cosmosArray,
requestCharge: 4,
activityId: Guid.NewGuid().ToString(),
queryMetricsText: null,
queryMetrics: null,
requestStatistics: null,
responseLengthBytes: responseLengthBytes,
disallowContinuationTokenMessage: null,
continuationToken: continuationToken);
return(message);
}
public static QueryResponse CreateQueryResponse(
IList <ToDoItem> items,
bool isOrderByQuery,
string continuationToken,
string collectionRid)
{
MemoryStream memoryStream;
string json;
if (isOrderByQuery)
{
memoryStream = SerializeForOrderByQuery(items);
using (StreamReader sr = new StreamReader(SerializeForOrderByQuery(items)))
{
json = sr.ReadToEnd();
}
}
else
{
memoryStream = (MemoryStream)cosmosSerializer.ToStream <IList <ToDoItem> >(items);
}
long responseLengthBytes = memoryStream.Length;
IJsonNavigator jsonNavigator = JsonNavigator.Create(memoryStream.ToArray());
IJsonNavigatorNode jsonNavigatorNode = jsonNavigator.GetRootNode();
CosmosArray cosmosArray = CosmosArray.Create(jsonNavigator, jsonNavigatorNode);
Headers headers = new Headers();
headers.ContinuationToken = continuationToken;
headers.ActivityId = Guid.NewGuid().ToString();
QueryResponse message = QueryResponse.CreateSuccess(
result: cosmosArray,
count: items.Count,
responseHeaders: CosmosQueryResponseMessageHeaders.ConvertToQueryHeaders(headers, ResourceType.Document, collectionRid),
responseLengthBytes: responseLengthBytes);
return(message);
}
private static void ExecuteNavigatorBenchmark(
CurratedDocsPayload payload,
SerializationFormat sourceFormat,
SerializationFormat destinationFormat)
{
IJsonNavigator navigator = sourceFormat switch
{
SerializationFormat.Text => JsonNavigator.Create(payload.Text),
SerializationFormat.Binary => JsonNavigator.Create(payload.Binary),
_ => throw new ArgumentException($"Unexpected {nameof(sourceFormat)} of type: '{sourceFormat}'"),
};
IJsonWriter writer = destinationFormat switch
{
SerializationFormat.Text => JsonWriter.Create(JsonSerializationFormat.Text),
SerializationFormat.Binary => JsonWriter.Create(JsonSerializationFormat.Binary),
_ => throw new ArgumentException($"Unexpected {nameof(destinationFormat)} of type: {destinationFormat}"),
};
writer.WriteJsonNode(navigator, navigator.GetRootNode());
}
private void VerifyNavigator(string input, Exception expectedException, bool performExtraChecks = true)
{
IJsonReader jsonReader = JsonReader.Create(Encoding.UTF8.GetBytes(input));
JsonTokenInfo[] tokensFromReader = JsonNavigatorTests.GetTokensWithReader(jsonReader);
// Test text
IJsonNavigator textNavigator = JsonNavigator.Create(Encoding.UTF8.GetBytes(input));
IJsonNavigatorNode textRootNode = textNavigator.GetRootNode();
JsonTokenInfo[] tokensFromTextNavigator = JsonNavigatorTests.GetTokensFromNode(textRootNode, textNavigator, performExtraChecks);
Assert.IsTrue(tokensFromTextNavigator.SequenceEqual(tokensFromReader));
// Test binary
byte[] binaryInput = JsonTestUtils.ConvertTextToBinary(input);
IJsonNavigator binaryNavigator = JsonNavigator.Create(binaryInput);
IJsonNavigatorNode binaryRootNode = binaryNavigator.GetRootNode();
JsonTokenInfo[] tokensFromBinaryNavigator = JsonNavigatorTests.GetTokensFromNode(binaryRootNode, binaryNavigator, performExtraChecks);
Assert.IsTrue(tokensFromBinaryNavigator.SequenceEqual(tokensFromReader));
// Test binary + user string encoding
JsonStringDictionary jsonStringDictionary = new JsonStringDictionary(capacity: 4096);
byte[] binaryWithUserStringEncodingInput = JsonTestUtils.ConvertTextToBinary(input, jsonStringDictionary);
if (jsonStringDictionary.TryGetStringAtIndex(index: 0, value: out string temp))
{
Assert.IsFalse(binaryWithUserStringEncodingInput.SequenceEqual(binaryInput), "Binary should be different with user string encoding");
}
IJsonNavigator binaryNavigatorWithUserStringEncoding = JsonNavigator.Create(binaryInput, jsonStringDictionary);
IJsonNavigatorNode binaryRootNodeWithUserStringEncoding = binaryNavigatorWithUserStringEncoding.GetRootNode();
JsonTokenInfo[] tokensFromBinaryNavigatorWithUserStringEncoding = JsonNavigatorTests.GetTokensFromNode(binaryRootNode, binaryNavigator, performExtraChecks);
Assert.IsTrue(tokensFromBinaryNavigatorWithUserStringEncoding.SequenceEqual(tokensFromReader));
}
private void VerifyNavigator(string input, Exception expectedException, bool performExtraChecks = true)
{
IJsonReader jsonReader = JsonReader.Create(Encoding.UTF8.GetBytes(input));
JsonTokenInfo[] tokensFromReader = JsonNavigatorTests.GetTokensWithReader(jsonReader);
// Test text
IJsonNavigator textNavigator = JsonNavigator.Create(Encoding.UTF8.GetBytes(input));
IJsonNavigatorNode textRootNode = textNavigator.GetRootNode();
JsonTokenInfo[] tokensFromTextNavigator = JsonNavigatorTests.GetTokensFromNode(textRootNode, textNavigator, performExtraChecks);
Assert.IsTrue(tokensFromTextNavigator.SequenceEqual(tokensFromReader));
// Test binary
byte[] binaryInput = JsonTestUtils.ConvertTextToBinary(input);
IJsonNavigator binaryNavigator = JsonNavigator.Create(binaryInput);
IJsonNavigatorNode binaryRootNode = binaryNavigator.GetRootNode();
JsonTokenInfo[] tokensFromBinaryNavigator = JsonNavigatorTests.GetTokensFromNode(binaryRootNode, binaryNavigator, performExtraChecks);
Assert.IsTrue(tokensFromBinaryNavigator.SequenceEqual(tokensFromReader));
}
/// <summary>
/// Converts a list of CosmosElements into a memory stream.
/// </summary>
/// <param name="memoryStream">The memory stream response from Azure Cosmos</param>
/// <param name="resourceType">The resource type</param>
/// <param name="cosmosSerializationOptions">The custom serialization options. This allows custom serialization types like BSON, JSON, or other formats</param>
/// <returns>Returns a memory stream of cosmos elements. By default the memory stream will contain JSON.</returns>
internal static CosmosArray ToCosmosElements(
MemoryStream memoryStream,
ResourceType resourceType,
CosmosSerializationFormatOptions cosmosSerializationOptions = null)
{
if (!memoryStream.CanRead)
{
throw new InvalidDataException("Stream can not be read");
}
// Execute the callback an each element of the page
// For example just could get a response like this
// {
// "_rid": "qHVdAImeKAQ=",
// "Documents": [{
// "id": "03230",
// "_rid": "qHVdAImeKAQBAAAAAAAAAA==",
// "_self": "dbs\/qHVdAA==\/colls\/qHVdAImeKAQ=\/docs\/qHVdAImeKAQBAAAAAAAAAA==\/",
// "_etag": "\"410000b0-0000-0000-0000-597916b00000\"",
// "_attachments": "attachments\/",
// "_ts": 1501107886
// }],
// "_count": 1
// }
// And you should execute the callback on each document in "Documents".
long responseLengthBytes = memoryStream.Length;
ReadOnlyMemory <byte> content;
if (memoryStream.TryGetBuffer(out ArraySegment <byte> buffer))
{
content = buffer;
}
else
{
content = memoryStream.ToArray();
}
IJsonNavigator jsonNavigator;
// Use the users custom navigator
if (cosmosSerializationOptions != null)
{
jsonNavigator = cosmosSerializationOptions.CreateCustomNavigatorCallback(content);
if (jsonNavigator == null)
{
throw new InvalidOperationException("The CosmosSerializationOptions did not return a JSON navigator.");
}
}
else
{
jsonNavigator = JsonNavigator.Create(content);
}
string resourceName = CosmosElementSerializer.GetRootNodeName(resourceType);
CosmosArray documents;
if ((jsonNavigator.SerializationFormat == JsonSerializationFormat.Binary) && jsonNavigator.TryGetObjectProperty(
jsonNavigator.GetRootNode(),
"stringDictionary",
out ObjectProperty stringDictionaryProperty))
{
// Payload is string dictionary encode so we have to decode using the string dictionary.
IJsonNavigatorNode stringDictionaryNode = stringDictionaryProperty.ValueNode;
JsonStringDictionary jsonStringDictionary = JsonStringDictionary.CreateFromStringArray(
jsonNavigator
.GetArrayItems(stringDictionaryNode)
.Select(item => jsonNavigator.GetStringValue(item))
.ToList());
if (!jsonNavigator.TryGetObjectProperty(
jsonNavigator.GetRootNode(),
resourceName,
out ObjectProperty resourceProperty))
{
throw new InvalidOperationException($"Response Body Contract was violated. QueryResponse did not have property: {resourceName}");
}
IJsonNavigatorNode resources = resourceProperty.ValueNode;
if (!jsonNavigator.TryGetBufferedBinaryValue(resources, out ReadOnlyMemory <byte> resourceBinary))
{
resourceBinary = jsonNavigator.GetBinaryValue(resources);
}
IJsonNavigator navigatorWithStringDictionary = JsonNavigator.Create(resourceBinary, jsonStringDictionary);
if (!(CosmosElement.Dispatch(
navigatorWithStringDictionary,
navigatorWithStringDictionary.GetRootNode()) is CosmosArray cosmosArray))
{
throw new InvalidOperationException($"QueryResponse did not have an array of : {resourceName}");
}
documents = cosmosArray;
}
else
{
// Payload is not string dictionary encoded so we can just do for the documents as is.
if (!jsonNavigator.TryGetObjectProperty(
jsonNavigator.GetRootNode(),
resourceName,
out ObjectProperty objectProperty))
{
throw new InvalidOperationException($"Response Body Contract was violated. QueryResponse did not have property: {resourceName}");
}
if (!(CosmosElement.Dispatch(
jsonNavigator,
objectProperty.ValueNode) is CosmosArray cosmosArray))
{
throw new InvalidOperationException($"QueryResponse did not have an array of : {resourceName}");
}
documents = cosmosArray;
}
return(documents);
}
private QueryResponse GetCosmosElementResponse(
QueryRequestOptions requestOptions,
ResourceType resourceType,
CosmosResponseMessage cosmosResponseMessage)
{
using (cosmosResponseMessage)
{
if (!cosmosResponseMessage.IsSuccessStatusCode)
{
return(QueryResponse.CreateFailure(
CosmosQueryResponseMessageHeaders.ConvertToQueryHeaders(cosmosResponseMessage.Headers),
cosmosResponseMessage.StatusCode,
cosmosResponseMessage.RequestMessage,
cosmosResponseMessage.ErrorMessage,
cosmosResponseMessage.Error));
}
// Execute the callback an each element of the page
// For example just could get a response like this
// {
// "_rid": "qHVdAImeKAQ=",
// "Documents": [{
// "id": "03230",
// "_rid": "qHVdAImeKAQBAAAAAAAAAA==",
// "_self": "dbs\/qHVdAA==\/colls\/qHVdAImeKAQ=\/docs\/qHVdAImeKAQBAAAAAAAAAA==\/",
// "_etag": "\"410000b0-0000-0000-0000-597916b00000\"",
// "_attachments": "attachments\/",
// "_ts": 1501107886
// }],
// "_count": 1
// }
// And you should execute the callback on each document in "Documents".
MemoryStream memoryStream = new MemoryStream();
cosmosResponseMessage.Content.CopyTo(memoryStream);
long responseLengthBytes = memoryStream.Length;
byte[] content = memoryStream.ToArray();
IJsonNavigator jsonNavigator = null;
// Use the users custom navigator
if (requestOptions.CosmosSerializationOptions != null)
{
jsonNavigator = requestOptions.CosmosSerializationOptions.CreateCustomNavigatorCallback(content);
if (jsonNavigator == null)
{
throw new InvalidOperationException("The CosmosSerializationOptions did not return a JSON navigator.");
}
}
else
{
jsonNavigator = JsonNavigator.Create(content);
}
string resourceName = this.GetRootNodeName(resourceType);
if (!jsonNavigator.TryGetObjectProperty(
jsonNavigator.GetRootNode(),
resourceName,
out ObjectProperty objectProperty))
{
throw new InvalidOperationException($"Response Body Contract was violated. QueryResponse did not have property: {resourceName}");
}
IJsonNavigatorNode cosmosElements = objectProperty.ValueNode;
if (!(CosmosElement.Dispatch(
jsonNavigator,
cosmosElements) is CosmosArray cosmosArray))
{
throw new InvalidOperationException($"QueryResponse did not have an array of : {resourceName}");
}
int itemCount = cosmosArray.Count;
return(QueryResponse.CreateSuccess(
result: cosmosArray,
count: itemCount,
responseHeaders: CosmosQueryResponseMessageHeaders.ConvertToQueryHeaders(cosmosResponseMessage.Headers),
responseLengthBytes: responseLengthBytes));
}
}
private DocumentFeedResponse <CosmosElement> GetFeedResponse(
DocumentServiceRequest documentServiceRequest,
DocumentServiceResponse documentServiceResponse)
{
// Execute the callback an each element of the page
// For example just could get a response like this
// {
// "_rid": "qHVdAImeKAQ=",
// "Documents": [{
// "id": "03230",
// "_rid": "qHVdAImeKAQBAAAAAAAAAA==",
// "_self": "dbs\/qHVdAA==\/colls\/qHVdAImeKAQ=\/docs\/qHVdAImeKAQBAAAAAAAAAA==\/",
// "_etag": "\"410000b0-0000-0000-0000-597916b00000\"",
// "_attachments": "attachments\/",
// "_ts": 1501107886
// }],
// "_count": 1
// }
// And you should execute the callback on each document in "Documents".
MemoryStream memoryStream = new MemoryStream();
documentServiceResponse.ResponseBody.CopyTo(memoryStream);
long responseLengthBytes = memoryStream.Length;
ReadOnlyMemory <byte> content;
if (memoryStream.TryGetBuffer(out ArraySegment <byte> buffer))
{
content = buffer;
}
else
{
content = memoryStream.ToArray();
}
IJsonNavigator jsonNavigator = null;
// Use the users custom navigator first. If it returns null back try the
// internal navigator.
if (this.feedOptions.CosmosSerializationFormatOptions != null)
{
jsonNavigator = this.feedOptions.CosmosSerializationFormatOptions.CreateCustomNavigatorCallback(content);
if (jsonNavigator == null)
{
throw new InvalidOperationException("The CosmosSerializationOptions did not return a JSON navigator.");
}
}
else
{
jsonNavigator = JsonNavigator.Create(content);
}
string resourceName = this.GetRootNodeName(documentServiceRequest.ResourceType);
if (!jsonNavigator.TryGetObjectProperty(
jsonNavigator.GetRootNode(),
resourceName,
out ObjectProperty objectProperty))
{
throw new InvalidOperationException($"Response Body Contract was violated. QueryResponse did not have property: {resourceName}");
}
IJsonNavigatorNode cosmosElements = objectProperty.ValueNode;
if (!(CosmosElement.Dispatch(
jsonNavigator,
cosmosElements) is CosmosArray cosmosArray))
{
throw new InvalidOperationException($"QueryResponse did not have an array of : {resourceName}");
}
int itemCount = cosmosArray.Count;
return(new DocumentFeedResponse <CosmosElement>(
cosmosArray,
itemCount,
documentServiceResponse.Headers,
documentServiceResponse.RequestStats,
responseLengthBytes));
}
public static async Task RewriteStreamAsTextAsync(ResponseMessage responseMessage, QueryRequestOptions requestOptions)
{
// Rewrite the payload to be in the specified format.
// If it's already in the correct format, then the following will be a memcpy.
MemoryStream memoryStream;
if (responseMessage.Content is MemoryStream responseContentAsMemoryStream)
{
memoryStream = responseContentAsMemoryStream;
}
else
{
memoryStream = new MemoryStream();
await responseMessage.Content.CopyToAsync(memoryStream);
}
ReadOnlyMemory <byte> buffer;
if (memoryStream.TryGetBuffer(out ArraySegment <byte> segment))
{
buffer = segment.Array.AsMemory().Slice(start: segment.Offset, length: segment.Count);
}
else
{
buffer = memoryStream.ToArray();
}
IJsonNavigator jsonNavigator = JsonNavigator.Create(buffer);
if (jsonNavigator.SerializationFormat == JsonSerializationFormat.Text)
{
// Exit to avoid the memory allocation.
return;
}
IJsonWriter jsonWriter;
if (requestOptions?.CosmosSerializationFormatOptions != null)
{
jsonWriter = requestOptions.CosmosSerializationFormatOptions.CreateCustomWriterCallback();
}
else
{
jsonWriter = JsonWriter.Create(JsonSerializationFormat.Text);
}
jsonNavigator.WriteTo(jsonNavigator.GetRootNode(), jsonWriter);
ReadOnlyMemory <byte> result = jsonWriter.GetResult();
MemoryStream rewrittenMemoryStream;
if (MemoryMarshal.TryGetArray(result, out ArraySegment <byte> rewrittenSegment))
{
rewrittenMemoryStream = new MemoryStream(rewrittenSegment.Array, index: rewrittenSegment.Offset, count: rewrittenSegment.Count, writable: false, publiclyVisible: true);
}
else
{
byte[] toArray = result.ToArray();
rewrittenMemoryStream = new MemoryStream(toArray, index: 0, count: toArray.Length, writable: false, publiclyVisible: true);
}
responseMessage.Content = rewrittenMemoryStream;
}
/// <summary>
/// Converts a list of CosmosElements into a memory stream.
/// </summary>
/// <param name="memoryStream">The memory stream response from Azure Cosmos</param>
/// <param name="resourceType">The resource type</param>
/// <param name="cosmosSerializationOptions">The custom serialization options. This allows custom serialization types like BSON, JSON, or other formats</param>
/// <returns>Returns a memory stream of cosmos elements. By default the memory stream will contain JSON.</returns>
internal static CosmosArray ToCosmosElements(
MemoryStream memoryStream,
ResourceType resourceType,
CosmosSerializationOptions cosmosSerializationOptions = null)
{
if (!memoryStream.CanRead)
{
throw new InvalidDataException("Stream can not be read");
}
// Execute the callback an each element of the page
// For example just could get a response like this
// {
// "_rid": "qHVdAImeKAQ=",
// "Documents": [{
// "id": "03230",
// "_rid": "qHVdAImeKAQBAAAAAAAAAA==",
// "_self": "dbs\/qHVdAA==\/colls\/qHVdAImeKAQ=\/docs\/qHVdAImeKAQBAAAAAAAAAA==\/",
// "_etag": "\"410000b0-0000-0000-0000-597916b00000\"",
// "_attachments": "attachments\/",
// "_ts": 1501107886
// }],
// "_count": 1
// }
// And you should execute the callback on each document in "Documents".
long responseLengthBytes = memoryStream.Length;
byte[] content = memoryStream.ToArray();
IJsonNavigator jsonNavigator = null;
// Use the users custom navigator
if (cosmosSerializationOptions != null)
{
jsonNavigator = cosmosSerializationOptions.CreateCustomNavigatorCallback(content);
if (jsonNavigator == null)
{
throw new InvalidOperationException("The CosmosSerializationOptions did not return a JSON navigator.");
}
}
else
{
jsonNavigator = JsonNavigator.Create(content);
}
string resourceName = CosmosElementSerializer.GetRootNodeName(resourceType);
if (!jsonNavigator.TryGetObjectProperty(
jsonNavigator.GetRootNode(),
resourceName,
out ObjectProperty objectProperty))
{
throw new InvalidOperationException($"Response Body Contract was violated. QueryResponse did not have property: {resourceName}");
}
IJsonNavigatorNode cosmosElements = objectProperty.ValueNode;
if (!(CosmosElement.Dispatch(
jsonNavigator,
cosmosElements) is CosmosArray cosmosArray))
{
throw new InvalidOperationException($"QueryResponse did not have an array of : {resourceName}");
}
return(cosmosArray);
}
private void MultiSerializationRoundTrip(string json)
{
// Normalize the json to get rid of any formatting issues
json = this.NewtonsoftFormat(json);
foreach (SerializationFormat sourceFormat in Enum.GetValues(typeof(SerializationFormat)))
{
foreach (SerializationFormat destinationFormat in Enum.GetValues(typeof(SerializationFormat)))
{
IJsonReader reader;
switch (sourceFormat)
{
case SerializationFormat.Text:
reader = JsonReader.Create(Encoding.UTF8.GetBytes(json));
break;
case SerializationFormat.Binary:
reader = JsonReader.Create(JsonTestUtils.ConvertTextToBinary(json));
break;
case SerializationFormat.NewtonsoftText:
reader = new JsonNewtonsoftNewtonsoftTextReader(json);
break;
default:
throw new ArgumentException($"Unexpected {nameof(sourceFormat)} of type: {sourceFormat}");
}
IJsonNavigator navigator;
switch (sourceFormat)
{
case SerializationFormat.Text:
navigator = JsonNavigator.Create(Encoding.UTF8.GetBytes(json));
break;
case SerializationFormat.Binary:
navigator = JsonNavigator.Create(JsonTestUtils.ConvertTextToBinary(json));
break;
case SerializationFormat.NewtonsoftText:
navigator = new JsonNewtonsoftNavigator(json);
break;
default:
throw new ArgumentException($"Unexpected {nameof(sourceFormat)} of type: {sourceFormat}");
}
object[] sources = new object[] { reader, navigator };
foreach (object source in sources)
{
IJsonWriter writer;
switch (destinationFormat)
{
case SerializationFormat.Text:
writer = JsonWriter.Create(JsonSerializationFormat.Text);
break;
case SerializationFormat.Binary:
writer = JsonWriter.Create(JsonSerializationFormat.Binary);
break;
case SerializationFormat.NewtonsoftText:
writer = new JsonNewtonsoftNewtonsoftTextWriter();
break;
default:
throw new ArgumentException($"Unexpected {nameof(destinationFormat)} of type: {destinationFormat}");
}
Stopwatch stopwatch = Stopwatch.StartNew();
if (source is IJsonReader)
{
IJsonReader sourceReader = source as IJsonReader;
writer.WriteAll(sourceReader);
}
else if (source is IJsonNavigator)
{
IJsonNavigator sourceNavigator = source as IJsonNavigator;
writer.WriteJsonNode(sourceNavigator, sourceNavigator.GetRootNode());
}
stopwatch.Stop();
string result;
switch (writer.SerializationFormat)
{
case JsonSerializationFormat.Text:
result = Encoding.UTF8.GetString(writer.GetResult());
break;
case JsonSerializationFormat.Binary:
result = JsonTestUtils.ConvertBinaryToText(writer.GetResult());
break;
default:
throw new ArgumentException();
}
result = this.NewtonsoftFormat(result);
Assert.AreEqual(json, result);
string sourceType = (source is IJsonReader) ? "Reader" : "Navigator";
Console.WriteLine($"{sourceFormat} {sourceType} to {destinationFormat} Writer took {stopwatch.ElapsedMilliseconds}ms");
}
}
}
}
