public static byte[] ConvertTextToBinary(string text, JsonStringDictionary jsonStringDictionary = null)
{
IJsonWriter binaryWriter = JsonWriter.Create(JsonSerializationFormat.Binary, jsonStringDictionary);
IJsonReader textReader = JsonReader.Create(Encoding.UTF8.GetBytes(text));
binaryWriter.WriteAll(textReader);
return(binaryWriter.GetResult().ToArray());
}
public static string ConvertBinaryToText(byte[] binary, JsonStringDictionary jsonStringDictionary = null)
{
IJsonReader binaryReader = JsonReader.Create(binary, jsonStringDictionary);
IJsonWriter textWriter = JsonWriter.Create(JsonSerializationFormat.Text);
textWriter.WriteAll(binaryReader);
return(Encoding.UTF8.GetString(textWriter.GetResult().ToArray()));
}
private static void VerifyNavigator(
string input,
bool performExtraChecks = true)
{
CultureInfo defaultCultureInfo = System.Threading.Thread.CurrentThread.CurrentCulture;
CultureInfo[] cultureInfoList = new CultureInfo[]
{
defaultCultureInfo,
System.Globalization.CultureInfo.GetCultureInfo("fr-FR")
};
try
{
foreach (CultureInfo cultureInfo in cultureInfoList)
{
System.Threading.Thread.CurrentThread.CurrentCulture = cultureInfo;
IJsonReader jsonReader = JsonReader.Create(Encoding.UTF8.GetBytes(input));
JsonToken[] tokensFromReader = JsonNavigatorTests.GetTokensWithReader(jsonReader);
// Test text
IJsonNavigator textNavigator = JsonNavigator.Create(Encoding.UTF8.GetBytes(input));
IJsonNavigatorNode textRootNode = textNavigator.GetRootNode();
JsonToken[] tokensFromTextNavigator = JsonNavigatorTests.GetTokensFromNode(textRootNode, textNavigator, performExtraChecks);
Assert.IsTrue(tokensFromTextNavigator.SequenceEqual(tokensFromReader));
// Test binary
byte[] binaryInput = JsonTestUtils.ConvertTextToBinary(input);
IJsonNavigator binaryNavigator = JsonNavigator.Create(binaryInput);
IJsonNavigatorNode binaryRootNode = binaryNavigator.GetRootNode();
JsonToken[] tokensFromBinaryNavigator = JsonNavigatorTests.GetTokensFromNode(binaryRootNode, binaryNavigator, performExtraChecks);
Assert.IsTrue(tokensFromBinaryNavigator.SequenceEqual(tokensFromReader));
// Test binary + user string encoding
JsonStringDictionary jsonStringDictionary = new JsonStringDictionary(capacity: 4096);
byte[] binaryWithUserStringEncodingInput = JsonTestUtils.ConvertTextToBinary(input, jsonStringDictionary);
if (jsonStringDictionary.TryGetStringAtIndex(index: 0, value: out _))
{
Assert.IsFalse(binaryWithUserStringEncodingInput.SequenceEqual(binaryInput), "Binary should be different with user string encoding");
}
IJsonNavigator binaryNavigatorWithUserStringEncoding = JsonNavigator.Create(binaryInput, jsonStringDictionary);
IJsonNavigatorNode binaryRootNodeWithUserStringEncoding = binaryNavigatorWithUserStringEncoding.GetRootNode();
JsonToken[] tokensFromBinaryNavigatorWithUserStringEncoding = JsonNavigatorTests.GetTokensFromNode(binaryRootNodeWithUserStringEncoding, binaryNavigatorWithUserStringEncoding, performExtraChecks);
Assert.IsTrue(tokensFromBinaryNavigatorWithUserStringEncoding.SequenceEqual(tokensFromReader));
}
}
finally
{
System.Threading.Thread.CurrentThread.CurrentCulture = defaultCultureInfo;
}
}
public void TestDifferentLengthStrings()
{
JsonStringDictionary jsonStringDictionary = new JsonStringDictionary(capacity: 256);
for (int replicationCount = 0; replicationCount < 128; replicationCount++)
{
JsonStringDictionaryTests.AddAndValidate(
jsonStringDictionary,
expectedString: new string('a', replicationCount),
expectedIndex: replicationCount);
}
}
public void TestBasicCase()
{
JsonStringDictionary jsonStringDictionary = new JsonStringDictionary(capacity: 100);
// First new string -> index 0
JsonStringDictionaryTests.AddAndValidate(jsonStringDictionary, expectedString: "str1", expectedIndex: 0);
// Second new string -> index 1
JsonStringDictionaryTests.AddAndValidate(jsonStringDictionary, expectedString: "str2", expectedIndex: 1);
// Re adding second string -> also index 1
JsonStringDictionaryTests.AddAndValidate(jsonStringDictionary, expectedString: "str2", expectedIndex: 1);
// Adding third new string -> index 2
JsonStringDictionaryTests.AddAndValidate(jsonStringDictionary, expectedString: "str3", expectedIndex: 2);
}
private static void AddAndValidate(JsonStringDictionary jsonStringDictionary, string expectedString, int expectedIndex)
{
// Try to add the string.
if (!jsonStringDictionary.TryAddString(expectedString, out int actualIndex))
{
throw new AssertFailedException($"{nameof(JsonStringDictionary.TryAddString)}({expectedString}, out int {actualIndex}) failed.");
}
Assert.AreEqual(expectedIndex, actualIndex);
// Try to read the string back by index
if (!jsonStringDictionary.TryGetStringAtIndex(expectedIndex, out UtfAllString actualString))
{
throw new AssertFailedException($"{nameof(JsonStringDictionary.TryGetStringAtIndex)}({expectedIndex}, out string {actualString}) failed.");
}
Assert.AreEqual(expectedString, actualString.Utf16String);
}
private void VerifyNavigator(string input, Exception expectedException, bool performExtraChecks = true)
{
IJsonReader jsonReader = JsonReader.Create(Encoding.UTF8.GetBytes(input));
JsonTokenInfo[] tokensFromReader = JsonNavigatorTests.GetTokensWithReader(jsonReader);
// Test text
IJsonNavigator textNavigator = JsonNavigator.Create(Encoding.UTF8.GetBytes(input));
IJsonNavigatorNode textRootNode = textNavigator.GetRootNode();
JsonTokenInfo[] tokensFromTextNavigator = JsonNavigatorTests.GetTokensFromNode(textRootNode, textNavigator, performExtraChecks);
Assert.IsTrue(tokensFromTextNavigator.SequenceEqual(tokensFromReader));
// Test binary
byte[] binaryInput = JsonTestUtils.ConvertTextToBinary(input);
IJsonNavigator binaryNavigator = JsonNavigator.Create(binaryInput);
IJsonNavigatorNode binaryRootNode = binaryNavigator.GetRootNode();
JsonTokenInfo[] tokensFromBinaryNavigator = JsonNavigatorTests.GetTokensFromNode(binaryRootNode, binaryNavigator, performExtraChecks);
Assert.IsTrue(tokensFromBinaryNavigator.SequenceEqual(tokensFromReader));
// Test binary + user string encoding
JsonStringDictionary jsonStringDictionary = new JsonStringDictionary(capacity: 4096);
byte[] binaryWithUserStringEncodingInput = JsonTestUtils.ConvertTextToBinary(input, jsonStringDictionary);
if (jsonStringDictionary.TryGetStringAtIndex(index: 0, value: out string temp))
{
Assert.IsFalse(binaryWithUserStringEncodingInput.SequenceEqual(binaryInput), "Binary should be different with user string encoding");
}
IJsonNavigator binaryNavigatorWithUserStringEncoding = JsonNavigator.Create(binaryInput, jsonStringDictionary);
IJsonNavigatorNode binaryRootNodeWithUserStringEncoding = binaryNavigatorWithUserStringEncoding.GetRootNode();
JsonTokenInfo[] tokensFromBinaryNavigatorWithUserStringEncoding = JsonNavigatorTests.GetTokensFromNode(binaryRootNode, binaryNavigator, performExtraChecks);
Assert.IsTrue(tokensFromBinaryNavigatorWithUserStringEncoding.SequenceEqual(tokensFromReader));
}
/// <summary>
/// Converts a list of CosmosElements into a memory stream.
/// </summary>
/// <param name="memoryStream">The memory stream response from Azure Cosmos</param>
/// <param name="resourceType">The resource type</param>
/// <param name="cosmosSerializationOptions">The custom serialization options. This allows custom serialization types like BSON, JSON, or other formats</param>
/// <returns>Returns a memory stream of cosmos elements. By default the memory stream will contain JSON.</returns>
internal static CosmosArray ToCosmosElements(
MemoryStream memoryStream,
ResourceType resourceType,
CosmosSerializationFormatOptions cosmosSerializationOptions = null)
{
if (!memoryStream.CanRead)
{
throw new InvalidDataException("Stream can not be read");
}
// Execute the callback an each element of the page
// For example just could get a response like this
// {
// "_rid": "qHVdAImeKAQ=",
// "Documents": [{
// "id": "03230",
// "_rid": "qHVdAImeKAQBAAAAAAAAAA==",
// "_self": "dbs\/qHVdAA==\/colls\/qHVdAImeKAQ=\/docs\/qHVdAImeKAQBAAAAAAAAAA==\/",
// "_etag": "\"410000b0-0000-0000-0000-597916b00000\"",
// "_attachments": "attachments\/",
// "_ts": 1501107886
// }],
// "_count": 1
// }
// And you should execute the callback on each document in "Documents".
long responseLengthBytes = memoryStream.Length;
ReadOnlyMemory <byte> content;
if (memoryStream.TryGetBuffer(out ArraySegment <byte> buffer))
{
content = buffer;
}
else
{
content = memoryStream.ToArray();
}
IJsonNavigator jsonNavigator;
// Use the users custom navigator
if (cosmosSerializationOptions != null)
{
jsonNavigator = cosmosSerializationOptions.CreateCustomNavigatorCallback(content);
if (jsonNavigator == null)
{
throw new InvalidOperationException("The CosmosSerializationOptions did not return a JSON navigator.");
}
}
else
{
jsonNavigator = JsonNavigator.Create(content);
}
string resourceName = CosmosElementSerializer.GetRootNodeName(resourceType);
CosmosArray documents;
if ((jsonNavigator.SerializationFormat == JsonSerializationFormat.Binary) && jsonNavigator.TryGetObjectProperty(
jsonNavigator.GetRootNode(),
"stringDictionary",
out ObjectProperty stringDictionaryProperty))
{
// Payload is string dictionary encode so we have to decode using the string dictionary.
IJsonNavigatorNode stringDictionaryNode = stringDictionaryProperty.ValueNode;
JsonStringDictionary jsonStringDictionary = JsonStringDictionary.CreateFromStringArray(
jsonNavigator
.GetArrayItems(stringDictionaryNode)
.Select(item => jsonNavigator.GetStringValue(item))
.ToList());
if (!jsonNavigator.TryGetObjectProperty(
jsonNavigator.GetRootNode(),
resourceName,
out ObjectProperty resourceProperty))
{
throw new InvalidOperationException($"Response Body Contract was violated. QueryResponse did not have property: {resourceName}");
}
IJsonNavigatorNode resources = resourceProperty.ValueNode;
if (!jsonNavigator.TryGetBufferedBinaryValue(resources, out ReadOnlyMemory <byte> resourceBinary))
{
resourceBinary = jsonNavigator.GetBinaryValue(resources);
}
IJsonNavigator navigatorWithStringDictionary = JsonNavigator.Create(resourceBinary, jsonStringDictionary);
if (!(CosmosElement.Dispatch(
navigatorWithStringDictionary,
navigatorWithStringDictionary.GetRootNode()) is CosmosArray cosmosArray))
{
throw new InvalidOperationException($"QueryResponse did not have an array of : {resourceName}");
}
documents = cosmosArray;
}
else
{
// Payload is not string dictionary encoded so we can just do for the documents as is.
if (!jsonNavigator.TryGetObjectProperty(
jsonNavigator.GetRootNode(),
resourceName,
out ObjectProperty objectProperty))
{
throw new InvalidOperationException($"Response Body Contract was violated. QueryResponse did not have property: {resourceName}");
}
if (!(CosmosElement.Dispatch(
jsonNavigator,
objectProperty.ValueNode) is CosmosArray cosmosArray))
{
throw new InvalidOperationException($"QueryResponse did not have an array of : {resourceName}");
}
documents = cosmosArray;
}
return(documents);
}
private void MultiSerializationRoundTrip(string json)
{
// Normalize the json to get rid of any formatting issues
json = this.NewtonsoftFormat(json);
foreach (SerializationFormat sourceFormat in Enum.GetValues(typeof(SerializationFormat)))
{
foreach (SerializationFormat destinationFormat in Enum.GetValues(typeof(SerializationFormat)))
{
IJsonReader reader;
switch (sourceFormat)
{
case SerializationFormat.Text:
reader = JsonReader.Create(Encoding.UTF8.GetBytes(json));
break;
case SerializationFormat.Binary:
reader = JsonReader.Create(JsonTestUtils.ConvertTextToBinary(json));
break;
case SerializationFormat.NewtonsoftText:
reader = NewtonsoftToCosmosDBReader.CreateFromString(json);
break;
case SerializationFormat.BinaryWithDictionaryEncoding:
JsonStringDictionary jsonStringDictionary = new JsonStringDictionary(capacity: 128);
reader = JsonReader.Create(JsonTestUtils.ConvertTextToBinary(json, jsonStringDictionary), jsonStringDictionary);
break;
default:
throw new ArgumentException($"Unexpected {nameof(sourceFormat)} of type: {sourceFormat}");
}
IJsonNavigator navigator;
switch (sourceFormat)
{
case SerializationFormat.Text:
navigator = JsonNavigator.Create(Encoding.UTF8.GetBytes(json));
break;
case SerializationFormat.Binary:
navigator = JsonNavigator.Create(JsonTestUtils.ConvertTextToBinary(json));
break;
case SerializationFormat.NewtonsoftText:
navigator = new JsonNewtonsoftNavigator(json);
break;
case SerializationFormat.BinaryWithDictionaryEncoding:
JsonStringDictionary jsonStringDictionary = new JsonStringDictionary(capacity: 128);
navigator = JsonNavigator.Create(JsonTestUtils.ConvertTextToBinary(json, jsonStringDictionary), jsonStringDictionary);
break;
default:
throw new ArgumentException($"Unexpected {nameof(sourceFormat)} of type: {sourceFormat}");
}
object[] sources = new object[] { reader, navigator };
foreach (object source in sources)
{
IJsonWriter writer;
JsonStringDictionary jsonStringDictionary;
switch (destinationFormat)
{
case SerializationFormat.Text:
writer = JsonWriter.Create(JsonSerializationFormat.Text);
jsonStringDictionary = null;
break;
case SerializationFormat.Binary:
writer = JsonWriter.Create(JsonSerializationFormat.Binary);
jsonStringDictionary = null;
break;
case SerializationFormat.NewtonsoftText:
writer = NewtonsoftToCosmosDBWriter.CreateTextWriter();
jsonStringDictionary = null;
break;
case SerializationFormat.BinaryWithDictionaryEncoding:
jsonStringDictionary = new JsonStringDictionary(capacity: 128);
writer = JsonWriter.Create(JsonSerializationFormat.Binary, jsonStringDictionary);
break;
default:
throw new ArgumentException($"Unexpected {nameof(destinationFormat)} of type: {destinationFormat}");
}
Stopwatch stopwatch = Stopwatch.StartNew();
switch (source)
{
case IJsonReader sourceReader:
writer.WriteAll(sourceReader);
break;
case IJsonNavigator sourceNavigator:
writer.WriteJsonNode(sourceNavigator, sourceNavigator.GetRootNode());
break;
default:
Assert.Fail("Failed to downcast source type.");
break;
}
stopwatch.Stop();
string result;
switch (writer.SerializationFormat)
{
case JsonSerializationFormat.Text:
result = Encoding.UTF8.GetString(writer.GetResult().ToArray());
break;
case JsonSerializationFormat.Binary:
result = JsonTestUtils.ConvertBinaryToText(writer.GetResult().ToArray(), jsonStringDictionary);
break;
default:
throw new ArgumentException();
}
result = this.NewtonsoftFormat(result);
Assert.AreEqual(json, result);
string sourceType = (source is IJsonReader) ? "Reader" : "Navigator";
Console.WriteLine($"{sourceFormat} {sourceType} to {destinationFormat} Writer took {stopwatch.ElapsedMilliseconds}ms");
}
}
}
}
private static void VerifyNavigator(
string input,
bool performExtraChecks = true)
{
CultureInfo defaultCultureInfo = System.Threading.Thread.CurrentThread.CurrentCulture;
CultureInfo[] cultureInfoList = new CultureInfo[]
{
defaultCultureInfo,
System.Globalization.CultureInfo.GetCultureInfo("fr-FR")
};
try
{
foreach (CultureInfo cultureInfo in cultureInfoList)
{
System.Threading.Thread.CurrentThread.CurrentCulture = cultureInfo;
IJsonReader jsonReader = JsonReader.Create(Encoding.UTF8.GetBytes(input));
JsonToken[] tokensFromReader = JsonNavigatorTests.GetTokensWithReader(jsonReader);
// Text
IJsonNavigator textNavigator = JsonNavigator.Create(Encoding.UTF8.GetBytes(input));
// Binary
byte[] binaryInput = JsonTestUtils.ConvertTextToBinary(input);
IJsonNavigator binaryNavigator = JsonNavigator.Create(binaryInput);
// Test binary + user string encoding
JsonStringDictionary jsonStringDictionary = new JsonStringDictionary(capacity: 4096);
byte[] binaryWithUserStringEncodingInput = JsonTestUtils.ConvertTextToBinary(input, jsonStringDictionary);
if (jsonStringDictionary.TryGetStringAtIndex(index: 0, value: out _))
{
Assert.IsFalse(binaryWithUserStringEncodingInput.SequenceEqual(binaryInput), "Binary should be different with user string encoding");
}
IJsonNavigator binaryNavigatorWithUserStringEncoding = JsonNavigator.Create(binaryInput, jsonStringDictionary);
// Test
foreach (IJsonNavigator jsonNavigator in new IJsonNavigator[] { textNavigator, binaryNavigator, binaryNavigatorWithUserStringEncoding })
{
IJsonNavigatorNode rootNode = jsonNavigator.GetRootNode();
JsonToken[] tokensFromNavigator = JsonNavigatorTests.GetTokensFromNode(rootNode, jsonNavigator, performExtraChecks);
Assert.IsTrue(tokensFromNavigator.SequenceEqual(tokensFromReader));
// Test materialize
JToken materializedToken = CosmosElement.Dispatch(jsonNavigator, rootNode).Materialize <JToken>();
try
{
if (materializedToken.ToString() != JToken.Parse(input).ToString())
{
throw new MaterilizationFailedToMatchException();
}
}
catch (Newtonsoft.Json.JsonReaderException)
{
// If the input is extended type we ignore this check.
}
}
}
}
finally
{
System.Threading.Thread.CurrentThread.CurrentCulture = defaultCultureInfo;
}
}
private static void PerformRoundTrip(
SerializationFormat sourceFormat,
SerializationFormat destinationFormat,
string json)
{
JsonStringDictionary sourceDictionary = null;
IJsonReader reader;
switch (sourceFormat)
{
case SerializationFormat.Text:
reader = JsonReader.Create(Encoding.UTF8.GetBytes(json));
break;
case SerializationFormat.Binary:
reader = JsonReader.Create(JsonTestUtils.ConvertTextToBinary(json));
break;
case SerializationFormat.NewtonsoftText:
reader = NewtonsoftToCosmosDBReader.CreateFromString(json);
break;
case SerializationFormat.BinaryWithDictionaryEncoding:
sourceDictionary = new JsonStringDictionary(capacity: 128);
reader = JsonReader.Create(JsonTestUtils.ConvertTextToBinary(json, sourceDictionary), sourceDictionary);
break;
default:
throw new ArgumentException($"Unexpected {nameof(sourceFormat)} of type: {sourceFormat}");
}
IJsonNavigator navigator;
switch (sourceFormat)
{
case SerializationFormat.Text:
navigator = JsonNavigator.Create(Encoding.UTF8.GetBytes(json));
break;
case SerializationFormat.Binary:
navigator = JsonNavigator.Create(JsonTestUtils.ConvertTextToBinary(json));
break;
case SerializationFormat.NewtonsoftText:
navigator = new JsonNewtonsoftNavigator(json);
break;
case SerializationFormat.BinaryWithDictionaryEncoding:
sourceDictionary = new JsonStringDictionary(capacity: 128);
navigator = JsonNavigator.Create(JsonTestUtils.ConvertTextToBinary(json, sourceDictionary), sourceDictionary);
break;
default:
throw new ArgumentException($"Unexpected {nameof(sourceFormat)} of type: {sourceFormat}");
}
object[] sources = new object[] { reader, navigator };
foreach (object source in sources)
{
IJsonWriter writer;
JsonStringDictionary jsonStringDictionary;
switch (destinationFormat)
{
case SerializationFormat.Text:
writer = JsonWriter.Create(JsonSerializationFormat.Text);
jsonStringDictionary = null;
break;
case SerializationFormat.Binary:
writer = JsonWriter.Create(JsonSerializationFormat.Binary);
jsonStringDictionary = null;
break;
case SerializationFormat.NewtonsoftText:
writer = NewtonsoftToCosmosDBWriter.CreateTextWriter();
jsonStringDictionary = null;
break;
case SerializationFormat.BinaryWithDictionaryEncoding:
jsonStringDictionary = new JsonStringDictionary(capacity: 128);
if (sourceFormat == SerializationFormat.BinaryWithDictionaryEncoding)
{
int index = 0;
while (sourceDictionary.TryGetStringAtIndex(index++, out UtfAllString value))
{
Assert.IsTrue(jsonStringDictionary.TryAddString(value.Utf16String, out _));
}
}
writer = JsonWriter.Create(JsonSerializationFormat.Binary, jsonStringDictionary);
break;
default:
throw new ArgumentException($"Unexpected {nameof(destinationFormat)} of type: {destinationFormat}");
}
switch (source)
{
case IJsonReader sourceReader:
sourceReader.WriteAll(writer);
break;
case IJsonNavigator sourceNavigator:
sourceNavigator.WriteNode(sourceNavigator.GetRootNode(), writer);
break;
default:
Assert.Fail("Failed to downcast source type.");
break;
}
string result;
switch (writer.SerializationFormat)
{
case JsonSerializationFormat.Text:
result = Utf8String.UnsafeFromUtf8BytesNoValidation(writer.GetResult()).ToString();
break;
case JsonSerializationFormat.Binary:
result = JsonTestUtils.ConvertBinaryToText(writer.GetResult(), jsonStringDictionary);
break;
default:
throw new ArgumentException();
}
string normalizedResult = JsonRoundTripsTests.NewtonsoftFormat(result);
string normalizedJson = JsonRoundTripsTests.NewtonsoftFormat(json);
Assert.AreEqual(normalizedJson, normalizedResult);
}
}
public void TestDictionarySizeLimit()
{
JsonStringDictionary jsonStringDictionary = new JsonStringDictionary(capacity: 0);
Assert.IsFalse(jsonStringDictionary.TryAddString("hello", out int index));
}
/// <summary>
/// Initializes a new instance of the NewtonsoftReader class.
/// </summary>
/// <param name="buffer">The buffer to read from.</param>
/// <param name="jsonStringDictionary">The json string dictionary to use.</param>
public CosmosDBToNewtonsoftReader(
ReadOnlyMemory <byte> buffer,
JsonStringDictionary jsonStringDictionary = null)
{
this.jsonReader = Microsoft.Azure.Cosmos.Json.JsonReader.Create(buffer, jsonStringDictionary);
}
