public static byte[] ConvertTextToBinary(string text, JsonStringDictionary jsonStringDictionary = null) { IJsonWriter binaryWriter = JsonWriter.Create(JsonSerializationFormat.Binary, jsonStringDictionary); IJsonReader textReader = JsonReader.Create(Encoding.UTF8.GetBytes(text)); binaryWriter.WriteAll(textReader); return(binaryWriter.GetResult().ToArray()); }
public static string ConvertBinaryToText(byte[] binary, JsonStringDictionary jsonStringDictionary = null) { IJsonReader binaryReader = JsonReader.Create(binary, jsonStringDictionary); IJsonWriter textWriter = JsonWriter.Create(JsonSerializationFormat.Text); textWriter.WriteAll(binaryReader); return(Encoding.UTF8.GetString(textWriter.GetResult().ToArray())); }
private static void VerifyNavigator( string input, bool performExtraChecks = true) { CultureInfo defaultCultureInfo = System.Threading.Thread.CurrentThread.CurrentCulture; CultureInfo[] cultureInfoList = new CultureInfo[] { defaultCultureInfo, System.Globalization.CultureInfo.GetCultureInfo("fr-FR") }; try { foreach (CultureInfo cultureInfo in cultureInfoList) { System.Threading.Thread.CurrentThread.CurrentCulture = cultureInfo; IJsonReader jsonReader = JsonReader.Create(Encoding.UTF8.GetBytes(input)); JsonToken[] tokensFromReader = JsonNavigatorTests.GetTokensWithReader(jsonReader); // Test text IJsonNavigator textNavigator = JsonNavigator.Create(Encoding.UTF8.GetBytes(input)); IJsonNavigatorNode textRootNode = textNavigator.GetRootNode(); JsonToken[] tokensFromTextNavigator = JsonNavigatorTests.GetTokensFromNode(textRootNode, textNavigator, performExtraChecks); Assert.IsTrue(tokensFromTextNavigator.SequenceEqual(tokensFromReader)); // Test binary byte[] binaryInput = JsonTestUtils.ConvertTextToBinary(input); IJsonNavigator binaryNavigator = JsonNavigator.Create(binaryInput); IJsonNavigatorNode binaryRootNode = binaryNavigator.GetRootNode(); JsonToken[] tokensFromBinaryNavigator = JsonNavigatorTests.GetTokensFromNode(binaryRootNode, binaryNavigator, performExtraChecks); Assert.IsTrue(tokensFromBinaryNavigator.SequenceEqual(tokensFromReader)); // Test binary + user string encoding JsonStringDictionary jsonStringDictionary = new JsonStringDictionary(capacity: 4096); byte[] binaryWithUserStringEncodingInput = JsonTestUtils.ConvertTextToBinary(input, jsonStringDictionary); if (jsonStringDictionary.TryGetStringAtIndex(index: 0, value: out _)) { Assert.IsFalse(binaryWithUserStringEncodingInput.SequenceEqual(binaryInput), "Binary should be different with user string encoding"); } IJsonNavigator binaryNavigatorWithUserStringEncoding = JsonNavigator.Create(binaryInput, jsonStringDictionary); IJsonNavigatorNode binaryRootNodeWithUserStringEncoding = binaryNavigatorWithUserStringEncoding.GetRootNode(); JsonToken[] tokensFromBinaryNavigatorWithUserStringEncoding = JsonNavigatorTests.GetTokensFromNode(binaryRootNodeWithUserStringEncoding, binaryNavigatorWithUserStringEncoding, performExtraChecks); Assert.IsTrue(tokensFromBinaryNavigatorWithUserStringEncoding.SequenceEqual(tokensFromReader)); } } finally { System.Threading.Thread.CurrentThread.CurrentCulture = defaultCultureInfo; } }
public void TestDifferentLengthStrings() { JsonStringDictionary jsonStringDictionary = new JsonStringDictionary(capacity: 256); for (int replicationCount = 0; replicationCount < 128; replicationCount++) { JsonStringDictionaryTests.AddAndValidate( jsonStringDictionary, expectedString: new string('a', replicationCount), expectedIndex: replicationCount); } }
public void TestBasicCase() { JsonStringDictionary jsonStringDictionary = new JsonStringDictionary(capacity: 100); // First new string -> index 0 JsonStringDictionaryTests.AddAndValidate(jsonStringDictionary, expectedString: "str1", expectedIndex: 0); // Second new string -> index 1 JsonStringDictionaryTests.AddAndValidate(jsonStringDictionary, expectedString: "str2", expectedIndex: 1); // Re adding second string -> also index 1 JsonStringDictionaryTests.AddAndValidate(jsonStringDictionary, expectedString: "str2", expectedIndex: 1); // Adding third new string -> index 2 JsonStringDictionaryTests.AddAndValidate(jsonStringDictionary, expectedString: "str3", expectedIndex: 2); }
private static void AddAndValidate(JsonStringDictionary jsonStringDictionary, string expectedString, int expectedIndex) { // Try to add the string. if (!jsonStringDictionary.TryAddString(expectedString, out int actualIndex)) { throw new AssertFailedException($"{nameof(JsonStringDictionary.TryAddString)}({expectedString}, out int {actualIndex}) failed."); } Assert.AreEqual(expectedIndex, actualIndex); // Try to read the string back by index if (!jsonStringDictionary.TryGetStringAtIndex(expectedIndex, out UtfAllString actualString)) { throw new AssertFailedException($"{nameof(JsonStringDictionary.TryGetStringAtIndex)}({expectedIndex}, out string {actualString}) failed."); } Assert.AreEqual(expectedString, actualString.Utf16String); }
private void VerifyNavigator(string input, Exception expectedException, bool performExtraChecks = true) { IJsonReader jsonReader = JsonReader.Create(Encoding.UTF8.GetBytes(input)); JsonTokenInfo[] tokensFromReader = JsonNavigatorTests.GetTokensWithReader(jsonReader); // Test text IJsonNavigator textNavigator = JsonNavigator.Create(Encoding.UTF8.GetBytes(input)); IJsonNavigatorNode textRootNode = textNavigator.GetRootNode(); JsonTokenInfo[] tokensFromTextNavigator = JsonNavigatorTests.GetTokensFromNode(textRootNode, textNavigator, performExtraChecks); Assert.IsTrue(tokensFromTextNavigator.SequenceEqual(tokensFromReader)); // Test binary byte[] binaryInput = JsonTestUtils.ConvertTextToBinary(input); IJsonNavigator binaryNavigator = JsonNavigator.Create(binaryInput); IJsonNavigatorNode binaryRootNode = binaryNavigator.GetRootNode(); JsonTokenInfo[] tokensFromBinaryNavigator = JsonNavigatorTests.GetTokensFromNode(binaryRootNode, binaryNavigator, performExtraChecks); Assert.IsTrue(tokensFromBinaryNavigator.SequenceEqual(tokensFromReader)); // Test binary + user string encoding JsonStringDictionary jsonStringDictionary = new JsonStringDictionary(capacity: 4096); byte[] binaryWithUserStringEncodingInput = JsonTestUtils.ConvertTextToBinary(input, jsonStringDictionary); if (jsonStringDictionary.TryGetStringAtIndex(index: 0, value: out string temp)) { Assert.IsFalse(binaryWithUserStringEncodingInput.SequenceEqual(binaryInput), "Binary should be different with user string encoding"); } IJsonNavigator binaryNavigatorWithUserStringEncoding = JsonNavigator.Create(binaryInput, jsonStringDictionary); IJsonNavigatorNode binaryRootNodeWithUserStringEncoding = binaryNavigatorWithUserStringEncoding.GetRootNode(); JsonTokenInfo[] tokensFromBinaryNavigatorWithUserStringEncoding = JsonNavigatorTests.GetTokensFromNode(binaryRootNode, binaryNavigator, performExtraChecks); Assert.IsTrue(tokensFromBinaryNavigatorWithUserStringEncoding.SequenceEqual(tokensFromReader)); }
/// <summary> /// Converts a list of CosmosElements into a memory stream. /// </summary> /// <param name="memoryStream">The memory stream response from Azure Cosmos</param> /// <param name="resourceType">The resource type</param> /// <param name="cosmosSerializationOptions">The custom serialization options. This allows custom serialization types like BSON, JSON, or other formats</param> /// <returns>Returns a memory stream of cosmos elements. By default the memory stream will contain JSON.</returns> internal static CosmosArray ToCosmosElements( MemoryStream memoryStream, ResourceType resourceType, CosmosSerializationFormatOptions cosmosSerializationOptions = null) { if (!memoryStream.CanRead) { throw new InvalidDataException("Stream can not be read"); } // Execute the callback an each element of the page // For example just could get a response like this // { // "_rid": "qHVdAImeKAQ=", // "Documents": [{ // "id": "03230", // "_rid": "qHVdAImeKAQBAAAAAAAAAA==", // "_self": "dbs\/qHVdAA==\/colls\/qHVdAImeKAQ=\/docs\/qHVdAImeKAQBAAAAAAAAAA==\/", // "_etag": "\"410000b0-0000-0000-0000-597916b00000\"", // "_attachments": "attachments\/", // "_ts": 1501107886 // }], // "_count": 1 // } // And you should execute the callback on each document in "Documents". long responseLengthBytes = memoryStream.Length; ReadOnlyMemory <byte> content; if (memoryStream.TryGetBuffer(out ArraySegment <byte> buffer)) { content = buffer; } else { content = memoryStream.ToArray(); } IJsonNavigator jsonNavigator; // Use the users custom navigator if (cosmosSerializationOptions != null) { jsonNavigator = cosmosSerializationOptions.CreateCustomNavigatorCallback(content); if (jsonNavigator == null) { throw new InvalidOperationException("The CosmosSerializationOptions did not return a JSON navigator."); } } else { jsonNavigator = JsonNavigator.Create(content); } string resourceName = CosmosElementSerializer.GetRootNodeName(resourceType); CosmosArray documents; if ((jsonNavigator.SerializationFormat == JsonSerializationFormat.Binary) && jsonNavigator.TryGetObjectProperty( jsonNavigator.GetRootNode(), "stringDictionary", out ObjectProperty stringDictionaryProperty)) { // Payload is string dictionary encode so we have to decode using the string dictionary. IJsonNavigatorNode stringDictionaryNode = stringDictionaryProperty.ValueNode; JsonStringDictionary jsonStringDictionary = JsonStringDictionary.CreateFromStringArray( jsonNavigator .GetArrayItems(stringDictionaryNode) .Select(item => jsonNavigator.GetStringValue(item)) .ToList()); if (!jsonNavigator.TryGetObjectProperty( jsonNavigator.GetRootNode(), resourceName, out ObjectProperty resourceProperty)) { throw new InvalidOperationException($"Response Body Contract was violated. QueryResponse did not have property: {resourceName}"); } IJsonNavigatorNode resources = resourceProperty.ValueNode; if (!jsonNavigator.TryGetBufferedBinaryValue(resources, out ReadOnlyMemory <byte> resourceBinary)) { resourceBinary = jsonNavigator.GetBinaryValue(resources); } IJsonNavigator navigatorWithStringDictionary = JsonNavigator.Create(resourceBinary, jsonStringDictionary); if (!(CosmosElement.Dispatch( navigatorWithStringDictionary, navigatorWithStringDictionary.GetRootNode()) is CosmosArray cosmosArray)) { throw new InvalidOperationException($"QueryResponse did not have an array of : {resourceName}"); } documents = cosmosArray; } else { // Payload is not string dictionary encoded so we can just do for the documents as is. if (!jsonNavigator.TryGetObjectProperty( jsonNavigator.GetRootNode(), resourceName, out ObjectProperty objectProperty)) { throw new InvalidOperationException($"Response Body Contract was violated. QueryResponse did not have property: {resourceName}"); } if (!(CosmosElement.Dispatch( jsonNavigator, objectProperty.ValueNode) is CosmosArray cosmosArray)) { throw new InvalidOperationException($"QueryResponse did not have an array of : {resourceName}"); } documents = cosmosArray; } return(documents); }
private void MultiSerializationRoundTrip(string json) { // Normalize the json to get rid of any formatting issues json = this.NewtonsoftFormat(json); foreach (SerializationFormat sourceFormat in Enum.GetValues(typeof(SerializationFormat))) { foreach (SerializationFormat destinationFormat in Enum.GetValues(typeof(SerializationFormat))) { IJsonReader reader; switch (sourceFormat) { case SerializationFormat.Text: reader = JsonReader.Create(Encoding.UTF8.GetBytes(json)); break; case SerializationFormat.Binary: reader = JsonReader.Create(JsonTestUtils.ConvertTextToBinary(json)); break; case SerializationFormat.NewtonsoftText: reader = NewtonsoftToCosmosDBReader.CreateFromString(json); break; case SerializationFormat.BinaryWithDictionaryEncoding: JsonStringDictionary jsonStringDictionary = new JsonStringDictionary(capacity: 128); reader = JsonReader.Create(JsonTestUtils.ConvertTextToBinary(json, jsonStringDictionary), jsonStringDictionary); break; default: throw new ArgumentException($"Unexpected {nameof(sourceFormat)} of type: {sourceFormat}"); } IJsonNavigator navigator; switch (sourceFormat) { case SerializationFormat.Text: navigator = JsonNavigator.Create(Encoding.UTF8.GetBytes(json)); break; case SerializationFormat.Binary: navigator = JsonNavigator.Create(JsonTestUtils.ConvertTextToBinary(json)); break; case SerializationFormat.NewtonsoftText: navigator = new JsonNewtonsoftNavigator(json); break; case SerializationFormat.BinaryWithDictionaryEncoding: JsonStringDictionary jsonStringDictionary = new JsonStringDictionary(capacity: 128); navigator = JsonNavigator.Create(JsonTestUtils.ConvertTextToBinary(json, jsonStringDictionary), jsonStringDictionary); break; default: throw new ArgumentException($"Unexpected {nameof(sourceFormat)} of type: {sourceFormat}"); } object[] sources = new object[] { reader, navigator }; foreach (object source in sources) { IJsonWriter writer; JsonStringDictionary jsonStringDictionary; switch (destinationFormat) { case SerializationFormat.Text: writer = JsonWriter.Create(JsonSerializationFormat.Text); jsonStringDictionary = null; break; case SerializationFormat.Binary: writer = JsonWriter.Create(JsonSerializationFormat.Binary); jsonStringDictionary = null; break; case SerializationFormat.NewtonsoftText: writer = NewtonsoftToCosmosDBWriter.CreateTextWriter(); jsonStringDictionary = null; break; case SerializationFormat.BinaryWithDictionaryEncoding: jsonStringDictionary = new JsonStringDictionary(capacity: 128); writer = JsonWriter.Create(JsonSerializationFormat.Binary, jsonStringDictionary); break; default: throw new ArgumentException($"Unexpected {nameof(destinationFormat)} of type: {destinationFormat}"); } Stopwatch stopwatch = Stopwatch.StartNew(); switch (source) { case IJsonReader sourceReader: writer.WriteAll(sourceReader); break; case IJsonNavigator sourceNavigator: writer.WriteJsonNode(sourceNavigator, sourceNavigator.GetRootNode()); break; default: Assert.Fail("Failed to downcast source type."); break; } stopwatch.Stop(); string result; switch (writer.SerializationFormat) { case JsonSerializationFormat.Text: result = Encoding.UTF8.GetString(writer.GetResult().ToArray()); break; case JsonSerializationFormat.Binary: result = JsonTestUtils.ConvertBinaryToText(writer.GetResult().ToArray(), jsonStringDictionary); break; default: throw new ArgumentException(); } result = this.NewtonsoftFormat(result); Assert.AreEqual(json, result); string sourceType = (source is IJsonReader) ? "Reader" : "Navigator"; Console.WriteLine($"{sourceFormat} {sourceType} to {destinationFormat} Writer took {stopwatch.ElapsedMilliseconds}ms"); } } } }
private static void VerifyNavigator( string input, bool performExtraChecks = true) { CultureInfo defaultCultureInfo = System.Threading.Thread.CurrentThread.CurrentCulture; CultureInfo[] cultureInfoList = new CultureInfo[] { defaultCultureInfo, System.Globalization.CultureInfo.GetCultureInfo("fr-FR") }; try { foreach (CultureInfo cultureInfo in cultureInfoList) { System.Threading.Thread.CurrentThread.CurrentCulture = cultureInfo; IJsonReader jsonReader = JsonReader.Create(Encoding.UTF8.GetBytes(input)); JsonToken[] tokensFromReader = JsonNavigatorTests.GetTokensWithReader(jsonReader); // Text IJsonNavigator textNavigator = JsonNavigator.Create(Encoding.UTF8.GetBytes(input)); // Binary byte[] binaryInput = JsonTestUtils.ConvertTextToBinary(input); IJsonNavigator binaryNavigator = JsonNavigator.Create(binaryInput); // Test binary + user string encoding JsonStringDictionary jsonStringDictionary = new JsonStringDictionary(capacity: 4096); byte[] binaryWithUserStringEncodingInput = JsonTestUtils.ConvertTextToBinary(input, jsonStringDictionary); if (jsonStringDictionary.TryGetStringAtIndex(index: 0, value: out _)) { Assert.IsFalse(binaryWithUserStringEncodingInput.SequenceEqual(binaryInput), "Binary should be different with user string encoding"); } IJsonNavigator binaryNavigatorWithUserStringEncoding = JsonNavigator.Create(binaryInput, jsonStringDictionary); // Test foreach (IJsonNavigator jsonNavigator in new IJsonNavigator[] { textNavigator, binaryNavigator, binaryNavigatorWithUserStringEncoding }) { IJsonNavigatorNode rootNode = jsonNavigator.GetRootNode(); JsonToken[] tokensFromNavigator = JsonNavigatorTests.GetTokensFromNode(rootNode, jsonNavigator, performExtraChecks); Assert.IsTrue(tokensFromNavigator.SequenceEqual(tokensFromReader)); // Test materialize JToken materializedToken = CosmosElement.Dispatch(jsonNavigator, rootNode).Materialize <JToken>(); try { if (materializedToken.ToString() != JToken.Parse(input).ToString()) { throw new MaterilizationFailedToMatchException(); } } catch (Newtonsoft.Json.JsonReaderException) { // If the input is extended type we ignore this check. } } } } finally { System.Threading.Thread.CurrentThread.CurrentCulture = defaultCultureInfo; } }
private static void PerformRoundTrip( SerializationFormat sourceFormat, SerializationFormat destinationFormat, string json) { JsonStringDictionary sourceDictionary = null; IJsonReader reader; switch (sourceFormat) { case SerializationFormat.Text: reader = JsonReader.Create(Encoding.UTF8.GetBytes(json)); break; case SerializationFormat.Binary: reader = JsonReader.Create(JsonTestUtils.ConvertTextToBinary(json)); break; case SerializationFormat.NewtonsoftText: reader = NewtonsoftToCosmosDBReader.CreateFromString(json); break; case SerializationFormat.BinaryWithDictionaryEncoding: sourceDictionary = new JsonStringDictionary(capacity: 128); reader = JsonReader.Create(JsonTestUtils.ConvertTextToBinary(json, sourceDictionary), sourceDictionary); break; default: throw new ArgumentException($"Unexpected {nameof(sourceFormat)} of type: {sourceFormat}"); } IJsonNavigator navigator; switch (sourceFormat) { case SerializationFormat.Text: navigator = JsonNavigator.Create(Encoding.UTF8.GetBytes(json)); break; case SerializationFormat.Binary: navigator = JsonNavigator.Create(JsonTestUtils.ConvertTextToBinary(json)); break; case SerializationFormat.NewtonsoftText: navigator = new JsonNewtonsoftNavigator(json); break; case SerializationFormat.BinaryWithDictionaryEncoding: sourceDictionary = new JsonStringDictionary(capacity: 128); navigator = JsonNavigator.Create(JsonTestUtils.ConvertTextToBinary(json, sourceDictionary), sourceDictionary); break; default: throw new ArgumentException($"Unexpected {nameof(sourceFormat)} of type: {sourceFormat}"); } object[] sources = new object[] { reader, navigator }; foreach (object source in sources) { IJsonWriter writer; JsonStringDictionary jsonStringDictionary; switch (destinationFormat) { case SerializationFormat.Text: writer = JsonWriter.Create(JsonSerializationFormat.Text); jsonStringDictionary = null; break; case SerializationFormat.Binary: writer = JsonWriter.Create(JsonSerializationFormat.Binary); jsonStringDictionary = null; break; case SerializationFormat.NewtonsoftText: writer = NewtonsoftToCosmosDBWriter.CreateTextWriter(); jsonStringDictionary = null; break; case SerializationFormat.BinaryWithDictionaryEncoding: jsonStringDictionary = new JsonStringDictionary(capacity: 128); if (sourceFormat == SerializationFormat.BinaryWithDictionaryEncoding) { int index = 0; while (sourceDictionary.TryGetStringAtIndex(index++, out UtfAllString value)) { Assert.IsTrue(jsonStringDictionary.TryAddString(value.Utf16String, out _)); } } writer = JsonWriter.Create(JsonSerializationFormat.Binary, jsonStringDictionary); break; default: throw new ArgumentException($"Unexpected {nameof(destinationFormat)} of type: {destinationFormat}"); } switch (source) { case IJsonReader sourceReader: sourceReader.WriteAll(writer); break; case IJsonNavigator sourceNavigator: sourceNavigator.WriteNode(sourceNavigator.GetRootNode(), writer); break; default: Assert.Fail("Failed to downcast source type."); break; } string result; switch (writer.SerializationFormat) { case JsonSerializationFormat.Text: result = Utf8String.UnsafeFromUtf8BytesNoValidation(writer.GetResult()).ToString(); break; case JsonSerializationFormat.Binary: result = JsonTestUtils.ConvertBinaryToText(writer.GetResult(), jsonStringDictionary); break; default: throw new ArgumentException(); } string normalizedResult = JsonRoundTripsTests.NewtonsoftFormat(result); string normalizedJson = JsonRoundTripsTests.NewtonsoftFormat(json); Assert.AreEqual(normalizedJson, normalizedResult); } }
public void TestDictionarySizeLimit() { JsonStringDictionary jsonStringDictionary = new JsonStringDictionary(capacity: 0); Assert.IsFalse(jsonStringDictionary.TryAddString("hello", out int index)); }
/// <summary> /// Initializes a new instance of the NewtonsoftReader class. /// </summary> /// <param name="buffer">The buffer to read from.</param> /// <param name="jsonStringDictionary">The json string dictionary to use.</param> public CosmosDBToNewtonsoftReader( ReadOnlyMemory <byte> buffer, JsonStringDictionary jsonStringDictionary = null) { this.jsonReader = Microsoft.Azure.Cosmos.Json.JsonReader.Create(buffer, jsonStringDictionary); }