public void InputContainsSomeInvalidDataFromUtf16() { string inputStringEndsWithLow = TextEncoderTestHelper.GenerateInvalidStringEndsWithLow(TextEncoderConstants.DataLength); byte[] inputBytes = Text.Encoding.Unicode.GetBytes(inputStringEndsWithLow); ReadOnlySpan <byte> input = MemoryMarshal.AsBytes(inputStringEndsWithLow.AsSpan()); ReadOnlySpan <byte> expected = Text.Encoding.Convert(Text.Encoding.Unicode, Text.Encoding.UTF8, inputBytes); int expectedWritten = TextEncoderTestHelper.GetUtf8ByteCount(inputStringEndsWithLow.AsSpan()); Span <byte> output = new byte[expectedWritten + 10]; Assert.Equal(OperationStatus.InvalidData, Encodings.Utf16.ToUtf8(input, output, out int consumed, out int written)); Assert.True(consumed < input.Length, "Consumed too many input characters"); Assert.Equal(expectedWritten, written); Assert.True(expected.Slice(0, written).SequenceEqual(output.Slice(0, written)), "Invalid output sequence [ends with low]"); string inputStringInvalid = TextEncoderTestHelper.GenerateStringWithInvalidChars(TextEncoderConstants.DataLength); inputBytes = Text.Encoding.Unicode.GetBytes(inputStringInvalid); input = MemoryMarshal.AsBytes(inputStringInvalid.AsSpan()); expected = Text.Encoding.Convert(Text.Encoding.Unicode, Text.Encoding.UTF8, inputBytes); expectedWritten = TextEncoderTestHelper.GetUtf8ByteCount(inputStringInvalid.AsSpan()); output = new byte[expectedWritten + 10]; Assert.Equal(OperationStatus.InvalidData, Encodings.Utf16.ToUtf8(input, output, out consumed, out written)); Assert.True(consumed < input.Length, "Consumed more input than expected"); Assert.Equal(expectedWritten, written); Assert.True(expected.Slice(0, written).SequenceEqual(output.Slice(0, written)), "Invalid output sequence [invalid]"); }
public void InputContainsSomeInvalidDataFromUtf32() { uint[] codepoints = TextEncoderTestHelper.GenerateOnlyInvalidUtf32CodePoints(TextEncoderConstants.DataLength / sizeof(uint)); ReadOnlySpan <byte> input = MemoryMarshal.AsBytes(codepoints.AsSpan()); ReadOnlySpan <byte> expected = Text.Encoding.Convert(Text.Encoding.UTF32, Text.Encoding.UTF8, input.ToArray()); int expectedWritten = TextEncoderTestHelper.GetUtf8ByteCount(codepoints); Span <byte> output = new byte[expectedWritten]; Assert.Equal(OperationStatus.InvalidData, Encodings.Utf32.ToUtf8(input, output, out int consumed, out int written)); Assert.True(consumed < input.Length, "Consumed more input than expected"); Assert.Equal(expectedWritten, written); Assert.True(expected.Slice(0, expectedWritten).SequenceEqual(output)); }
//[InlineData(TextEncoderTestHelper.SupportedEncoding.FromUtf32)] // Open issue: https://github.com/dotnet/corefxlab/issues/1513 public void InputBufferEndsTooEarlyAndRestart(TextEncoderTestHelper.SupportedEncoding from) { string inputString1 = TextEncoderTestHelper.GenerateValidStringEndsWithHighStartsWithLow(TextEncoderConstants.DataLength, false); string inputString2 = inputString1 + TextEncoderTestHelper.GenerateValidStringEndsWithHighStartsWithLow(TextEncoderConstants.DataLength, true); byte[] inputUtf8Bytes1 = TextEncoderTestHelper.GenerateValidUtf8BytesEndsWithHighStartsWithLow(TextEncoderConstants.DataLength, false); byte[] tempForUtf8Bytes2 = TextEncoderTestHelper.GenerateValidUtf8BytesEndsWithHighStartsWithLow(TextEncoderConstants.DataLength, true); byte[] inputUtf8Bytes2 = new byte[inputUtf8Bytes1.Length + tempForUtf8Bytes2.Length]; Array.Copy(inputUtf8Bytes1, inputUtf8Bytes2, inputUtf8Bytes1.Length); Array.Copy(tempForUtf8Bytes2, 0, inputUtf8Bytes2, inputUtf8Bytes1.Length, tempForUtf8Bytes2.Length); uint[] inputUtf32Bytes1 = TextEncoderTestHelper.GenerateValidUtf32EndsWithHighStartsWithLow(TextEncoderConstants.DataLength, false); uint[] tempForUtf32Bytes2 = TextEncoderTestHelper.GenerateValidUtf32EndsWithHighStartsWithLow(TextEncoderConstants.DataLength, true); uint[] inputUtf32Bytes2 = new uint[inputUtf32Bytes1.Length + tempForUtf32Bytes2.Length]; Array.Copy(inputUtf32Bytes1, inputUtf32Bytes2, inputUtf32Bytes1.Length); Array.Copy(tempForUtf32Bytes2, 0, inputUtf32Bytes2, inputUtf32Bytes1.Length, tempForUtf32Bytes2.Length); byte[] uint32Bytes1 = TextEncoderTestHelper.GenerateValidBytesUtf32EndsWithHighStartsWithLow(TextEncoderConstants.DataLength, false); byte[] tempUint32Bytes = TextEncoderTestHelper.GenerateValidBytesUtf32EndsWithHighStartsWithLow(TextEncoderConstants.DataLength, true); byte[] uint32Bytes2 = new byte[uint32Bytes1.Length + tempUint32Bytes.Length]; Array.Copy(uint32Bytes1, uint32Bytes2, uint32Bytes1.Length); Array.Copy(tempUint32Bytes, 0, uint32Bytes2, uint32Bytes1.Length, tempUint32Bytes.Length); byte[] expectedBytes; Span <byte> encodedBytes; int charactersConsumed1; int charactersConsumed2; int bytesWritten1; int bytesWritten2; switch (from) { case TextEncoderTestHelper.SupportedEncoding.FromUtf8: expectedBytes = Text.Encoding.Convert(testEncoder, testEncoder, inputUtf8Bytes2); ReadOnlySpan <byte> firstUtf8 = inputUtf8Bytes1; ReadOnlySpan <byte> secondUtf8 = inputUtf8Bytes2; encodedBytes = new Span <byte>(new byte[expectedBytes.Length]); Assert.False(utf8.TryEncode(firstUtf8, encodedBytes, out charactersConsumed1, out bytesWritten1)); Assert.True(utf8.TryEncode(secondUtf8.Slice(charactersConsumed1), encodedBytes.Slice(bytesWritten1), out charactersConsumed2, out bytesWritten2)); break; case TextEncoderTestHelper.SupportedEncoding.FromUtf16: byte[] inputStringUtf16 = testEncoderUnicode.GetBytes(inputString2); expectedBytes = Text.Encoding.Convert(testEncoderUnicode, testEncoder, inputStringUtf16); ReadOnlySpan <char> firstUtf16 = inputString1.AsSpan(); ReadOnlySpan <char> secondUtf16 = inputString2.AsSpan(); encodedBytes = new Span <byte>(new byte[expectedBytes.Length]); Assert.False(utf8.TryEncode(firstUtf16, encodedBytes, out charactersConsumed1, out bytesWritten1)); Assert.True(utf8.TryEncode(secondUtf16.Slice(charactersConsumed1), encodedBytes.Slice(bytesWritten1), out charactersConsumed2, out bytesWritten2)); break; case TextEncoderTestHelper.SupportedEncoding.FromString: // Open issue: https://github.com/dotnet/corefxlab/issues/1515 /*inputStringUtf16 = testEncoderUnicode.GetBytes(inputString2); * expectedBytes = Text.Encoding.Convert(testEncoderUnicode, testEncoder, inputStringUtf16); * string firstInputStr = inputString1; * string secondInputStr = inputString2; * encodedBytes = new Span<byte>(new byte[expectedBytes.Length]); * Assert.False(utf8.TryEncode(firstInputStr, encodedBytes, out bytesWritten1)); * Assert.True(utf8.TryEncode(secondInputStr.Substring(charactersConsumed1), encodedBytes.Slice(bytesWritten1), out bytesWritten1));*/ return; case TextEncoderTestHelper.SupportedEncoding.FromUtf32: default: expectedBytes = Text.Encoding.Convert(testEncoderUtf32, testEncoder, uint32Bytes2); ReadOnlySpan <uint> firstInput = inputUtf32Bytes1.AsSpan(); ReadOnlySpan <uint> secondInput = inputUtf32Bytes2.AsSpan(); encodedBytes = new Span <byte>(new byte[TextEncoderTestHelper.GetUtf8ByteCount(inputUtf32Bytes2)]); Assert.False(utf8.TryEncode(firstInput, encodedBytes, out charactersConsumed1, out bytesWritten1)); Assert.True(utf8.TryEncode(secondInput.Slice(charactersConsumed1), encodedBytes.Slice(bytesWritten1), out charactersConsumed2, out bytesWritten2)); break; } Assert.Equal(TextEncoderConstants.DataLength * 2, charactersConsumed1 + charactersConsumed2); Assert.Equal(expectedBytes.Length, bytesWritten1 + bytesWritten2); Assert.True(expectedBytes.AsSpan().SequenceEqual(encodedBytes)); }
//[InlineData(TextEncoderTestHelper.SupportedEncoding.FromUtf32)] // Open issue: https://github.com/dotnet/corefxlab/issues/1513 public void InputBufferContainsSomeInvalidData(TextEncoderTestHelper.SupportedEncoding from) { string inputStringEndsWithLow = TextEncoderTestHelper.GenerateInvalidStringEndsWithLow(TextEncoderConstants.DataLength); string inputStringInvalid = TextEncoderTestHelper.GenerateStringWithInvalidChars(TextEncoderConstants.DataLength); byte[] inputUtf8Bytes = TextEncoderTestHelper.GenerateUtf8BytesWithInvalidBytes(TextEncoderConstants.DataLength); byte[] expectedBytes; Span <byte> encodedBytes; int expectedBytesWritten; int bytesWritten; switch (from) { case TextEncoderTestHelper.SupportedEncoding.FromUtf8: expectedBytes = Text.Encoding.Convert(testEncoder, testEncoder, inputUtf8Bytes); ReadOnlySpan <byte> inputUtf8 = inputUtf8Bytes; expectedBytesWritten = TextEncoderTestHelper.GetUtf8ByteCount(inputUtf8); encodedBytes = new Span <byte>(new byte[expectedBytesWritten]); Assert.False(utf8.TryEncode(inputUtf8, encodedBytes, out int charactersConsumed, out bytesWritten)); Assert.True(charactersConsumed < inputUtf8Bytes.Length); break; case TextEncoderTestHelper.SupportedEncoding.FromUtf16: byte[] inputStringUtf16 = testEncoderUnicode.GetBytes(inputStringEndsWithLow); expectedBytes = Text.Encoding.Convert(testEncoderUnicode, testEncoder, inputStringUtf16); ReadOnlySpan <char> inputUtf16 = inputStringEndsWithLow.AsSpan(); expectedBytesWritten = TextEncoderTestHelper.GetUtf8ByteCount(inputUtf16); encodedBytes = new Span <byte>(new byte[expectedBytesWritten + 10]); Assert.False(utf8.TryEncode(inputUtf16, encodedBytes, out charactersConsumed, out bytesWritten)); Assert.True(charactersConsumed < inputStringEndsWithLow.Length); inputStringUtf16 = testEncoderUnicode.GetBytes(inputStringInvalid); expectedBytes = Text.Encoding.Convert(testEncoderUnicode, testEncoder, inputStringUtf16); inputUtf16 = inputStringInvalid.AsSpan(); expectedBytesWritten = TextEncoderTestHelper.GetUtf8ByteCount(inputUtf16); encodedBytes = new Span <byte>(new byte[expectedBytesWritten]); Assert.False(utf8.TryEncode(inputUtf16, encodedBytes, out charactersConsumed, out bytesWritten)); Assert.True(charactersConsumed < inputStringInvalid.Length); break; case TextEncoderTestHelper.SupportedEncoding.FromString: inputStringUtf16 = testEncoderUnicode.GetBytes(inputStringEndsWithLow); expectedBytes = Text.Encoding.Convert(testEncoderUnicode, testEncoder, inputStringUtf16); string inputStr = inputStringEndsWithLow; expectedBytesWritten = TextEncoderTestHelper.GetUtf8ByteCount(inputStr); encodedBytes = new Span <byte>(new byte[expectedBytesWritten + 10]); Assert.False(utf8.TryEncode(inputStr, encodedBytes, out bytesWritten)); inputStringUtf16 = testEncoderUnicode.GetBytes(inputStringInvalid); expectedBytes = Text.Encoding.Convert(testEncoderUnicode, testEncoder, inputStringUtf16); inputStr = inputStringInvalid; expectedBytesWritten = TextEncoderTestHelper.GetUtf8ByteCount(inputStr); encodedBytes = new Span <byte>(new byte[expectedBytesWritten]); Assert.False(utf8.TryEncode(inputStr, encodedBytes, out bytesWritten)); break; case TextEncoderTestHelper.SupportedEncoding.FromUtf32: // Invalid if codePoint > 0x10FFFF default: expectedBytes = Text.Encoding.Convert(testEncoderUtf32, testEncoder, inputUtf8Bytes); ReadOnlySpan <uint> input = inputUtf8Bytes.AsSpan().NonPortableCast <byte, uint>(); expectedBytesWritten = TextEncoderTestHelper.GetUtf8ByteCount(input); encodedBytes = new Span <byte>(new byte[expectedBytesWritten]); Assert.False(utf8.TryEncode(input, encodedBytes, out charactersConsumed, out bytesWritten)); Assert.True(charactersConsumed < inputUtf8Bytes.Length); break; } Assert.Equal(expectedBytesWritten, bytesWritten); Assert.True(expectedBytes.AsSpan().Slice(0, expectedBytesWritten).SequenceEqual(encodedBytes)); }