public static void Convert_Decoder_ReadOnlySpan_IBufferWriter()
{
Decoder decoder = Encoding.UTF8.GetDecoder();
ArrayBufferWriter <char> writer = new ArrayBufferWriter <char>();
// First, a small input with no flushing and no leftover data.
ReadOnlySpan <byte> inputData = Encoding.UTF8.GetBytes("Hello");
EncodingExtensions.Convert(decoder, inputData, writer, flush: false, out long charsUsed, out bool completed);
Assert.Equal(5, charsUsed);
Assert.True(completed);
// Then, a large input with no flushing and leftover data.
inputData = Encoding.UTF8.GetBytes(new string('x', 20_000_000)).Concat(new byte[] { 0xE0, 0xA0 }).ToArray();
EncodingExtensions.Convert(decoder, inputData, writer, flush: false, out charsUsed, out completed);
Assert.Equal(20_000_000, charsUsed);
Assert.False(completed);
// Then, a large input with flushing and leftover data (should be replaced).
inputData = new byte[] { 0x80 }.Concat(Encoding.UTF8.GetBytes(new string('x', 20_000_000))).Concat(new byte[] { 0xE0 }).ToArray();
EncodingExtensions.Convert(decoder, inputData, writer, flush: true, out charsUsed, out completed);
Assert.Equal(20_000_002, charsUsed); // 1 for leftover at beginning, 1 for replacement at end
Assert.True(completed);
// Now make sure all of the data was decoded properly.
Assert.Equal(
expected: "Hello" + new string('x', 20_000_000) + '\u0800' + new string('x', 20_000_000) + '\ufffd',
actual: writer.WrittenSpan.ToString());
}
public static void Convert_Encoder_ReadOnlySpan_IBufferWriter()
{
Encoder encoder = Encoding.UTF8.GetEncoder();
ArrayBufferWriter <byte> writer = new ArrayBufferWriter <byte>();
// First, a small input with no flushing and no leftover data.
ReadOnlySpan <char> inputData = "Hello";
EncodingExtensions.Convert(encoder, inputData, writer, flush: false, out long bytesUsed, out bool completed);
Assert.Equal(5, bytesUsed);
Assert.True(completed);
// Then, a large input with no flushing and leftover data.
inputData = new string('x', 20_000_000) + '\ud800';
EncodingExtensions.Convert(encoder, inputData, writer, flush: false, out bytesUsed, out completed);
Assert.Equal(20_000_000, bytesUsed);
Assert.False(completed);
// Then, a large input with flushing and leftover data (should be replaced).
inputData = '\udc00' + new string('x', 20_000_000) + '\ud800';
EncodingExtensions.Convert(encoder, inputData, writer, flush: true, out bytesUsed, out completed);
Assert.Equal(20_000_007, bytesUsed); // 4 for supplementary at beginning, 3 for replacement at end
Assert.True(completed);
// Now make sure all of the data was encoded properly.
// Use SequenceEqual instead of Assert.Equal for perf.
Assert.True(
Encoding.UTF8.GetBytes("Hello" + new string('x', 20_000_000) + "\U00010000" + new string('x', 20_000_000) + '\ufffd').AsSpan().SequenceEqual(writer.WrittenSpan));
}
public static void Convert_Decoder_ReadOnlySpan_IBufferWriter_ParamChecks()
{
Decoder decoder = Encoding.UTF8.GetDecoder();
IBufferWriter <char> writer = new ArrayBufferWriter <char>();
Assert.Throws <ArgumentNullException>("decoder", () => EncodingExtensions.Convert((Decoder)null, ReadOnlySpan <byte> .Empty, writer, true, out _, out _));
Assert.Throws <ArgumentNullException>("writer", () => EncodingExtensions.Convert(decoder, ReadOnlySpan <byte> .Empty, (IBufferWriter <char>)null, true, out _, out _));
}
public static void Convert_Encoder_ReadOnlySequence_IBufferWriter_ParamChecks()
{
Encoder encoder = Encoding.UTF8.GetEncoder();
IBufferWriter <byte> writer = new ArrayBufferWriter <byte>();
Assert.Throws <ArgumentNullException>("encoder", () => EncodingExtensions.Convert((Encoder)null, ReadOnlySequence <char> .Empty, writer, true, out _, out _));
Assert.Throws <ArgumentNullException>("writer", () => EncodingExtensions.Convert(encoder, ReadOnlySequence <char> .Empty, (IBufferWriter <byte>)null, true, out _, out _));
}
public static void Convert_Decoder_ReadOnlySequence_IBufferWriter()
{
Decoder decoder = Encoding.UTF8.GetDecoder();
ArrayBufferWriter <char> writer = new ArrayBufferWriter <char>();
// First, input with no flushing and no leftover data.
ReadOnlySequence <byte> inputData = SequenceFactory.Create(
new byte[] { 0x20 }, // U+0020
new byte[] { 0x61, 0xC2 }, // U+0061 and U+0080 (continues on next line)
new byte[] { 0x80, 0xED, 0x9F, 0xBF }); // (cont.) + U+D7FF
EncodingExtensions.Convert(decoder, inputData, writer, flush: false, out long charsUsed, out bool completed);
Assert.Equal(4, charsUsed);
Assert.True(completed);
// Then, input with no flushing and leftover data.
inputData = SequenceFactory.Create(
new byte[] { 0xF4, 0x80 }); // U+100000 (continues on next line)
EncodingExtensions.Convert(decoder, inputData, writer, flush: false, out charsUsed, out completed);
Assert.Equal(0, charsUsed);
Assert.False(completed);
// Then, input with flushing and leftover data (should be replaced).
inputData = SequenceFactory.Create(
new byte[] { 0x80, 0x80 }, // (cont.)
new byte[] { 0xC2 }); // leftover data (should be replaced)
EncodingExtensions.Convert(decoder, inputData, writer, flush: true, out charsUsed, out completed);
Assert.Equal(3, charsUsed);
Assert.True(completed);
// Now make sure all of the data was decoded properly.
Assert.Equal("\u0020\u0061\u0080\ud7ff\U00100000\ufffd", writer.WrittenSpan.ToString());
}
public static void Convert_Encoder_ReadOnlySequence_IBufferWriter()
{
Encoder encoder = Encoding.UTF8.GetEncoder();
ArrayBufferWriter <byte> writer = new ArrayBufferWriter <byte>();
// First, input with no flushing and no leftover data.
ReadOnlySequence <char> inputData = SequenceFactory.Create(
new char[] { '\u0020' }, // U+0020
new char[] { '\ud7ff' }); // U+D7FF
EncodingExtensions.Convert(encoder, inputData, writer, flush: false, out long bytesUsed, out bool completed);
Assert.Equal(4, bytesUsed);
Assert.True(completed);
// Then, input with no flushing and leftover data.
inputData = SequenceFactory.Create(
new char[] { '\udbc0' }); // U+100000 (continues on next line)
EncodingExtensions.Convert(encoder, inputData, writer, flush: false, out bytesUsed, out completed);
Assert.Equal(0, bytesUsed);
Assert.False(completed);
// Then, input with flushing and leftover data (should be replaced).
inputData = SequenceFactory.Create(
new char[] { '\udc00' }, // (cont.)
new char[] { '\ud800' }); // leftover data (should be replaced)
EncodingExtensions.Convert(encoder, inputData, writer, flush: true, out bytesUsed, out completed);
Assert.Equal(7, bytesUsed);
Assert.True(completed);
// Now make sure all of the data was decoded properly.
Assert.Equal(Encoding.UTF8.GetBytes("\u0020\ud7ff\U00100000\ufffd"), writer.WrittenSpan.ToArray());
}
