public void EncodeAllUnicodeCodePoints(bool useUtf8Encoder) { Text.Encoding systemEncoder = useUtf8Encoder ? Text.Encoding.UTF8 : Text.Encoding.Unicode; const uint maximumValidCodePoint = 0x10FFFF; uint[] codePoints = new uint[maximumValidCodePoint + 1]; var plainText = new StringBuilder(); for (int i = 0; i <= maximumValidCodePoint; i++) { if (!EncodingHelper.IsValidScalarValue((uint)i)) { codePoints[i] = 0; // skip unsupported characters plainText.Append((char)0); } else { codePoints[i] = (uint)i; if (i > 0xFFFF) { plainText.Append(char.ConvertFromUtf32(i)); } else { plainText.Append((char)i); } } } ReadOnlySpan <uint> allCodePoints = codePoints; Span <byte> buffer = new byte[4 * (maximumValidCodePoint + 1)]; int written; int consumed; if (useUtf8Encoder) { Assert.Equal(Buffers.OperationStatus.Done, Encodings.Utf32.ToUtf8(allCodePoints.AsBytes(), buffer, out consumed, out written)); } else { Assert.Equal(Buffers.OperationStatus.Done, Encodings.Utf32.ToUtf16(allCodePoints.AsBytes(), buffer, out consumed, out written)); } buffer = buffer.Slice(0, written); string unicodeString = plainText.ToString(); ReadOnlySpan <char> characters = unicodeString.AsReadOnlySpan(); int byteCount = systemEncoder.GetByteCount(unicodeString); byte[] expectedBytes = new byte[byteCount]; systemEncoder.GetBytes(characters.ToArray(), 0, characters.Length, expectedBytes, 0); Assert.Equal(expectedBytes.Length, buffer.Length); Assert.True(buffer.SequenceEqual(expectedBytes), "Bad output from system encoding comparison"); }
public void BruteTestingRoundtripEncodeDecodeAllUnicodeCodePoints(bool useUtf8Encoder) { const int maximumValidCodePoint = 0x10FFFF; uint[] expectedCodePoints = new uint[maximumValidCodePoint + 1]; for (uint i = 0; i <= maximumValidCodePoint; i++) { if (!EncodingHelper.IsValidScalarValue(i)) { expectedCodePoints[i] = 0; // skip unsupported code points. } else { expectedCodePoints[i] = i; } } ReadOnlySpan <uint> allCodePoints = expectedCodePoints; Span <byte> buffer = new byte[4 * (maximumValidCodePoint + 1)]; int consumed; int written; if (useUtf8Encoder) { Assert.Equal(OperationStatus.Done, Encodings.Utf32.ToUtf8(allCodePoints.AsBytes(), buffer, out consumed, out written)); } else { Assert.Equal(OperationStatus.Done, Encodings.Utf32.ToUtf16(allCodePoints.AsBytes(), buffer, out consumed, out written)); } Assert.Equal(allCodePoints.AsBytes().Length, consumed); buffer = buffer.Slice(0, written); Span <uint> utf32 = new uint[maximumValidCodePoint + 1]; if (useUtf8Encoder) { Assert.Equal(OperationStatus.Done, Encodings.Utf8.ToUtf32(buffer, utf32.AsBytes(), out consumed, out written)); } else { Assert.Equal(OperationStatus.Done, Encodings.Utf16.ToUtf32(buffer, utf32.AsBytes(), out consumed, out written)); } Assert.Equal(buffer.Length, consumed); Assert.Equal((maximumValidCodePoint + 1) * sizeof(uint), written); Assert.True(allCodePoints.SequenceEqual(utf32), "Bad output from round-trip"); }
public static bool TryComputeKeyBytes(this ReadOnlySpan <char> key, Span <byte> keyBytes) { var utf16Bytes = key.AsBytes(); int size = utf16Bytes.Length; // the input must be ASCII (i.e. Base64 encoded) var buffer = size < 128 ? stackalloc byte[size] : new byte[size]; var result = Encodings.Utf16.ToUtf8(utf16Bytes, buffer, out int consumed, out int written); if (result != OperationStatus.Done) { throw new ArgumentOutOfRangeException(nameof(key), $"ToUtf8 returned {result}"); } result = Base64.DecodeFromUtf8(buffer.Slice(0, written), keyBytes, out consumed, out written); if (result == OperationStatus.Done) { return(true); } if (result == OperationStatus.DestinationTooSmall) { return(false); } throw new ArgumentOutOfRangeException(nameof(key), $"Base64.Decode returned {result}"); }
internal static bool Encode( ReadOnlySpan <Color8> data, ref TextureFormat format, Vector2I dimensions, bool hasAlpha, bool isPunchthroughAlpha, bool isMasky, bool hasR, bool hasG, bool hasB, out PinnedSpan <byte> result ) { TextureFormat resultFormat = hasAlpha ? TextureFormat.BC3 : TextureFormat.BC1; var resultBytes = SpanExt.MakePinned <byte>(RequiredSize(dimensions, hasAlpha)); var compressionMode = Configuration.Config.Resample.BlockCompression.Quality switch { CompressionQuality.Low => CompressionMode.Normal, CompressionQuality.Medium => CompressionMode.Dither, CompressionQuality.High => CompressionMode.HighQuality, _ => ThrowHelper.ThrowInvalidOperationException <CompressionMode>($"Unknown Quality: '{Configuration.Config.Resample.BlockCompression.Quality}'") }; CompressDxt(resultBytes, data.AsBytes(), dimensions, hasAlpha, compressionMode); result = resultBytes; format = resultFormat; return(true); }
public static void AsBytesUIntToByte() { uint[] a = { 0x44332211, 0x88776655 }; ReadOnlySpan <uint> span = new ReadOnlySpan <uint>(a); ReadOnlySpan <byte> asBytes = span.AsBytes <uint>(); Assert.True(Unsafe.AreSame(ref Unsafe.As <uint, byte>(ref span.DangerousGetPinnableReference()), ref asBytes.DangerousGetPinnableReference())); asBytes.Validate <byte>(0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88); }
public static void AsBytesUIntToByte() { uint[] a = { 0x44332211, 0x88776655 }; ReadOnlySpan <uint> span = new ReadOnlySpan <uint>(a); ReadOnlySpan <byte> asBytes = span.AsBytes <uint>(); Assert.True(Unsafe.AreSame(ref Unsafe.As <uint, byte>(ref Unsafe.AsRef(in MemoryMarshal.GetReference(span))), ref Unsafe.AsRef(in MemoryMarshal.GetReference(asBytes)))); asBytes.Validate <byte>(0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88); }
public static void AsBytesUIntToByte() { uint[] a = { 0x44332211, 0x88776655 }; ReadOnlySpan<uint> span = new ReadOnlySpan<uint>(a); ReadOnlySpan<byte> asBytes = span.AsBytes<uint>(); Assert.True(Unsafe.AreSame<byte>(ref Unsafe.As<uint, byte>(ref span.DangerousGetPinnableReference()), ref asBytes.DangerousGetPinnableReference())); asBytes.Validate<byte>(0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88); }
public bool TryEncode(ReadOnlySpan <char> source, Span <byte> destination, out int bytesConsumed, out int bytesWritten) { ReadOnlySpan <byte> srcBytes = source.AsBytes(); if (this == SymbolTable.InvariantUtf16) { return(TryEncodeUtf16(srcBytes, destination, out bytesConsumed, out bytesWritten)); } const int BufferSize = 256; int srcLength = srcBytes.Length; if (srcLength <= 0) { bytesConsumed = bytesWritten = 0; return(true); } Span <byte> temp; unsafe { byte *pTemp = stackalloc byte[BufferSize]; temp = new Span <byte>(pTemp, BufferSize); } bytesWritten = 0; bytesConsumed = 0; while (srcLength > bytesConsumed) { var status = Encoders.Utf16.ToUtf8(srcBytes, temp, out int consumed, out int written); if (status == Buffers.TransformationStatus.InvalidData) { goto ExitFailed; } srcBytes = srcBytes.Slice(consumed); bytesConsumed += consumed; if (!TryEncode(temp.Slice(0, written), destination, out consumed, out written)) { goto ExitFailed; } destination = destination.Slice(written); bytesWritten += written; } return(true); ExitFailed: return(false); }
private void Write(ReadOnlySpan <char> value) { ReadOnlySpan <byte> source = value.AsBytes(); if (UseFastUtf8) { Span <byte> destination = _output.GetSpan(); while (true) { var status = Encodings.Utf16.ToUtf8(source, destination, out int consumed, out int written); if (status == Buffers.OperationStatus.Done) { _output.Advance(written); return; } if (status == Buffers.OperationStatus.DestinationTooSmall) { destination = EnsureBuffer(); continue; } // This is a failure due to bad input. This shouldn't happen under normal circumstances. throw new FormatException(); } } else if (UseFastUtf16) { Span <byte> destination = EnsureBuffer(source.Length); source.CopyTo(destination); _output.Advance(source.Length); } else { Span <byte> destination = _output.GetSpan(); if (!_output.SymbolTable.TryEncode(source, destination, out int consumed, out int written)) { destination = EnsureBuffer(); } _output.Advance(written); } }
public static byte[] ComputeKeyBytes(this ReadOnlySpan <char> key) { int size = key.Length * 2; var buffer = size < 128 ? stackalloc byte[size] : new byte[size]; if (Encodings.Utf16.ToUtf8(key.AsBytes(), buffer, out int consumed, out int written) != OperationStatus.Done) { throw new NotImplementedException("need to resize buffer"); } var keyBytes = new byte[64]; var result = Base64.DecodeFromUtf8(buffer.Slice(0, written), keyBytes, out consumed, out written); if (result != OperationStatus.Done || written != 64) { throw new NotImplementedException("need to resize buffer"); } return(keyBytes); }
public unsafe void DecimalPositiveTests(string text, int length, decimal expectedValue, int expectedConsumed) { byte[] byteBuffer = Text.Encoding.UTF8.GetBytes(text); ReadOnlySpan <byte> byteSpan = new ReadOnlySpan <byte>(byteBuffer); char[] charBuffer = text.ToCharArray(); ReadOnlySpan <char> charSpan = new ReadOnlySpan <char>(charBuffer); bool result; result = CustomParser.TryParseDecimal(byteSpan, out decimal actualValue, out int actualConsumed, SymbolTable.InvariantUtf8); Assert.True(result); Assert.Equal(expectedValue, actualValue); Assert.Equal(expectedConsumed, actualConsumed); result = Utf8Parser.TryParse(byteSpan, out actualValue, out actualConsumed); Assert.True(result); Assert.Equal(expectedValue, actualValue); Assert.Equal(expectedConsumed, actualConsumed); ReadOnlySpan <byte> utf16ByteSpan = charSpan.AsBytes(); result = CustomParser.TryParseDecimal(utf16ByteSpan, out actualValue, out actualConsumed, SymbolTable.InvariantUtf16); Assert.True(result); Assert.Equal(expectedValue, actualValue); Assert.Equal(expectedConsumed, actualConsumed / 2); result = Utf16Parser.TryParseDecimal(charSpan, out actualValue); Assert.True(result); Assert.Equal(expectedValue, actualValue); result = Utf16Parser.TryParseDecimal(charSpan, out actualValue, out actualConsumed); Assert.True(result); Assert.Equal(expectedValue, actualValue); Assert.Equal(expectedConsumed, actualConsumed); }
public unsafe void DecimalPositiveTests(string text, int length, decimal expectedValue, int expectedConsumed) { byte[] byteBuffer = new Utf8String(text).CopyBytes(); ReadOnlySpan <byte> byteSpan = new ReadOnlySpan <byte>(byteBuffer); char[] charBuffer = text.ToCharArray(); ReadOnlySpan <char> charSpan = new ReadOnlySpan <char>(charBuffer); bool result; decimal actualValue; int actualConsumed; result = PrimitiveParser.TryParseDecimal(byteSpan, out actualValue, out actualConsumed, TextEncoder.Utf8); Assert.True(result); Assert.Equal(expectedValue, actualValue); Assert.Equal(expectedConsumed, actualConsumed); fixed(byte *bytePointer = byteBuffer) { result = PrimitiveParser.InvariantUtf8.TryParseDecimal(bytePointer, length, out actualValue); Assert.True(result); Assert.Equal(expectedValue, actualValue); result = PrimitiveParser.InvariantUtf8.TryParseDecimal(bytePointer, length, out actualValue, out actualConsumed); Assert.True(result); Assert.Equal(expectedValue, actualValue); Assert.Equal(expectedConsumed, actualConsumed); } result = PrimitiveParser.InvariantUtf8.TryParseDecimal(byteSpan, out actualValue); Assert.True(result); Assert.Equal(expectedValue, actualValue); result = PrimitiveParser.InvariantUtf8.TryParseDecimal(byteSpan, out actualValue, out actualConsumed); Assert.True(result); Assert.Equal(expectedValue, actualValue); Assert.Equal(expectedConsumed, actualConsumed); ReadOnlySpan <byte> utf16ByteSpan = charSpan.AsBytes(); result = PrimitiveParser.TryParseDecimal(utf16ByteSpan, out actualValue, out actualConsumed, TextEncoder.Utf16); Assert.True(result); Assert.Equal(expectedValue, actualValue); Assert.Equal(expectedConsumed, actualConsumed / 2); fixed(char *charPointer = charBuffer) { result = PrimitiveParser.InvariantUtf16.TryParseDecimal(charPointer, length, out actualValue); Assert.True(result); Assert.Equal(expectedValue, actualValue); result = PrimitiveParser.InvariantUtf16.TryParseDecimal(charPointer, length, out actualValue, out actualConsumed); Assert.True(result); Assert.Equal(expectedValue, actualValue); Assert.Equal(expectedConsumed, actualConsumed); } result = PrimitiveParser.InvariantUtf16.TryParseDecimal(charSpan, out actualValue); Assert.True(result); Assert.Equal(expectedValue, actualValue); result = PrimitiveParser.InvariantUtf16.TryParseDecimal(charSpan, out actualValue, out actualConsumed); Assert.True(result); Assert.Equal(expectedValue, actualValue); Assert.Equal(expectedConsumed, actualConsumed); }
public static void Lerp(ReadOnlySpan <Alpha8> left, ReadOnlySpan <Alpha8> right, int amount, Span <Alpha8> dst) { Vector4Streaming.Lerp(left.AsBytes(), right.AsBytes(), amount, dst.AsBytes()); }
public unsafe void BooleanPositiveTests(string text, int length, bool expectedValue, int expectedConsumed) { byte[] byteBuffer = Text.Encoding.UTF8.GetBytes(text); ReadOnlySpan <byte> byteSpan = new ReadOnlySpan <byte>(byteBuffer); char[] charBuffer = text.ToCharArray(); ReadOnlySpan <char> charSpan = new ReadOnlySpan <char>(charBuffer); bool result; bool actualValue; int actualConsumed; result = CustomParser.TryParseBoolean(byteSpan, out actualValue, out actualConsumed, SymbolTable.InvariantUtf8); Assert.True(result); Assert.Equal(expectedValue, actualValue); Assert.Equal(expectedConsumed, actualConsumed); //fixed (byte* bytePointer = byteBuffer) //{ // result = Parsers.Utf8.TryParseBoolean(bytePointer, length, out actualValue); // Assert.True(result); // Assert.Equal(expectedValue, actualValue); // result = Parsers.Utf8.TryParseBoolean(bytePointer, length, out actualValue, out actualConsumed); // Assert.True(result); // Assert.Equal(expectedValue, actualValue); // Assert.Equal(expectedConsumed, actualConsumed); //} result = Utf8Parser.TryParseBoolean(byteSpan, out actualValue); Assert.True(result); Assert.Equal(expectedValue, actualValue); result = Utf8Parser.TryParseBoolean(byteSpan, out actualValue, out actualConsumed); Assert.True(result); Assert.Equal(expectedValue, actualValue); Assert.Equal(expectedConsumed, actualConsumed); ReadOnlySpan <byte> utf16ByteSpan = charSpan.AsBytes(); result = CustomParser.TryParseBoolean(utf16ByteSpan, out actualValue, out actualConsumed, SymbolTable.InvariantUtf16); Assert.True(result); Assert.Equal(expectedValue, actualValue); Assert.Equal(expectedConsumed, actualConsumed / 2); //fixed (char* charPointer = charBuffer) //{ // result = Parsers.Utf16.TryParseBoolean(charPointer, length, out actualValue); // Assert.True(result); // Assert.Equal(expectedValue, actualValue); // result = Parsers.Utf16.TryParseBoolean(charPointer, length, out actualValue, out actualConsumed); // Assert.True(result); // Assert.Equal(expectedValue, actualValue); // Assert.Equal(expectedConsumed, actualConsumed); //} result = Utf16Parser.TryParseBoolean(charSpan, out actualValue); Assert.True(result); Assert.Equal(expectedValue, actualValue); result = Utf16Parser.TryParseBoolean(charSpan, out actualValue, out actualConsumed); Assert.True(result); Assert.Equal(expectedValue, actualValue); Assert.Equal(expectedConsumed, actualConsumed); }