public void EncodeAllUnicodeCodePoints(bool useUtf8Encoder)
{
Text.Encoding systemEncoder = useUtf8Encoder ? Text.Encoding.UTF8 : Text.Encoding.Unicode;
const uint maximumValidCodePoint = 0x10FFFF;
uint[] codePoints = new uint[maximumValidCodePoint + 1];
var plainText = new StringBuilder();
for (int i = 0; i <= maximumValidCodePoint; i++)
{
if (!EncodingHelper.IsValidScalarValue((uint)i))
{
codePoints[i] = 0; // skip unsupported characters
plainText.Append((char)0);
}
else
{
codePoints[i] = (uint)i;
if (i > 0xFFFF)
{
plainText.Append(char.ConvertFromUtf32(i));
}
else
{
plainText.Append((char)i);
}
}
}
ReadOnlySpan <uint> allCodePoints = codePoints;
Span <byte> buffer = new byte[4 * (maximumValidCodePoint + 1)];
int written;
int consumed;
if (useUtf8Encoder)
{
Assert.Equal(Buffers.OperationStatus.Done, Encodings.Utf32.ToUtf8(allCodePoints.AsBytes(), buffer, out consumed, out written));
}
else
{
Assert.Equal(Buffers.OperationStatus.Done, Encodings.Utf32.ToUtf16(allCodePoints.AsBytes(), buffer, out consumed, out written));
}
buffer = buffer.Slice(0, written);
string unicodeString = plainText.ToString();
ReadOnlySpan <char> characters = unicodeString.AsReadOnlySpan();
int byteCount = systemEncoder.GetByteCount(unicodeString);
byte[] expectedBytes = new byte[byteCount];
systemEncoder.GetBytes(characters.ToArray(), 0, characters.Length, expectedBytes, 0);
Assert.Equal(expectedBytes.Length, buffer.Length);
Assert.True(buffer.SequenceEqual(expectedBytes), "Bad output from system encoding comparison");
}
public void BruteTestingRoundtripEncodeDecodeAllUnicodeCodePoints(bool useUtf8Encoder)
{
const int maximumValidCodePoint = 0x10FFFF;
uint[] expectedCodePoints = new uint[maximumValidCodePoint + 1];
for (uint i = 0; i <= maximumValidCodePoint; i++)
{
if (!EncodingHelper.IsValidScalarValue(i))
{
expectedCodePoints[i] = 0; // skip unsupported code points.
}
else
{
expectedCodePoints[i] = i;
}
}
ReadOnlySpan <uint> allCodePoints = expectedCodePoints;
Span <byte> buffer = new byte[4 * (maximumValidCodePoint + 1)];
int consumed;
int written;
if (useUtf8Encoder)
{
Assert.Equal(OperationStatus.Done, Encodings.Utf32.ToUtf8(allCodePoints.AsBytes(), buffer, out consumed, out written));
}
else
{
Assert.Equal(OperationStatus.Done, Encodings.Utf32.ToUtf16(allCodePoints.AsBytes(), buffer, out consumed, out written));
}
Assert.Equal(allCodePoints.AsBytes().Length, consumed);
buffer = buffer.Slice(0, written);
Span <uint> utf32 = new uint[maximumValidCodePoint + 1];
if (useUtf8Encoder)
{
Assert.Equal(OperationStatus.Done, Encodings.Utf8.ToUtf32(buffer, utf32.AsBytes(), out consumed, out written));
}
else
{
Assert.Equal(OperationStatus.Done, Encodings.Utf16.ToUtf32(buffer, utf32.AsBytes(), out consumed, out written));
}
Assert.Equal(buffer.Length, consumed);
Assert.Equal((maximumValidCodePoint + 1) * sizeof(uint), written);
Assert.True(allCodePoints.SequenceEqual(utf32), "Bad output from round-trip");
}
public static bool TryComputeKeyBytes(this ReadOnlySpan <char> key, Span <byte> keyBytes)
{
var utf16Bytes = key.AsBytes();
int size = utf16Bytes.Length; // the input must be ASCII (i.e. Base64 encoded)
var buffer = size < 128 ? stackalloc byte[size] : new byte[size];
var result = Encodings.Utf16.ToUtf8(utf16Bytes, buffer, out int consumed, out int written);
if (result != OperationStatus.Done)
{
throw new ArgumentOutOfRangeException(nameof(key), $"ToUtf8 returned {result}");
}
result = Base64.DecodeFromUtf8(buffer.Slice(0, written), keyBytes, out consumed, out written);
if (result == OperationStatus.Done)
{
return(true);
}
if (result == OperationStatus.DestinationTooSmall)
{
return(false);
}
throw new ArgumentOutOfRangeException(nameof(key), $"Base64.Decode returned {result}");
}
internal static bool Encode(
ReadOnlySpan <Color8> data,
ref TextureFormat format,
Vector2I dimensions,
bool hasAlpha,
bool isPunchthroughAlpha,
bool isMasky,
bool hasR,
bool hasG,
bool hasB,
out PinnedSpan <byte> result
)
{
TextureFormat resultFormat = hasAlpha ? TextureFormat.BC3 : TextureFormat.BC1;
var resultBytes = SpanExt.MakePinned <byte>(RequiredSize(dimensions, hasAlpha));
var compressionMode = Configuration.Config.Resample.BlockCompression.Quality switch {
CompressionQuality.Low => CompressionMode.Normal,
CompressionQuality.Medium => CompressionMode.Dither,
CompressionQuality.High => CompressionMode.HighQuality,
_ => ThrowHelper.ThrowInvalidOperationException <CompressionMode>($"Unknown Quality: '{Configuration.Config.Resample.BlockCompression.Quality}'")
};
CompressDxt(resultBytes, data.AsBytes(), dimensions, hasAlpha, compressionMode);
result = resultBytes;
format = resultFormat;
return(true);
}
public static void AsBytesUIntToByte()
{
uint[] a = { 0x44332211, 0x88776655 };
ReadOnlySpan <uint> span = new ReadOnlySpan <uint>(a);
ReadOnlySpan <byte> asBytes = span.AsBytes <uint>();
Assert.True(Unsafe.AreSame(ref Unsafe.As <uint, byte>(ref span.DangerousGetPinnableReference()), ref asBytes.DangerousGetPinnableReference()));
asBytes.Validate <byte>(0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88);
}
public static void AsBytesUIntToByte()
{
uint[] a = { 0x44332211, 0x88776655 };
ReadOnlySpan <uint> span = new ReadOnlySpan <uint>(a);
ReadOnlySpan <byte> asBytes = span.AsBytes <uint>();
Assert.True(Unsafe.AreSame(ref Unsafe.As <uint, byte>(ref Unsafe.AsRef(in MemoryMarshal.GetReference(span))), ref Unsafe.AsRef(in MemoryMarshal.GetReference(asBytes))));
asBytes.Validate <byte>(0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88);
}
public static void AsBytesUIntToByte()
{
uint[] a = { 0x44332211, 0x88776655 };
ReadOnlySpan<uint> span = new ReadOnlySpan<uint>(a);
ReadOnlySpan<byte> asBytes = span.AsBytes<uint>();
Assert.True(Unsafe.AreSame<byte>(ref Unsafe.As<uint, byte>(ref span.DangerousGetPinnableReference()), ref asBytes.DangerousGetPinnableReference()));
asBytes.Validate<byte>(0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88);
}
public bool TryEncode(ReadOnlySpan <char> source, Span <byte> destination, out int bytesConsumed, out int bytesWritten)
{
ReadOnlySpan <byte> srcBytes = source.AsBytes();
if (this == SymbolTable.InvariantUtf16)
{
return(TryEncodeUtf16(srcBytes, destination, out bytesConsumed, out bytesWritten));
}
const int BufferSize = 256;
int srcLength = srcBytes.Length;
if (srcLength <= 0)
{
bytesConsumed = bytesWritten = 0;
return(true);
}
Span <byte> temp;
unsafe
{
byte *pTemp = stackalloc byte[BufferSize];
temp = new Span <byte>(pTemp, BufferSize);
}
bytesWritten = 0;
bytesConsumed = 0;
while (srcLength > bytesConsumed)
{
var status = Encoders.Utf16.ToUtf8(srcBytes, temp, out int consumed, out int written);
if (status == Buffers.TransformationStatus.InvalidData)
{
goto ExitFailed;
}
srcBytes = srcBytes.Slice(consumed);
bytesConsumed += consumed;
if (!TryEncode(temp.Slice(0, written), destination, out consumed, out written))
{
goto ExitFailed;
}
destination = destination.Slice(written);
bytesWritten += written;
}
return(true);
ExitFailed:
return(false);
}
private void Write(ReadOnlySpan <char> value)
{
ReadOnlySpan <byte> source = value.AsBytes();
if (UseFastUtf8)
{
Span <byte> destination = _output.GetSpan();
while (true)
{
var status = Encodings.Utf16.ToUtf8(source, destination, out int consumed, out int written);
if (status == Buffers.OperationStatus.Done)
{
_output.Advance(written);
return;
}
if (status == Buffers.OperationStatus.DestinationTooSmall)
{
destination = EnsureBuffer();
continue;
}
// This is a failure due to bad input. This shouldn't happen under normal circumstances.
throw new FormatException();
}
}
else if (UseFastUtf16)
{
Span <byte> destination = EnsureBuffer(source.Length);
source.CopyTo(destination);
_output.Advance(source.Length);
}
else
{
Span <byte> destination = _output.GetSpan();
if (!_output.SymbolTable.TryEncode(source, destination, out int consumed, out int written))
{
destination = EnsureBuffer();
}
_output.Advance(written);
}
}
public static byte[] ComputeKeyBytes(this ReadOnlySpan <char> key)
{
int size = key.Length * 2;
var buffer = size < 128 ? stackalloc byte[size] : new byte[size];
if (Encodings.Utf16.ToUtf8(key.AsBytes(), buffer, out int consumed, out int written) != OperationStatus.Done)
{
throw new NotImplementedException("need to resize buffer");
}
var keyBytes = new byte[64];
var result = Base64.DecodeFromUtf8(buffer.Slice(0, written), keyBytes, out consumed, out written);
if (result != OperationStatus.Done || written != 64)
{
throw new NotImplementedException("need to resize buffer");
}
return(keyBytes);
}
public unsafe void DecimalPositiveTests(string text, int length, decimal expectedValue, int expectedConsumed)
{
byte[] byteBuffer = Text.Encoding.UTF8.GetBytes(text);
ReadOnlySpan <byte> byteSpan = new ReadOnlySpan <byte>(byteBuffer);
char[] charBuffer = text.ToCharArray();
ReadOnlySpan <char> charSpan = new ReadOnlySpan <char>(charBuffer);
bool result;
result = CustomParser.TryParseDecimal(byteSpan, out decimal actualValue, out int actualConsumed, SymbolTable.InvariantUtf8);
Assert.True(result);
Assert.Equal(expectedValue, actualValue);
Assert.Equal(expectedConsumed, actualConsumed);
result = Utf8Parser.TryParse(byteSpan, out actualValue, out actualConsumed);
Assert.True(result);
Assert.Equal(expectedValue, actualValue);
Assert.Equal(expectedConsumed, actualConsumed);
ReadOnlySpan <byte> utf16ByteSpan = charSpan.AsBytes();
result = CustomParser.TryParseDecimal(utf16ByteSpan, out actualValue, out actualConsumed, SymbolTable.InvariantUtf16);
Assert.True(result);
Assert.Equal(expectedValue, actualValue);
Assert.Equal(expectedConsumed, actualConsumed / 2);
result = Utf16Parser.TryParseDecimal(charSpan, out actualValue);
Assert.True(result);
Assert.Equal(expectedValue, actualValue);
result = Utf16Parser.TryParseDecimal(charSpan, out actualValue, out actualConsumed);
Assert.True(result);
Assert.Equal(expectedValue, actualValue);
Assert.Equal(expectedConsumed, actualConsumed);
}
public unsafe void DecimalPositiveTests(string text, int length, decimal expectedValue, int expectedConsumed)
{
byte[] byteBuffer = new Utf8String(text).CopyBytes();
ReadOnlySpan <byte> byteSpan = new ReadOnlySpan <byte>(byteBuffer);
char[] charBuffer = text.ToCharArray();
ReadOnlySpan <char> charSpan = new ReadOnlySpan <char>(charBuffer);
bool result;
decimal actualValue;
int actualConsumed;
result = PrimitiveParser.TryParseDecimal(byteSpan, out actualValue, out actualConsumed, TextEncoder.Utf8);
Assert.True(result);
Assert.Equal(expectedValue, actualValue);
Assert.Equal(expectedConsumed, actualConsumed);
fixed(byte *bytePointer = byteBuffer)
{
result = PrimitiveParser.InvariantUtf8.TryParseDecimal(bytePointer, length, out actualValue);
Assert.True(result);
Assert.Equal(expectedValue, actualValue);
result = PrimitiveParser.InvariantUtf8.TryParseDecimal(bytePointer, length, out actualValue, out actualConsumed);
Assert.True(result);
Assert.Equal(expectedValue, actualValue);
Assert.Equal(expectedConsumed, actualConsumed);
}
result = PrimitiveParser.InvariantUtf8.TryParseDecimal(byteSpan, out actualValue);
Assert.True(result);
Assert.Equal(expectedValue, actualValue);
result = PrimitiveParser.InvariantUtf8.TryParseDecimal(byteSpan, out actualValue, out actualConsumed);
Assert.True(result);
Assert.Equal(expectedValue, actualValue);
Assert.Equal(expectedConsumed, actualConsumed);
ReadOnlySpan <byte> utf16ByteSpan = charSpan.AsBytes();
result = PrimitiveParser.TryParseDecimal(utf16ByteSpan, out actualValue, out actualConsumed, TextEncoder.Utf16);
Assert.True(result);
Assert.Equal(expectedValue, actualValue);
Assert.Equal(expectedConsumed, actualConsumed / 2);
fixed(char *charPointer = charBuffer)
{
result = PrimitiveParser.InvariantUtf16.TryParseDecimal(charPointer, length, out actualValue);
Assert.True(result);
Assert.Equal(expectedValue, actualValue);
result = PrimitiveParser.InvariantUtf16.TryParseDecimal(charPointer, length, out actualValue, out actualConsumed);
Assert.True(result);
Assert.Equal(expectedValue, actualValue);
Assert.Equal(expectedConsumed, actualConsumed);
}
result = PrimitiveParser.InvariantUtf16.TryParseDecimal(charSpan, out actualValue);
Assert.True(result);
Assert.Equal(expectedValue, actualValue);
result = PrimitiveParser.InvariantUtf16.TryParseDecimal(charSpan, out actualValue, out actualConsumed);
Assert.True(result);
Assert.Equal(expectedValue, actualValue);
Assert.Equal(expectedConsumed, actualConsumed);
}
public static void Lerp(ReadOnlySpan <Alpha8> left, ReadOnlySpan <Alpha8> right, int amount, Span <Alpha8> dst)
{
Vector4Streaming.Lerp(left.AsBytes(), right.AsBytes(), amount, dst.AsBytes());
}
public unsafe void BooleanPositiveTests(string text, int length, bool expectedValue, int expectedConsumed)
{
byte[] byteBuffer = Text.Encoding.UTF8.GetBytes(text);
ReadOnlySpan <byte> byteSpan = new ReadOnlySpan <byte>(byteBuffer);
char[] charBuffer = text.ToCharArray();
ReadOnlySpan <char> charSpan = new ReadOnlySpan <char>(charBuffer);
bool result;
bool actualValue;
int actualConsumed;
result = CustomParser.TryParseBoolean(byteSpan, out actualValue, out actualConsumed, SymbolTable.InvariantUtf8);
Assert.True(result);
Assert.Equal(expectedValue, actualValue);
Assert.Equal(expectedConsumed, actualConsumed);
//fixed (byte* bytePointer = byteBuffer)
//{
// result = Parsers.Utf8.TryParseBoolean(bytePointer, length, out actualValue);
// Assert.True(result);
// Assert.Equal(expectedValue, actualValue);
// result = Parsers.Utf8.TryParseBoolean(bytePointer, length, out actualValue, out actualConsumed);
// Assert.True(result);
// Assert.Equal(expectedValue, actualValue);
// Assert.Equal(expectedConsumed, actualConsumed);
//}
result = Utf8Parser.TryParseBoolean(byteSpan, out actualValue);
Assert.True(result);
Assert.Equal(expectedValue, actualValue);
result = Utf8Parser.TryParseBoolean(byteSpan, out actualValue, out actualConsumed);
Assert.True(result);
Assert.Equal(expectedValue, actualValue);
Assert.Equal(expectedConsumed, actualConsumed);
ReadOnlySpan <byte> utf16ByteSpan = charSpan.AsBytes();
result = CustomParser.TryParseBoolean(utf16ByteSpan, out actualValue, out actualConsumed, SymbolTable.InvariantUtf16);
Assert.True(result);
Assert.Equal(expectedValue, actualValue);
Assert.Equal(expectedConsumed, actualConsumed / 2);
//fixed (char* charPointer = charBuffer)
//{
// result = Parsers.Utf16.TryParseBoolean(charPointer, length, out actualValue);
// Assert.True(result);
// Assert.Equal(expectedValue, actualValue);
// result = Parsers.Utf16.TryParseBoolean(charPointer, length, out actualValue, out actualConsumed);
// Assert.True(result);
// Assert.Equal(expectedValue, actualValue);
// Assert.Equal(expectedConsumed, actualConsumed);
//}
result = Utf16Parser.TryParseBoolean(charSpan, out actualValue);
Assert.True(result);
Assert.Equal(expectedValue, actualValue);
result = Utf16Parser.TryParseBoolean(charSpan, out actualValue, out actualConsumed);
Assert.True(result);
Assert.Equal(expectedValue, actualValue);
Assert.Equal(expectedConsumed, actualConsumed);
}