public static void Ctor_CharPointer_Empty_ReturnsEmpty()
{
char[] inputData = new char[] { '\0' }; // standalone null char
using (BoundedMemory <char> boundedMemory = BoundedMemory.AllocateFromExistingData(inputData))
{
AssertSameAsEmpty(new Utf8String((char *)Unsafe.AsPointer(ref MemoryMarshal.GetReference(boundedMemory.Span))));
}
}
public static void Ctor_BytePointer_Empty_ReturnsEmpty()
{
byte[] inputData = new byte[] { 0 }; // standalone null byte
using (BoundedMemory <byte> boundedMemory = BoundedMemory.AllocateFromExistingData(inputData))
{
AssertSameAsEmpty(new Utf8String((byte *)Unsafe.AsPointer(ref MemoryMarshal.GetReference(boundedMemory.Span))));
}
}
public static void Ctor_BytePointer_InvalidData_Throws()
{
byte[] inputData = new byte[] { (byte)'H', (byte)'e', (byte)0xFF, (byte)'l', (byte)'o', (byte)'\0' };
using (BoundedMemory <byte> boundedMemory = BoundedMemory.AllocateFromExistingData(inputData))
{
Assert.Throws <ArgumentException>(() => new Utf8String((byte *)Unsafe.AsPointer(ref MemoryMarshal.GetReference(boundedMemory.Span))));
}
}
public static void Ctor_BytePointer_ValidData_ReturnsOriginalContents()
{
byte[] inputData = new byte[] { (byte)'H', (byte)'e', (byte)'l', (byte)'l', (byte)'o', (byte)'\0' };
using (BoundedMemory <byte> boundedMemory = BoundedMemory.AllocateFromExistingData(inputData))
{
Assert.Equal(u8("Hello"), new Utf8String((byte *)Unsafe.AsPointer(ref MemoryMarshal.GetReference(boundedMemory.Span))));
}
}
public static void Ctor_BytePointer_InvalidData_FixesUpData()
{
byte[] inputData = new byte[] { (byte)'H', (byte)'e', (byte)0xFF, (byte)'l', (byte)'o', (byte)'\0' };
using (BoundedMemory <byte> boundedMemory = BoundedMemory.AllocateFromExistingData(inputData))
{
Assert.Equal(u8("He\uFFFDlo"), new Utf8String((byte *)Unsafe.AsPointer(ref MemoryMarshal.GetReference(boundedMemory.Span))));
}
}
public static void Ctor_CharPointer_InvalidData_FixesUpData()
{
char[] inputData = new char[] { 'H', 'e', '\uD800', 'l', 'o', '\0' }; // standalone surrogate
using (BoundedMemory <char> boundedMemory = BoundedMemory.AllocateFromExistingData(inputData))
{
Assert.Equal(u8("He\uFFFDlo"), new Utf8String((char *)Unsafe.AsPointer(ref MemoryMarshal.GetReference(boundedMemory.Span))));
}
}
public static void Ctor_CharPointer_ValidData_ReturnsOriginalContents()
{
char[] inputData = new char[] { 'H', 'e', 'l', 'l', 'o', '\0' };
using (BoundedMemory <char> boundedMemory = BoundedMemory.AllocateFromExistingData(inputData))
{
Assert.Equal(u8("Hello"), new Utf8String((char *)Unsafe.AsPointer(ref MemoryMarshal.GetReference(boundedMemory.Span))));
}
}
public static void Ctor_CharPointer_InvalidData_Throws()
{
char[] inputData = "He\ud800llo\0".ToCharArray(); // need to manually null-terminate
using (BoundedMemory <char> boundedMemory = BoundedMemory.AllocateFromExistingData(inputData))
{
Assert.Throws <ArgumentException>(() => new Utf8String((char *)Unsafe.AsPointer(ref MemoryMarshal.GetReference(boundedMemory.Span))));
}
}
public static void Ctor_CharPointer_ValidData_ReturnsOriginalContents()
{
char[] inputData = "Hello\0".ToCharArray(); // need to manually null-terminate
using (BoundedMemory <char> boundedMemory = BoundedMemory.AllocateFromExistingData(inputData))
{
Assert.Equal(u8("Hello"), new Utf8String((char *)Unsafe.AsPointer(ref MemoryMarshal.GetReference(boundedMemory.Span))));
}
}
static void RunSpanCompareTest(CompareInfo compareInfo, ReadOnlySpan <char> string1, ReadOnlySpan <char> string2, CompareOptions options, int expected)
{
using BoundedMemory <char> string1BoundedMemory = BoundedMemory.AllocateFromExistingData(string1);
string1BoundedMemory.MakeReadonly();
using BoundedMemory <char> string2BoundedMemory = BoundedMemory.AllocateFromExistingData(string2);
string2BoundedMemory.MakeReadonly();
Assert.Equal(expected, Math.Sign(compareInfo.Compare(string1, string2, options)));
Assert.Equal(-expected, Math.Sign(compareInfo.Compare(string2, string1, options)));
}
static void RunSpanIndexOfTest(CompareInfo compareInfo, ReadOnlySpan <char> source, ReadOnlySpan <char> value, CompareOptions options, int expected)
{
using BoundedMemory <char> sourceBoundedMemory = BoundedMemory.AllocateFromExistingData(source);
sourceBoundedMemory.MakeReadonly();
using BoundedMemory <char> valueBoundedMemory = BoundedMemory.AllocateFromExistingData(value);
valueBoundedMemory.MakeReadonly();
Assert.Equal(expected, compareInfo.IndexOf(sourceBoundedMemory.Span, valueBoundedMemory.Span, options));
if (TryCreateRuneFrom(value, out Rune rune))
{
Assert.Equal(expected, compareInfo.IndexOf(sourceBoundedMemory.Span, rune, options)); // try the Rune-based version
}
}
private static unsafe void GetIndexOfFirstInvalidUtf8Sequence_Test_Core(byte[] input, int expectedRetVal, int expectedRuneCount, int expectedSurrogatePairCount)
{
// Arrange
using BoundedMemory <byte> boundedMemory = BoundedMemory.AllocateFromExistingData(input);
boundedMemory.MakeReadonly();
// Act
int actualRetVal;
int actualSurrogatePairCount;
int actualRuneCount;
fixed(byte *pInputBuffer = &MemoryMarshal.GetReference(boundedMemory.Span))
{
byte *pFirstInvalidByte = _getPointerToFirstInvalidByteFn.Value(pInputBuffer, input.Length, out int utf16CodeUnitCountAdjustment, out int scalarCountAdjustment);
long ptrDiff = pFirstInvalidByte - pInputBuffer;
Assert.True((ulong)ptrDiff <= (uint)input.Length, "ptrDiff was outside expected range.");
Assert.True(utf16CodeUnitCountAdjustment <= 0, "UTF-16 code unit count adjustment must be 0 or negative.");
Assert.True(scalarCountAdjustment <= 0, "Scalar count adjustment must be 0 or negative.");
actualRetVal = (ptrDiff == input.Length) ? -1 : (int)ptrDiff;
// The last two 'out' parameters are:
// a) The number to be added to the "bytes processed" return value to come up with the total UTF-16 code unit count, and
// b) The number to be added to the "total UTF-16 code unit count" value to come up with the total scalar count.
int totalUtf16CodeUnitCount = (int)ptrDiff + utf16CodeUnitCountAdjustment;
actualRuneCount = totalUtf16CodeUnitCount + scalarCountAdjustment;
// Surrogate pair count is number of UTF-16 code units less the number of scalars.
actualSurrogatePairCount = totalUtf16CodeUnitCount - actualRuneCount;
}
// Assert
Assert.Equal(expectedRetVal, actualRetVal);
Assert.Equal(expectedRuneCount, actualRuneCount);
Assert.Equal(expectedSurrogatePairCount, actualSurrogatePairCount);
}
public void EncodeUtf16_OperationStatus_SurrogateHandlingEdgeCases(char[] input, int destBufferSize, bool isFinalBlock, string expectedOutput, int expectedCharsConsumed, OperationStatus expectedResult)
{
// Arrange
var encoder = new ConfigurableScalarTextEncoder(_ => true); // allow all well-formed scalars
using BoundedMemory <char> boundedInput = BoundedMemory.AllocateFromExistingData(input);
using BoundedMemory <char> boundedOutput = BoundedMemory.Allocate <char>(destBufferSize);
// Act
OperationStatus actualResult = encoder.Encode(boundedInput.Span, boundedOutput.Span, out int actualCharsConsumed, out int actualCharsWritten, isFinalBlock);
// Assert
Assert.Equal(expectedResult, actualResult);
Assert.Equal(expectedCharsConsumed, actualCharsConsumed);
Assert.Equal(expectedOutput, boundedOutput.Span.Slice(0, actualCharsWritten).ToString());
}
unsafe static void RunSpanSortKeyTest(CompareInfo compareInfo, ReadOnlySpan <char> source, CompareOptions options, byte[] expectedSortKey)
{
using BoundedMemory <char> sourceBoundedMemory = BoundedMemory.AllocateFromExistingData(source);
sourceBoundedMemory.MakeReadonly();
Assert.Equal(expectedSortKey.Length, compareInfo.GetSortKeyLength(sourceBoundedMemory.Span, options));
using BoundedMemory <byte> sortKeyBoundedMemory = BoundedMemory.Allocate <byte>(expectedSortKey.Length);
// First try with a destination which is too small - should result in an error
Assert.Throws <ArgumentException>("destination", () => compareInfo.GetSortKey(sourceBoundedMemory.Span, sortKeyBoundedMemory.Span.Slice(1), options));
// Next, try with a destination which is perfectly sized - should succeed
Span <byte> sortKeyBoundedSpan = sortKeyBoundedMemory.Span;
sortKeyBoundedSpan.Clear();
Assert.Equal(expectedSortKey.Length, compareInfo.GetSortKey(sourceBoundedMemory.Span, sortKeyBoundedSpan, options));
Assert.Equal(expectedSortKey, sortKeyBoundedSpan[0..expectedSortKey.Length].ToArray());
private static void ToBytes_Test_Core(ReadOnlySpan <char> utf16Input, int destinationSize, bool replaceInvalidSequences, bool isFinalChunk, OperationStatus expectedOperationStatus, int expectedNumCharsRead, ReadOnlySpan <byte> expectedUtf8Transcoding)
{
// Arrange
using (BoundedMemory <char> boundedSource = BoundedMemory.AllocateFromExistingData(utf16Input))
using (BoundedMemory <byte> boundedDestination = BoundedMemory.Allocate <byte>(destinationSize))
{
boundedSource.MakeReadonly();
// Act
OperationStatus actualOperationStatus = Utf8.FromUtf16(boundedSource.Span, boundedDestination.Span, out int actualNumCharsRead, out int actualNumBytesWritten, replaceInvalidSequences, isFinalChunk);
// Assert
Assert.Equal(expectedOperationStatus, actualOperationStatus);
Assert.Equal(expectedNumCharsRead, actualNumCharsRead);
Assert.Equal(expectedUtf8Transcoding.Length, actualNumBytesWritten);
Assert.Equal(expectedUtf8Transcoding.ToArray(), boundedDestination.Span.Slice(0, actualNumBytesWritten).ToArray());
}
}
private static unsafe void GetIndexOfFirstInvalidUtf16Sequence_Test_Core(char[] input, int expectedRetVal, int expectedRuneCount, long expectedUtf8ByteCount)
{
// Arrange
using BoundedMemory <char> boundedMemory = BoundedMemory.AllocateFromExistingData(input);
boundedMemory.MakeReadonly();
// Act
int actualRetVal;
long actualUtf8CodeUnitCount;
int actualRuneCount;
fixed(char *pInputBuffer = &MemoryMarshal.GetReference(boundedMemory.Span))
{
char *pFirstInvalidChar = _getPointerToFirstInvalidCharFn.Value(pInputBuffer, input.Length, out long utf8CodeUnitCountAdjustment, out int scalarCountAdjustment);
long ptrDiff = pFirstInvalidChar - pInputBuffer;
Assert.True((ulong)ptrDiff <= (uint)input.Length, "ptrDiff was outside expected range.");
Assert.True(utf8CodeUnitCountAdjustment >= 0, "UTF-16 code unit count adjustment must be non-negative.");
Assert.True(scalarCountAdjustment <= 0, "Scalar count adjustment must be 0 or negative.");
actualRetVal = (ptrDiff == input.Length) ? -1 : (int)ptrDiff;
// The last two 'out' parameters are:
// a) The number to be added to the "chars processed" return value to come up with the total UTF-8 code unit count, and
// b) The number to be added to the "total UTF-16 code unit count" value to come up with the total scalar count.
actualUtf8CodeUnitCount = ptrDiff + utf8CodeUnitCountAdjustment;
actualRuneCount = (int)ptrDiff + scalarCountAdjustment;
}
// Assert
Assert.Equal(expectedRetVal, actualRetVal);
Assert.Equal(expectedRuneCount, actualRuneCount);
Assert.Equal(actualUtf8CodeUnitCount, expectedUtf8ByteCount);
}
public BoundedUtf8Span(ReadOnlySpan <byte> utf8Data, PoisonPagePlacement placement = PoisonPagePlacement.After)
{
_boundedMemory = BoundedMemory.AllocateFromExistingData(utf8Data, placement);
}