public void GetIndexOfFirstCharacterToEncodeUtf8_AllDataValid() { // Loop from 96 elements all the way down to 0 elements, which tests that we're // not overrunning our read buffers. var span = _boundedBytes.Span; _boundedBytes.MakeWriteable(); span.Fill((byte)_allowedChar); // make buffer all-valid _boundedBytes.MakeReadonly(); for (int i = 96; i >= 0; i--) { Assert.Equal(-1, _encoder.FindFirstCharacterToEncodeUtf8(span.Slice(span.Length - i))); } // Also check from the beginning of the buffer just in case there's some alignment weirdness // in the SIMD-optimized code that causes us to read past where we should. _boundedBytes.MakeWriteable(); for (int i = 96; i >= 0; i--) { span[i] = (byte)_disallowedChar; // make this char invalid Assert.Equal(-1, _encoder.FindFirstCharacterToEncodeUtf8(span.Slice(0, i))); } }
public void IsSuffix(CompareInfo compareInfo, string source, string value, CompareOptions options, bool expected, int expectedMatchLength) { if (options == CompareOptions.None) { Assert.Equal(expected, compareInfo.IsSuffix(source, value)); } Assert.Equal(expected, compareInfo.IsSuffix(source, value, options)); if ((compareInfo == s_invariantCompare) && ((options == CompareOptions.None) || (options == CompareOptions.IgnoreCase))) { StringComparison stringComparison = (options == CompareOptions.IgnoreCase) ? StringComparison.InvariantCultureIgnoreCase : StringComparison.InvariantCulture; Assert.Equal(expected, source.EndsWith(value, stringComparison)); Assert.Equal(expected, source.AsSpan().EndsWith(value.AsSpan(), stringComparison)); } // Now test the span version - use BoundedMemory to detect buffer overruns using BoundedMemory <char> sourceBoundedMemory = BoundedMemory.AllocateFromExistingData <char>(source); sourceBoundedMemory.MakeReadonly(); using BoundedMemory <char> valueBoundedMemory = BoundedMemory.AllocateFromExistingData <char>(value); valueBoundedMemory.MakeReadonly(); Assert.Equal(expected, compareInfo.IsSuffix(sourceBoundedMemory.Span, valueBoundedMemory.Span, options)); Assert.Equal(expected, compareInfo.IsSuffix(sourceBoundedMemory.Span, valueBoundedMemory.Span, options, out int actualMatchLength)); Assert.Equal(expectedMatchLength, actualMatchLength); }
static void RunSpanCompareTest(CompareInfo compareInfo, ReadOnlySpan <char> string1, ReadOnlySpan <char> string2, CompareOptions options, int expected) { using BoundedMemory <char> string1BoundedMemory = BoundedMemory.AllocateFromExistingData(string1); string1BoundedMemory.MakeReadonly(); using BoundedMemory <char> string2BoundedMemory = BoundedMemory.AllocateFromExistingData(string2); string2BoundedMemory.MakeReadonly(); Assert.Equal(expected, Math.Sign(compareInfo.Compare(string1, string2, options))); Assert.Equal(-expected, Math.Sign(compareInfo.Compare(string2, string1, options))); }
public static void WidenLatin1ToUtf16() { using BoundedMemory <byte> latin1Mem = BoundedMemory.Allocate <byte>(128); using BoundedMemory <char> utf16Mem = BoundedMemory.Allocate <char>(128); // Fill the source with [deterministic] pseudo-random bytes, then make readonly. new Random(0x12345).NextBytes(latin1Mem.Span); latin1Mem.MakeReadonly(); // We'll write to the UTF-16 span. // We test with a variety of span lengths to test alignment and fallthrough code paths. ReadOnlySpan <byte> latin1Span = latin1Mem.Span; Span <char> utf16Span = utf16Mem.Span; for (int i = 0; i < latin1Span.Length; i++) { utf16Span.Clear(); // remove any data from previous iteration // First, transcode the data from Latin-1 to UTF-16. CallWidenLatin1ToUtf16(latin1Span.Slice(i), utf16Span.Slice(i)); // Then, validate that the data was transcoded properly. for (int j = i; j < 128; j++) { Assert.Equal((ushort)latin1Span[i], (ushort)utf16Span[i]); } } // Now run the test with a bunch of sliding 48-byte windows. // This tests that we handle correctly the scenario where neither the beginning nor the // end of the buffer is properly vector-aligned. const int WindowSize = 48; for (int i = 0; i < latin1Span.Length - WindowSize; i++) { utf16Span.Clear(); // remove any data from previous iteration // First, transcode the data from Latin-1 to UTF-16. CallWidenLatin1ToUtf16(latin1Span.Slice(i, WindowSize), utf16Span.Slice(i, WindowSize)); // Then, validate that the data was transcoded properly. for (int j = 0; j < WindowSize; j++) { Assert.Equal((ushort)latin1Span[i + j], (ushort)utf16Span[i + j]); } } }
static void RunSpanIndexOfTest(CompareInfo compareInfo, ReadOnlySpan <char> source, ReadOnlySpan <char> value, CompareOptions options, int expected) { using BoundedMemory <char> sourceBoundedMemory = BoundedMemory.AllocateFromExistingData(source); sourceBoundedMemory.MakeReadonly(); using BoundedMemory <char> valueBoundedMemory = BoundedMemory.AllocateFromExistingData(value); valueBoundedMemory.MakeReadonly(); Assert.Equal(expected, compareInfo.IndexOf(sourceBoundedMemory.Span, valueBoundedMemory.Span, options)); if (TryCreateRuneFrom(value, out Rune rune)) { Assert.Equal(expected, compareInfo.IndexOf(sourceBoundedMemory.Span, rune, options)); // try the Rune-based version } }
private static unsafe void GetIndexOfFirstInvalidUtf8Sequence_Test_Core(byte[] input, int expectedRetVal, int expectedRuneCount, int expectedSurrogatePairCount) { // Arrange using BoundedMemory <byte> boundedMemory = BoundedMemory.AllocateFromExistingData(input); boundedMemory.MakeReadonly(); // Act int actualRetVal; int actualSurrogatePairCount; int actualRuneCount; fixed(byte *pInputBuffer = &MemoryMarshal.GetReference(boundedMemory.Span)) { byte *pFirstInvalidByte = _getPointerToFirstInvalidByteFn.Value(pInputBuffer, input.Length, out int utf16CodeUnitCountAdjustment, out int scalarCountAdjustment); long ptrDiff = pFirstInvalidByte - pInputBuffer; Assert.True((ulong)ptrDiff <= (uint)input.Length, "ptrDiff was outside expected range."); Assert.True(utf16CodeUnitCountAdjustment <= 0, "UTF-16 code unit count adjustment must be 0 or negative."); Assert.True(scalarCountAdjustment <= 0, "Scalar count adjustment must be 0 or negative."); actualRetVal = (ptrDiff == input.Length) ? -1 : (int)ptrDiff; // The last two 'out' parameters are: // a) The number to be added to the "bytes processed" return value to come up with the total UTF-16 code unit count, and // b) The number to be added to the "total UTF-16 code unit count" value to come up with the total scalar count. int totalUtf16CodeUnitCount = (int)ptrDiff + utf16CodeUnitCountAdjustment; actualRuneCount = totalUtf16CodeUnitCount + scalarCountAdjustment; // Surrogate pair count is number of UTF-16 code units less the number of scalars. actualSurrogatePairCount = totalUtf16CodeUnitCount - actualRuneCount; } // Assert Assert.Equal(expectedRetVal, actualRetVal); Assert.Equal(expectedRuneCount, actualRuneCount); Assert.Equal(expectedSurrogatePairCount, actualSurrogatePairCount); }
unsafe static void RunSpanSortKeyTest(CompareInfo compareInfo, ReadOnlySpan <char> source, CompareOptions options, byte[] expectedSortKey) { using BoundedMemory <char> sourceBoundedMemory = BoundedMemory.AllocateFromExistingData(source); sourceBoundedMemory.MakeReadonly(); Assert.Equal(expectedSortKey.Length, compareInfo.GetSortKeyLength(sourceBoundedMemory.Span, options)); using BoundedMemory <byte> sortKeyBoundedMemory = BoundedMemory.Allocate <byte>(expectedSortKey.Length); // First try with a destination which is too small - should result in an error Assert.Throws <ArgumentException>("destination", () => compareInfo.GetSortKey(sourceBoundedMemory.Span, sortKeyBoundedMemory.Span.Slice(1), options)); // Next, try with a destination which is perfectly sized - should succeed Span <byte> sortKeyBoundedSpan = sortKeyBoundedMemory.Span; sortKeyBoundedSpan.Clear(); Assert.Equal(expectedSortKey.Length, compareInfo.GetSortKey(sourceBoundedMemory.Span, sortKeyBoundedSpan, options)); Assert.Equal(expectedSortKey, sortKeyBoundedSpan[0..expectedSortKey.Length].ToArray());
private static void ToBytes_Test_Core(ReadOnlySpan <char> utf16Input, int destinationSize, bool replaceInvalidSequences, bool isFinalChunk, OperationStatus expectedOperationStatus, int expectedNumCharsRead, ReadOnlySpan <byte> expectedUtf8Transcoding) { // Arrange using (BoundedMemory <char> boundedSource = BoundedMemory.AllocateFromExistingData(utf16Input)) using (BoundedMemory <byte> boundedDestination = BoundedMemory.Allocate <byte>(destinationSize)) { boundedSource.MakeReadonly(); // Act OperationStatus actualOperationStatus = Utf8.FromUtf16(boundedSource.Span, boundedDestination.Span, out int actualNumCharsRead, out int actualNumBytesWritten, replaceInvalidSequences, isFinalChunk); // Assert Assert.Equal(expectedOperationStatus, actualOperationStatus); Assert.Equal(expectedNumCharsRead, actualNumCharsRead); Assert.Equal(expectedUtf8Transcoding.Length, actualNumBytesWritten); Assert.Equal(expectedUtf8Transcoding.ToArray(), boundedDestination.Span.Slice(0, actualNumBytesWritten).ToArray()); } }
private static unsafe void GetIndexOfFirstInvalidUtf16Sequence_Test_Core(char[] input, int expectedRetVal, int expectedRuneCount, long expectedUtf8ByteCount) { // Arrange using BoundedMemory <char> boundedMemory = BoundedMemory.AllocateFromExistingData(input); boundedMemory.MakeReadonly(); // Act int actualRetVal; long actualUtf8CodeUnitCount; int actualRuneCount; fixed(char *pInputBuffer = &MemoryMarshal.GetReference(boundedMemory.Span)) { char *pFirstInvalidChar = _getPointerToFirstInvalidCharFn.Value(pInputBuffer, input.Length, out long utf8CodeUnitCountAdjustment, out int scalarCountAdjustment); long ptrDiff = pFirstInvalidChar - pInputBuffer; Assert.True((ulong)ptrDiff <= (uint)input.Length, "ptrDiff was outside expected range."); Assert.True(utf8CodeUnitCountAdjustment >= 0, "UTF-16 code unit count adjustment must be non-negative."); Assert.True(scalarCountAdjustment <= 0, "Scalar count adjustment must be 0 or negative."); actualRetVal = (ptrDiff == input.Length) ? -1 : (int)ptrDiff; // The last two 'out' parameters are: // a) The number to be added to the "chars processed" return value to come up with the total UTF-8 code unit count, and // b) The number to be added to the "total UTF-16 code unit count" value to come up with the total scalar count. actualUtf8CodeUnitCount = ptrDiff + utf8CodeUnitCountAdjustment; actualRuneCount = (int)ptrDiff + scalarCountAdjustment; } // Assert Assert.Equal(expectedRetVal, actualRetVal); Assert.Equal(expectedRuneCount, actualRuneCount); Assert.Equal(actualUtf8CodeUnitCount, expectedUtf8ByteCount); }
public static void NarrowUtf16ToLatin1_AllLatin1Input() { using BoundedMemory <char> utf16Mem = BoundedMemory.Allocate <char>(128); using BoundedMemory <byte> latin1Mem = BoundedMemory.Allocate <byte>(128); // Fill the source with [deterministic] pseudo-random chars U+0000..U+00FF, then make readonly. Random rnd = new Random(0x54321); Span <char> utf16Span = utf16Mem.Span; for (int i = 0; i < utf16Span.Length; i++) { utf16Span[i] = (char)(byte)rnd.Next(); } utf16Mem.MakeReadonly(); // We'll write to the Latin-1 span. // We test with a variety of span lengths to test alignment and fallthrough code paths. Span <byte> latin1Span = latin1Mem.Span; for (int i = 0; i < utf16Span.Length; i++) { latin1Span.Clear(); // remove any data from previous iteration // First, validate that the workhorse saw the incoming data as all-Latin-1. Assert.Equal(128 - i, CallNarrowUtf16ToLatin1(utf16Span.Slice(i), latin1Span.Slice(i))); // Then, validate that the data was transcoded properly. for (int j = i; j < 128; j++) { Assert.Equal((ushort)utf16Span[i], (ushort)latin1Span[i]); } } }
public static void NarrowUtf16ToAscii_AllAsciiInput() { using BoundedMemory <char> utf16Mem = BoundedMemory.Allocate <char>(128); using BoundedMemory <byte> asciiMem = BoundedMemory.Allocate <byte>(128); // Fill source with 00 .. 7F. Span <char> utf16Span = utf16Mem.Span; for (int i = 0; i < utf16Span.Length; i++) { utf16Span[i] = (char)i; } utf16Mem.MakeReadonly(); // We'll write to the ASCII span. // We test with a variety of span lengths to test alignment and fallthrough code paths. Span <byte> asciiSpan = asciiMem.Span; for (int i = 0; i < utf16Span.Length; i++) { asciiSpan.Clear(); // remove any data from previous iteration // First, validate that the workhorse saw the incoming data as all-ASCII. Assert.Equal(128 - i, CallNarrowUtf16ToAscii(utf16Span.Slice(i), asciiSpan.Slice(i))); // Then, validate that the data was transcoded properly. for (int j = i; j < 128; j++) { Assert.Equal((ushort)utf16Span[i], (ushort)asciiSpan[i]); } } }