public static Rune FromString(string value, ref int index)
{
if (value == null)
{
throw new ArgumentNullException("value");
}
if (index >= value.Length)
{
throw new ArgumentOutOfRangeException("index", "Index is out of range of string.");
}
char c = value[index++];
if (Utf16.IsLeadingSurrogate(c))
{
if (index >= value.Length)
{
throw new ArgumentOutOfRangeException("index", "Index is out of range of string.");
}
char d = value[index++];
return(Rune.FromUtf16(c, d));
}
else
{
return(Rune.FromUtf16(c));
}
}
public static Rune FromUtf16(ushort[] array, ref int index)
{
if (index >= array.Length)
{
throw new ArgumentOutOfRangeException("index", "Index out of bounds.");
}
var value = array[index++];
if (Utf16.IsLeadingSurrogate(value))
{
if (index >= array.Length)
{
throw new ArgumentOutOfRangeException("index", "Need two items to process surrogate pair.");
}
var trailing = array[index++];
return(FromUtf16(value, trailing));
}
else
{
return(new Rune(value));
}
}
public static Rune FromUtf16(byte[] array, ref int index, ByteOrder endianness = ByteOrder.LittleEndian)
{
var decoder = new Utf16.ByteDecoder(endianness);
if (index + 2 > array.Length)
{
throw new ArgumentOutOfRangeException("index", "Need at least 2 bytes to process Utf16");
}
decoder.Process(array[index++]);
var result = decoder.Process(array[index++]);
if (result == null)
{
if (index + 2 > array.Length)
{
throw new ArgumentOutOfRangeException("index", "Need 4 bytes to process Utf16 surrogate pair");
}
decoder.Process(array[index++]);
result = decoder.Process(array[index++]);
if (result == null)
{
throw new RuneException(Utf16.MissingTrailingSurrogateMessage());
}
}
return(new Rune(result.GetValueOrDefault()));
}
public void TestNonBasic()
{
uint code;
Assert.AreEqual(2, Utf16.ToCode(out code, 0xD834, 0xDD1E));
Assert.AreEqual((uint)0x1D11E, code);
}
public static Rune FromUtf16(ushort leadingSurrogate, ushort trailingSurrogate)
{
uint utf32;
utf32 = Utf16.ToUtf32(leadingSurrogate, trailingSurrogate);
return(new Rune(utf32));
}
public void TestLatinCharacter()
{
uint code;
Assert.AreEqual(1, Utf16.ToCode(out code, 0x7A));
Assert.AreEqual((uint)0x7A, code);
Assert.AreEqual(1, Utf16.ToCode(out code, 0x7A, 0xABCD), "Trail is ignored");
Assert.AreEqual((uint)0x7A, code);
}
public void TestHighSingleComponent()
{
uint code;
Assert.AreEqual(1, Utf16.ToCode(out code, 0x6C34));
Assert.AreEqual((uint)0x6C34, code);
Assert.AreEqual(1, Utf16.ToCode(out code, 0x6C34, 0xABCD), "Trail is ignored");
Assert.AreEqual((uint)0x6C34, code);
}
public void TestHighSingleComponent()
{
ushort lead, trail;
Assert.AreEqual(1, Utf16.FromCode(0x6c34, out lead, out trail));
Assert.AreEqual(0x6c34, lead);
Assert.AreEqual(0, trail);
Assert.AreEqual("\u6C34", Utf16.FromCode(0x6C34));
}
public void TestNonBasic()
{
ushort lead, trail;
Assert.AreEqual(2, Utf16.FromCode(0x1D11E, out lead, out trail));
Assert.AreEqual(0xD834, lead);
Assert.AreEqual(0xDD1E, trail);
Assert.AreEqual("\uD834\uDD1E", Utf16.FromCode(0x1D11E));
}
public void TestFromLatinCharacter()
{
ushort lead, trail;
Assert.AreEqual(1, Utf16.FromCode(0x7a, out lead, out trail));
Assert.AreEqual(0x7a, lead);
Assert.AreEqual(0, trail);
Assert.AreEqual("\u007A", Utf16.FromCode(0x7A));
}
/// <summary>
/// Attempts to create a <see cref="Rune"/> from the provided input value.
/// </summary>
public static Boolean TryCreate(Char ch, out Rune result)
{
UInt32 extendedValue = ch;
if (!Utf16.IsSurrogate(extendedValue))
{
result = new Rune(extendedValue);
return(true);
}
else
{
result = default;
return(false);
}
}
public static bool TryFormat(DateTimeOffset value, Span <byte> buffer, out int bytesWritten, ParsedFormat format = default, SymbolTable symbolTable = null)
{
if (symbolTable == null || symbolTable == SymbolTable.InvariantUtf8)
{
return(Utf8.TryFormat(value, buffer, out bytesWritten, format));
}
else if (symbolTable == SymbolTable.InvariantUtf16)
{
return(Utf16.TryFormat(value, buffer, out bytesWritten, format));
}
else
{
throw new NotSupportedException();
}
}
/// <summary>
/// Reads plain text from the input stream.
/// </summary>
public static Span ReadSpan(InputBuffer buffer)
{
var span = new StringBuilder();
while (true)
{
var text = buffer.ConsumeUntil(b => _escapeChars.Contains(b));
span.Append(Encoding.UTF8.GetString(text, 0, text.Length));
byte?next;
if (buffer.At(0) == '\\' && (next = buffer.At(1)).HasValue)
{
ControlWord word;
int consumed;
if (ParseControlWord(buffer, 0, out word, out consumed))
{
if (word.Text == "u" && word.Parameter.HasValue && word.Parameter > 0)
{
// Handle unicode characters.
span.Append(Utf16.FromCode((uint)word.Parameter));
buffer.Discard(consumed + 1);
}
else
{
break;
}
}
else if (!IsAsciiLetter(next))
{
// Handle control symbols.
// TODO: Translate control symbol to special character, e.g. \~ => nbsp.
span.Append((char)next.Value);
buffer.Discard(2);
}
else
{
break;
}
}
else
{
break;
}
}
return(new Span(span.ToString()));
}
private static Int32 ReadRuneFromString(String input, Int32 index)
{
if (input is null)
{
throw new ArgumentNullException(nameof(input));
}
if ((UInt32)index >= (UInt32)input !.Length)
{
throw new ArgumentOutOfRangeException(nameof(index));
}
// Optimistically assume input is within BMP.
UInt32 returnValue = input[index];
if (Utf16.IsSurrogate(returnValue))
{
if (!Utf16.IsHighSurrogate(returnValue))
{
return(-1);
}
// Treat 'returnValue' as the high surrogate.
//
// If this becomes a hot code path, we can skip the below bounds check by reading
// off the end of the string using unsafe code. Since strings are null-terminated,
// we're guaranteed not to read a valid low surrogate, so we'll fail correctly if
// the string terminates unexpectedly.
index++;
if ((UInt32)index >= (UInt32)input.Length)
{
return(-1); // not an argument exception - just a "bad data" failure
}
UInt32 potentialLowSurrogate = input[index];
if (!Utf16.IsLowSurrogate(potentialLowSurrogate))
{
return(-1);
}
returnValue = Unsafe.Utf16Decode(returnValue, potentialLowSurrogate);
}
return((Int32)returnValue);
}
private static Rune[] UncheckedToRuneArray(string value, int start, int amount)
{
if (start < 0 || start > value.Length)
{
throw new ArgumentOutOfRangeException("start");
}
if (amount < 0 || start + amount > value.Length)
{
throw new ArgumentOutOfRangeException("amount");
}
var runes = new Rune[amount];
int index = 0;
var decoder = new Utf16.CharDecoder();
bool result = true;
for (int i = start; i < amount; i++)
{
var utf32 = decoder.Process(value[i]);
if (utf32 != null)
{
runes[index++] = (Rune)utf32.GetValueOrDefault();
}
}
if (!result)
{
throw new RuneException(Utf16.MissingTrailingSurrogateMessage());
}
if (index < runes.Length)
{
Array.Resize(ref runes, index);
}
return(runes);
}
public override string ToString()
{
string result;
// Convert UTF32 -> UTF16
if (value < Utf32.SupplementaryPlanePrefix)
{
result = ((char)value).ToString();
}
else
{
Utf16.FromUtf32(value, out ushort leading, out ushort trailing);
result = "" + (char)leading + (char)trailing;
}
return(result);
}
public static bool TryFormat(byte value, Span <byte> buffer, out int bytesWritten, ParsedFormat format = default, SymbolTable symbolTable = null)
{
if (format.IsDefault)
{
format = 'G';
}
if (symbolTable == null || symbolTable == SymbolTable.InvariantUtf8)
{
return(Utf8.TryFormat(value, buffer, out bytesWritten, format));
}
else if (symbolTable == SymbolTable.InvariantUtf16)
{
return(Utf16.TryFormat(value, buffer, out bytesWritten, format));
}
else
{
return(TryFormatUInt64(value, buffer, out bytesWritten, format, symbolTable));
}
}
public unsafe static byte[] ComputeKeyBytes(string key)
{
const int bufferLength = 128;
byte *pBuffer = stackalloc byte[bufferLength];
int written, consumed;
var buffer = new Span <byte>(pBuffer, bufferLength);
if (Utf16.ToUtf8(key.AsReadOnlySpan().AsBytes(), buffer, out consumed, out written) != TransformationStatus.Done)
{
throw new NotImplementedException("need to resize buffer");
}
var keyBytes = new byte[64];
var result = Base64.Decode(buffer.Slice(0, written), keyBytes, out consumed, out written);
if (result != TransformationStatus.Done || written != 64)
{
throw new NotImplementedException("need to resize buffer");
}
return(keyBytes);
}
/// <summary>
/// Writes plain text content to the output stream.
/// </summary>
public void Span(string text)
{
if (text == null)
{
throw new ArgumentNullException("text");
}
int start, end;
for (start = 0, end = 0; end < text.Length; ++end)
{
if (_escapeChars.Contains(text[end]))
{
_writer.Write(text.Substring(start, end - start));
// Write control symbol.
_writer.Write('\\');
_writer.Write(text[end]);
start = end = end + 1;
}
else if (text[end] > 127)
{
_writer.Write(text.Substring(start, end - start));
// Write unicode character.
uint code;
start = end = end + Utf16.ToCode(out code, text, end);
_writer.Write(@"\u{0}?", code);
}
}
if (start < end)
{
_writer.Write(text.Substring(start, end - start));
}
}
// returns a negative number on failure
internal static int ReadFirstRuneFromUtf16Buffer(ReadOnlySpan <char> input)
{
if (input.IsEmpty)
{
return(-1);
}
// Optimistically assume input is within BMP.
uint returnValue = input[0];
if (Utf16.IsSurrogate(returnValue))
{
if (!Utf16.IsHighSurrogate(returnValue))
{
return(-1);
}
// Treat 'returnValue' as the high surrogate.
if (1 >= (uint)input.Length)
{
return(-1); // not an argument exception - just a "bad data" failure
}
uint potentialLowSurrogate = input[1];
if (!Utf16.IsLowSurrogate(potentialLowSurrogate))
{
return(-1);
}
returnValue = Unsafe.Utf16Decode(returnValue, potentialLowSurrogate);
}
return((int)returnValue);
}
private static IEnumerable <Rune> UncheckedToRunes(string value, int start, int amount)
{
if (start < 0 || start > value.Length)
{
throw new ArgumentOutOfRangeException("start");
}
if (amount < 0 || start + amount > value.Length)
{
throw new ArgumentOutOfRangeException("amount");
}
var runes = new Rune[amount];
var decoder = new Utf16.CharDecoder();
bool result = true;
for (int i = start; i < amount; i++)
{
var utf32 = decoder.Process(value[i]);
if (utf32 != null)
{
yield return((Rune)utf32.GetValueOrDefault());
}
}
if (!result)
{
throw new RuneException(Utf16.MissingTrailingSurrogateMessage());
}
}
/// <summary> /// Calculates the byte count needed to encode the UTF-8 bytes from the specified UTF-16 sequence. /// /// This method will consume as many of the input bytes as possible. /// </summary> /// <param name="source">A span containing a sequence of UTF-16 bytes.</param> /// <param name="bytesNeeded">On exit, contains the number of bytes required for encoding from the <paramref name="source"/>.</param> /// <returns>A <see cref="OperationStatus"/> value representing the expected state of the conversion.</returns> public static OperationStatus FromUtf16Length(ReadOnlySpan <byte> source, out int bytesNeeded) => Utf16.ToUtf8Length(source, out bytesNeeded);
public static bool TryWrite(Span <byte> output, Sha256 hash, string keyType, string verb, string resourceId, string resourceType, string tokenVersion, DateTime utc, out int bytesWritten)
{
int written, consumed, totalWritten = 0;
bytesWritten = 0;
Span <byte> buffer;
unsafe
{
var pBuffer = stackalloc byte[AuthenticationHeaderBufferSize];
buffer = new Span <byte>(pBuffer, AuthenticationHeaderBufferSize);
}
s_type.CopyTo(buffer);
totalWritten += s_type.Length;
var bufferSlice = buffer.Slice(totalWritten);
if (Utf16.ToUtf8(keyType.AsReadOnlySpan().AsBytes(), bufferSlice, out consumed, out written) != TransformationStatus.Done)
{
throw new NotImplementedException("need to resize buffer");
}
totalWritten += written;
bufferSlice = buffer.Slice(totalWritten);
s_ver.CopyTo(bufferSlice);
totalWritten += s_ver.Length;
bufferSlice = buffer.Slice(totalWritten);
if (Utf16.ToUtf8(tokenVersion.AsReadOnlySpan().AsBytes(), bufferSlice, out consumed, out written) != TransformationStatus.Done)
{
throw new NotImplementedException("need to resize buffer");
}
totalWritten += written;
bufferSlice = buffer.Slice(totalWritten);
s_sig.CopyTo(bufferSlice);
totalWritten += s_sig.Length;
var front = buffer.Slice(0, totalWritten);
var payload = buffer.Slice(totalWritten);
totalWritten = 0;
if (verb.Equals("GET", StringComparison.Ordinal) || verb.Equals("get", StringComparison.Ordinal))
{
s_get.CopyTo(payload);
totalWritten += s_get.Length;
}
else if (verb.Equals("POST", StringComparison.Ordinal) || verb.Equals("post", StringComparison.Ordinal))
{
s_post.CopyTo(payload);
totalWritten += s_post.Length;
}
else if (verb.Equals("DELETE", StringComparison.Ordinal) || verb.Equals("delete", StringComparison.Ordinal))
{
s_delete.CopyTo(payload);
totalWritten += s_delete.Length;
}
else
{
if (Utf16.ToUtf8(verb.AsReadOnlySpan().AsBytes(), payload, out consumed, out written) != TransformationStatus.Done)
{
throw new NotImplementedException("need to resize buffer");
}
if (Ascii.ToLowerInPlace(payload.Slice(0, written), out written) != TransformationStatus.Done)
{
throw new NotImplementedException("need to resize buffer");
}
payload[written] = (byte)'\n';
totalWritten += written + 1;
}
bufferSlice = payload.Slice(totalWritten);
if (Utf16.ToUtf8(resourceType.AsReadOnlySpan().AsBytes(), bufferSlice, out consumed, out written) != TransformationStatus.Done)
{
throw new NotImplementedException("need to resize buffer");
}
if (Ascii.ToLowerInPlace(bufferSlice.Slice(0, written), out written) != TransformationStatus.Done)
{
throw new NotImplementedException("need to resize buffer");
}
bufferSlice[written] = (byte)'\n';
totalWritten += written + 1;
bufferSlice = payload.Slice(totalWritten);
if (Utf16.ToUtf8(resourceId.AsReadOnlySpan().AsBytes(), bufferSlice, out consumed, out written) != TransformationStatus.Done)
{
throw new NotImplementedException("need to resize buffer");
}
bufferSlice[written] = (byte)'\n';
totalWritten += written + 1;
bufferSlice = payload.Slice(totalWritten);
if (!Text.Formatters.Utf8.TryFormat(utc, bufferSlice, out written, 'l'))
{
throw new NotImplementedException("need to resize buffer");
}
bufferSlice[written] = (byte)'\n';
totalWritten += written + 1;
bufferSlice = payload.Slice(totalWritten);
bufferSlice[0] = (byte)'\n';
totalWritten += 1;
hash.Append(buffer.Slice(front.Length, totalWritten));
hash.GetHash(buffer.Slice(front.Length, hash.OutputSize));
if (!Base64.EncodeInPlace(buffer.Slice(front.Length), hash.OutputSize, out written))
{
throw new NotImplementedException("need to resize buffer");
}
var len = front.Length + written;
if (!UrlEncoder.TryEncode(buffer.Slice(0, len), output, out bytesWritten))
{
bytesWritten = 0;
return(false);
}
return(true);
}
public Rune[] Stringier() => Utf16.Decode(Buffer);
public void IsFirstUnit(UInt16 unit, Boolean expected) => Assert.Equal(expected, Utf16.IsFirstUnit(unit));
public void NextUnitRange(UInt16 unit, UInt16 lowerExp, UInt16 upperExp)
{
Utf16.NextUnitRange(unit, out UInt16 lower, out UInt16 upper);
Assert.Equal(lowerExp, lower);
Assert.Equal(upperExp, upper);
}
public void SequenceLength(UInt16 unit, Int32 expected) => Assert.Equal(expected, Utf16.SequenceLength(unit));
/// <summary> /// Converts a span containing a sequence of UTF-16 bytes into UTF-8 bytes. /// /// This method will consume as many of the input bytes as possible. /// /// On successful exit, the entire input was consumed and encoded successfully. In this case, <paramref name="bytesConsumed"/> will be /// equal to the length of the <paramref name="source"/> and <paramref name="bytesWritten"/> will equal the total number of bytes written to /// the <paramref name="destination"/>. /// </summary> /// <param name="source">A span containing a sequence of UTF-16 bytes.</param> /// <param name="destination">A span to write the UTF-8 bytes into.</param> /// <param name="bytesConsumed">On exit, contains the number of bytes that were consumed from the <paramref name="source"/>.</param> /// <param name="bytesWritten">On exit, contains the number of bytes written to <paramref name="destination"/></param> /// <returns>A <see cref="OperationStatus"/> value representing the state of the conversion.</returns> public static OperationStatus FromUtf16(ReadOnlySpan <byte> source, Span <byte> destination, out int bytesConsumed, out int bytesWritten) => Utf16.ToUtf8(source, destination, out bytesConsumed, out bytesWritten);
public static bool TryWrite(Span <byte> output, Sha256 hash, string verb, string canonicalizedResource, DateTime utc, out int bytesWritten)
{
int written, consumed;
bytesWritten = 0;
if (verb.Equals("GET", StringComparison.Ordinal))
{
if (output.Length < 3)
{
bytesWritten = 0;
return(false);
}
s_GET.CopyTo(output);
bytesWritten += s_GET.Length;
}
else
{
if (Utf16.ToUtf8(verb.AsReadOnlySpan().AsBytes(), output, out consumed, out written) != TransformationStatus.Done)
{
bytesWritten = 0;
return(false);
}
output[written] = (byte)'\n';
bytesWritten += written + 1;
}
var free = output.Slice(bytesWritten);
s_emptyHeaders.CopyTo(free);
bytesWritten += s_emptyHeaders.Length;
free = output.Slice(bytesWritten);
if (!Text.Formatters.Utf8.TryFormat(utc, free, out written, 'R'))
{
bytesWritten = 0;
return(false);
}
free[written] = (byte)'\n';
bytesWritten += written + 1;
free = output.Slice(bytesWritten);
if (Utf16.ToUtf8(canonicalizedResource.AsReadOnlySpan().AsBytes(), free, out consumed, out written) != TransformationStatus.Done)
{
bytesWritten = 0;
return(false);
}
bytesWritten += written;
var formatted = output.Slice(0, bytesWritten);
hash.Append(formatted);
hash.GetHash(output.Slice(0, hash.OutputSize));
if (!Base64.EncodeInPlace(output, hash.OutputSize, out written))
{
bytesWritten = 0;
return(false);
}
bytesWritten = written;
return(true);
}
// char... -> Rune...
public static IEnumerable <Rune> ToRunes(this IEnumerable <char> chars)
{
var decoder = new Utf16.CharDecoder();
return(DecodeValues(chars, decoder, Utf16.MissingTrailingSurrogateMessage()));
}
