void VerifyUserAgentHeader(HttpRequestMessage request)
{
var expected = Utf8.ToString(HttpHeader.Common.CreateUserAgent("Azure.Configuration", "1.0.0").Value);
Assert.True(request.Headers.Contains("User-Agent"));
var userAgentValues = request.Headers.GetValues("User-Agent");
foreach (var value in userAgentValues)
{
if (expected.StartsWith(value))
{
return;
}
}
Assert.Fail("could not find User-Agent header value " + expected);
}
public static byte[]? ToUTF8NullTerminated(ReadOnlySpan <char> str)
{
if (str.IsEmpty)
{
return(null);
}
int maxByteCount = str.Length * sizeof(char);
byte[] data = new byte[maxByteCount + 1]; //Extra charactor acts as the null terminator
var res = Utf8.FromUtf16(str, data.AsSpan(0, maxByteCount), out _, out _);
Debug.Assert(res == OperationStatus.Done);
return(data);
}
public void EnumerateArrayOfString()
{
JwtHeaderDocument.TryParseHeader(Utf8.GetBytes("{\"array\":[\"text1\", \"text2\", \"text3\", \"text4\", \"text5\"]}"), null, TokenValidationPolicy.NoValidation, out var header, out _);
header.TryGetHeaderParameter("array", out var element);
var enumerator = element.EnumerateArray <string>();
enumerator.MoveNext();
Assert.Equal("text1", enumerator.Current.GetString());
enumerator.MoveNext();
Assert.Equal("text2", enumerator.Current.GetString());
enumerator.MoveNext();
Assert.Equal("text3", enumerator.Current.GetString());
enumerator.MoveNext();
Assert.Equal("text4", enumerator.Current.GetString());
enumerator.MoveNext();
Assert.Equal("text5", enumerator.Current.GetString());
}
public void Pbkdf2_DeriveKey()
{
var salt = new byte[16] {
217, 96, 147, 112, 150, 117, 70, 247, 127, 8, 155, 137, 174, 42, 80, 215
};
var password = Utf8.GetBytes(_password);
using var pbkdf2_managed = new Rfc2898DeriveBytes(password, salt, 4096, HashAlgorithmName.SHA256);
var result1 = pbkdf2_managed.GetBytes(16);
Span <byte> result2 = stackalloc byte[16];
Pbkdf2.DeriveKey(password, salt, Sha256.Shared, 4096, result2);
Assert.Equal(result1, result2.ToArray());
}
private Dictionary <string, List <string> > BuildConcordanceOfHeadwords()
{
var concordance = new Dictionary <string, List <string> >();
string[] lineSplit;
foreach (string line in _lines)
{
lineSplit = line.Split('â•‘');
if (lineSplit.Length < 2)
{
continue;
}
string[] content = lineSplit[1].Split(' ');
foreach (string cjk in content)
{
if (!_dbTerms.Contains(cjk))
{
foreach (string c in Utf8.AsCodePoints(cjk))
{
if (!_nonCjk.Contains(c))
{
if (!concordance.ContainsKey(c))
{
concordance[c] = new List <string>();
}
concordance[c].Add(line);
}
}
}
else
{
if (!_nonCjk.Contains(cjk))
{
if (!concordance.ContainsKey(cjk))
{
concordance[cjk] = new List <string>();
}
concordance[cjk].Add(line);
}
}
}
}
return(concordance);
}
public void EnumerateArrayOfInteger()
{
JwtHeaderDocument.TryParseHeader(Utf8.GetBytes("{\"array\":[1, 2, 3, 4, 5]}"), null, TokenValidationPolicy.NoValidation, out var header, out _);
header.TryGetHeaderParameter("array", out var element);
var enumerator = element.EnumerateArray <long>();
enumerator.MoveNext();
Assert.Equal(1, enumerator.Current.GetInt64());
enumerator.MoveNext();
Assert.Equal(2, enumerator.Current.GetInt64());
enumerator.MoveNext();
Assert.Equal(3, enumerator.Current.GetInt64());
enumerator.MoveNext();
Assert.Equal(4, enumerator.Current.GetInt64());
enumerator.MoveNext();
Assert.Equal(5, enumerator.Current.GetInt64());
}
protected internal override bool TryGetHeader(ReadOnlySpan <byte> name, out ReadOnlySpan <byte> value)
{
string nameString = Utf8.ToString(name);
if (!_responseMessage.Headers.TryGetValues(nameString, out var values))
{
if (!_responseMessage.Content.Headers.TryGetValues(nameString, out values))
{
value = default;
return(false);
}
}
var all = string.Join(",", values);
value = Encoding.ASCII.GetBytes(all);
return(true);
}
public Header(ReadOnlySpan <byte> name, string value)
{
int length = name.Length + value.Length + 3;
_utf8 = new byte[length];
var utf8 = _utf8.AsSpan();
name.CopyTo(_utf8);
_utf8[name.Length] = (byte)':';
utf8 = utf8.Slice(name.Length + 1);
if (Utf8.FromUtf16(MemoryMarshal.AsBytes(value.AsSpan()), utf8, out var consumed, out int written) != OperationStatus.Done)
{
throw new Exception("value is not ASCII");
}
_utf8[length - 2] = (byte)'\r';
_utf8[length - 1] = (byte)'\n';
}
public static Rune FromUtf8(byte[] value, ref int index)
{
if (index >= value.Length)
{
throw new ArgumentOutOfRangeException("index", "Index out of bounds.");
}
var leading = value[index++];
if (!Utf8.ProcessLeading(leading, out var result, out var bytesRemaining))
{
while (Utf8.ProcessTrailing(leading, ref result, ref bytesRemaining))
{
;
}
}
return(new Rune(result));
}
public override int Peek()
{
if (_index >= _buffer.Length)
{
return(-1);
}
var utf8Unread = _buffer.Span.Slice(_index);
char result = default(char);
unsafe
{
var destination = new Span <char>(&result, 1).AsBytes();
if (Utf8.ToUtf16(utf8Unread, destination, out int consumed, out int written) == TransformationStatus.InvalidData)
{
throw new Exception("invalid UTF8 byte at " + _index.ToString());
}
}
return(result);
}
public static bool TryFormat(byte value, Span <byte> buffer, out int bytesWritten, ParsedFormat format = default, SymbolTable symbolTable = null)
{
if (format.IsDefault)
{
format = 'G';
}
if (symbolTable == null || symbolTable == SymbolTable.InvariantUtf8)
{
return(Utf8.TryFormat(value, buffer, out bytesWritten, format));
}
else if (symbolTable == SymbolTable.InvariantUtf16)
{
return(Utf16.TryFormat(value, buffer, out bytesWritten, format));
}
else
{
return(TryFormatUInt64(value, buffer, out bytesWritten, format, symbolTable));
}
}
private static bool TryDecodeUtf8(ReadOnlySpan <byte> input, out string decoded)
{
char[] rented = ArrayPool <char> .Shared.Rent(input.Length);
try
{
if (Utf8.ToUtf16(input, rented, out _, out int charsWritten, replaceInvalidSequences: false) == OperationStatus.Done)
{
decoded = new string(rented, 0, charsWritten);
return(true);
}
}
finally
{
ArrayPool <char> .Shared.Return(rented);
}
decoded = null;
return(false);
}
public void Wrap_Rfc7518_Appendix_C()
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
{
return;
}
var kwp = new EcdhKeyWrapper(_bobKey, EncryptionAlgorithm.A128Gcm, KeyManagementAlgorithm.EcdhEs);
var header = new JwtHeader
{
{ JwtHeaderParameterNames.Apu, Utf8.GetString(Base64Url.Encode("Alice")) },
{ JwtHeaderParameterNames.Apv, Utf8.GetString(Base64Url.Encode("Bob")) }
};
Span <byte> wrappedKey = stackalloc byte[kwp.GetKeyWrapSize()];
var cek = kwp.WrapKey(_aliceKey, header, wrappedKey);
var expected = new byte[] { 86, 170, 141, 234, 248, 35, 109, 32, 92, 34, 40, 205, 113, 167, 16, 26 };
Assert.Equal(expected, cek.AsSpan().ToArray());
}
/// <summary>
/// Converts an email string to a Gravatar-compatible MD5 hash. See https://en.gravatar.com/site/implement/hash/
/// </summary>
public static string HashEmail(string email)
{
// 1) Trim leading and trailing space characters.
email = email.Trim(' ');
if (email == "")
{
return("");
}
// 2) Force all characters to lowercase.
email = email.ToLowerInvariant();
// 3) (assumed) UTF8-encode.
var bytes = Utf8.GetBytes(email);
// 4) MD5 hash
var hash = Md5.ComputeHash(bytes);
// 5) (assumed from example) Convert to lowercase hex string.
return(hash.ToLowercaseHexString());
}
public static void LogRequest(this Log log, HttpRequest request, ReadOnlySequence <byte> body)
{
if (log.IsVerbose)
{
// TODO: this is much ceremony. We need to do something with this. ReadOnlyBytes.AsUtf8 maybe?
log.LogMessage(Log.Level.Verbose, "\tMethod: {0}", request.Method);
log.LogMessage(Log.Level.Verbose, "\tRequest-URI: {0}", request.Path);
log.LogMessage(Log.Level.Verbose, "\tHTTP-Version: {0}", request.Version);
log.LogMessage(Log.Level.Verbose, "\tHttp Headers:");
var headers = request.Headers;
for (int i = 0; i < headers.Length; i++)
{
var header = headers[i];
log.LogMessage(Log.Level.Verbose, "\t\t{0}: {1}", Utf8.ToString(header.Name), Utf8.ToString(header.Value));
}
// TODO: Logger should support Utf8Span
log.LogMessage(Log.Level.Verbose, Utf8.ToString(body.First.Span));
}
}
private static void ToBytes_Test_Core(ReadOnlySpan <char> utf16Input, int destinationSize, bool replaceInvalidSequences, bool isFinalChunk, OperationStatus expectedOperationStatus, int expectedNumCharsRead, ReadOnlySpan <byte> expectedUtf8Transcoding)
{
// Arrange
using (BoundedMemory <char> boundedSource = BoundedMemory.AllocateFromExistingData(utf16Input))
using (BoundedMemory <byte> boundedDestination = BoundedMemory.Allocate <byte>(destinationSize))
{
boundedSource.MakeReadonly();
// Act
OperationStatus actualOperationStatus = Utf8.FromUtf16(boundedSource.Span, boundedDestination.Span, out int actualNumCharsRead, out int actualNumBytesWritten, replaceInvalidSequences, isFinalChunk);
// Assert
Assert.Equal(expectedOperationStatus, actualOperationStatus);
Assert.Equal(expectedNumCharsRead, actualNumCharsRead);
Assert.Equal(expectedUtf8Transcoding.Length, actualNumBytesWritten);
Assert.Equal(expectedUtf8Transcoding.ToArray(), boundedDestination.Span.Slice(0, actualNumBytesWritten).ToArray());
}
}
async Task ReceiveAsync()
{
PipeWriter writer = _responsePipe.Writer;
try
{
while (true)
{
// just wait for data in the socket
await ReadFromSocketAsync(Memory <byte> .Empty);
while (HasData)
{
Memory <byte> buffer = writer.GetMemory();
int readBytes = await ReadFromSocketAsync(buffer).ConfigureAwait(false);
if (readBytes == 0)
{
break;
}
if (Log != null && Log.Switch.ShouldTrace(TraceEventType.Verbose))
{
string data = Utf8.ToString(buffer.Span.Slice(0, readBytes));
if (!string.IsNullOrWhiteSpace(data))
{
Log.TraceInformation(data);
}
}
writer.Advance(readBytes);
await writer.FlushAsync();
}
}
}
finally
{
writer.Complete();
}
}
public void Log(string sawmillName, LogEvent message)
{
var robustLevel = message.Level.ToRobust();
lock (_stream)
{
_line
.Clear()
.Append(LogLevelToString(robustLevel))
.Append(sawmillName)
.Append(": ")
.AppendLine(message.RenderMessage());
if (message.Exception != null)
{
_line.AppendLine(message.Exception.ToString());
}
// ReSharper disable once SuggestVarOrType_Elsewhere
if (!_isUtf16Out)
{
Span <byte> buf = stackalloc byte[1024];
var totalChars = _line.Length;
foreach (var chunk in _line.GetChunks())
{
var chunkSize = chunk.Length;
var totalRead = 0;
var span = chunk.Span;
for (;;)
{
var finalChunk = totalRead + chunkSize >= totalChars;
Utf8.FromUtf16(span, buf, out var read, out var wrote, isFinalBlock: finalChunk);
_stream.Write(buf.Slice(0, wrote));
totalRead += read;
if (read >= chunkSize)
{
break;
}
span = span[read..];
static void JsonSerialize_Helper(ReadOnlySpan <char> input, IBufferWriter <byte> writer, Span <byte> scratch1, Span <byte> scratch2)
{
// Transcode UTF-16 -> UTF-8 in a loop.
var utf16InputToEncode = input;
var utf8TranscodingOperationStatus = default(OperationStatus);
do
{
utf8TranscodingOperationStatus = Utf8.ConvertFromUtf16(utf16InputToEncode, scratch1, out var utf16CharsConsumed, out var utf8BytesTranscoded);
if (utf8TranscodingOperationStatus == OperationStatus.InvalidData)
{
throw new Exception("Bad input.");
}
if (utf16CharsConsumed == 0 && utf8TranscodingOperationStatus != OperationStatus.Done)
{
throw new Exception("Unexpected return value. Bailing so we don't end up in an infinite loop.");
}
utf16InputToEncode = utf16InputToEncode.Slice(utf16CharsConsumed);
// Now that we've properly transcoded UTF-16 -> UTF-8, JSON-encode it and write it to the output.
var utf8JsonInputToEncode = scratch1.Slice(0, utf8BytesTranscoded);
var jsonEncodingOperationStatus = default(OperationStatus);
do
{
jsonEncodingOperationStatus = JsonEncoder.Default.Encode(utf8JsonInputToEncode, scratch2, out var utf8BytesConsumed, out var jsonEncodedBytesWritten);
if (jsonEncodingOperationStatus == OperationStatus.InvalidData)
{
throw new Exception("Bad input.");
}
if (utf8BytesConsumed == 0 && jsonEncodingOperationStatus != OperationStatus.Done)
{
throw new Exception("Unexpected return value. Bailing so we don't end up in an infinite loop.");
}
utf8JsonInputToEncode = utf8JsonInputToEncode.Slice(utf8BytesConsumed);
writer.Write(scratch2.Slice(0, jsonEncodedBytesWritten));
} while (jsonEncodingOperationStatus != OperationStatus.Done);
} while (utf8TranscodingOperationStatus != OperationStatus.Done);
}
public static unsafe string BufferToString(ReadOnlySpan <byte> buffer)
{
int size = buffer.IndexOf(byte.MinValue);
if (size == 0)
{
return(string.Empty);
}
if ((uint)size < (uint)buffer.Length)
{
buffer = buffer.Slice(0, size);
}
if (buffer.Length <= 1024)
{
Span <char> tmp = stackalloc char[buffer.Length];
var res = Utf8.ToUtf16(buffer, tmp, out _, out int written);
Debug.Assert(res == OperationStatus.Done);
return(new string(tmp.Slice(0, written)));
}
else
{
var pool = ArrayPool <char> .Shared;
char[] arr = pool.Rent(buffer.Length);
var res = Utf8.ToUtf16(buffer, arr, out _, out int written);
Debug.Assert(res == OperationStatus.Done);
var ret = new string(arr, 0, written);
pool.Return(arr);
return(ret);
}
}
void RenderUstr(ustring ustr, int col, int line, int width)
{
int byteLen = ustr.Length;
int used = 0;
for (int i = 0; i < byteLen;)
{
(var rune, var size) = Utf8.DecodeRune(ustr, i, i - byteLen);
var count = Rune.ColumnWidth(rune);
if (used + count >= width)
{
break;
}
Driver.AddRune(rune);
used += count;
i += size;
}
for (; used < width; used++)
{
Driver.AddRune(' ');
}
}
public static void WritePrimitiveCore(Stream stream, string value)
{
if (value == null)
{
Primitives.WritePrimitive(stream, (uint)0);
return;
}
if (value.Length == 0)
{
Primitives.WritePrimitive(stream, (uint)1);
return;
}
Span <byte> buf = stackalloc byte[StringByteBufferLength];
var totalChars = value.Length;
var totalBytes = Encoding.UTF8.GetByteCount(value);
Primitives.WritePrimitive(stream, (uint)totalBytes + 1);
Primitives.WritePrimitive(stream, (uint)totalChars);
var totalRead = 0;
ReadOnlySpan <char> span = value;
for (;;)
{
var finalChunk = totalRead + totalChars >= totalChars;
Utf8.FromUtf16(span, buf, out var read, out var wrote, isFinalBlock: finalChunk);
stream.Write(buf.Slice(0, wrote));
totalRead += read;
if (read >= totalChars)
{
break;
}
span = span[read..];
// A slightly adapted method from gui.cs: https://github.com/migueldeicaza/gui.cs/blob/fc1faba7452ccbdf49028ac49f0c9f0f42bbae91/Terminal.Gui/Views/ListView.cs#L433-L461
private void RenderUstr(ConsoleDriver driver, ustring ustr, int col, int line, int width)
{
int used = 0;
int index = 0;
while (index < ustr.Length)
{
(var rune, var size) = Utf8.DecodeRune(ustr, index, index - ustr.Length);
var count = Rune.ColumnWidth(rune);
if (used + count > width)
{
break;
}
driver.AddRune(rune);
used += count;
index += size;
}
while (used < width)
{
driver.AddRune(' ');
used++;
}
}
// TODO: this should be moved to System.Text.Primitives. Probably to Utf8 class
static string Utf8ToString(ReadOnlySpan <byte> utf8)
{
var result = Utf8.ToUtf16Length(utf8, out int bytesNeeded);
if (result == TransformationStatus.InvalidData || result == TransformationStatus.NeedMoreSourceData)
{
throw new Exception("invalid UTF8 byte");
}
var str = new string(' ', bytesNeeded / sizeof(char));
unsafe
{
fixed(char *pStr = str)
{
var strSpan = new Span <char>(pStr, str.Length);
if (Utf8.ToUtf16(utf8, strSpan.AsBytes(), out int consumed, out int written) != TransformationStatus.Done)
{
throw new Exception();
}
}
}
return(str);
}
public string ReadString(int len = 4)
{
var stringLength = ReadInt32(len);
return(Utf8.Convert(ReadBytes(stringLength)));
}
public override string ToString()
{
return(Utf8.Utf8ToString() + "/" + Bucket.ToString("0.0", CultureInfo.InvariantCulture));
}
private byte[] BuildSecretAppend(string?apuS, string?apvS)
{
byte[]? apuToReturn = null;
byte[]? apvToReturn = null;
byte[] secretAppend;
try
{
int apuLength = apuS == null ? 0 : Utf8.GetMaxByteCount(apuS.Length);
Span <byte> apu = apuLength <= Constants.MaxStackallocBytes
? stackalloc byte[apuLength]
: (apuToReturn = ArrayPool <byte> .Shared.Rent(apuLength));
if (apuS != null)
{
apuLength = Utf8.GetBytes(apuS, apu);
apu = apu.Slice(0, apuLength);
}
int apvLength = apvS == null ? 0 : Utf8.GetMaxByteCount(apvS.Length);
Span <byte> apv = apvLength <= Constants.MaxStackallocBytes
? stackalloc byte[apvLength]
: (apvToReturn = ArrayPool <byte> .Shared.Rent(apvLength));
if (apvS != null)
{
apvLength = Utf8.GetBytes(apvS, apv);
apv = apv.Slice(0, apvLength);
}
apuLength = Base64Url.GetArraySizeRequiredToDecode(apuLength);
apvLength = Base64Url.GetArraySizeRequiredToDecode(apvLength);
int algorithmLength = sizeof(int) + _algorithmNameLength;
int partyUInfoLength = sizeof(int) + apuLength;
int partyVInfoLength = sizeof(int) + apvLength;
const int suppPubInfoLength = sizeof(int);
int secretAppendLength = algorithmLength + partyUInfoLength + partyVInfoLength + suppPubInfoLength;
secretAppend = new byte[secretAppendLength];
var secretAppendSpan = secretAppend.AsSpan();
WriteAlgorithmId(secretAppendSpan);
secretAppendSpan = secretAppendSpan.Slice(algorithmLength);
WritePartyInfo(apu, apuLength, secretAppendSpan);
secretAppendSpan = secretAppendSpan.Slice(partyUInfoLength);
WritePartyInfo(apv, apvLength, secretAppendSpan);
secretAppendSpan = secretAppendSpan.Slice(partyVInfoLength);
BinaryPrimitives.WriteInt32BigEndian(secretAppendSpan, _keySizeInBytes << 3);
}
finally
{
if (apuToReturn != null)
{
ArrayPool <byte> .Shared.Return(apuToReturn);
}
if (apvToReturn != null)
{
ArrayPool <byte> .Shared.Return(apvToReturn);
}
}
return(secretAppend);
}
/// <summary> /// Converts a span containing a sequence of UTF-8 bytes into UTF-32 bytes. /// /// This method will consume as many of the input bytes as possible. /// /// On successful exit, the entire input was consumed and encoded successfully. In this case, <paramref name="bytesConsumed"/> will be /// equal to the length of the <paramref name="source"/> and <paramref name="bytesWritten"/> will equal the total number of bytes written to /// the <paramref name="destination"/>. /// </summary> /// <param name="source">A span containing a sequence of UTF-8 bytes.</param> /// <param name="destination">A span to write the UTF-32 bytes into.</param> /// <param name="bytesConsumed">On exit, contains the number of bytes that were consumed from the <paramref name="source"/>.</param> /// <param name="bytesWritten">On exit, contains the number of bytes written to <paramref name="destination"/></param> /// <returns>A <see cref="OperationStatus"/> value representing the state of the conversion.</returns> public static OperationStatus FromUtf8(ReadOnlySpan <byte> source, Span <byte> destination, out int bytesConsumed, out int bytesWritten) => Utf8.ToUtf32(source, destination, out bytesConsumed, out bytesWritten);
/// <summary> /// Calculates the byte count needed to encode the UTF-32 bytes from the specified UTF-8 sequence. /// /// This method will consume as many of the input bytes as possible. /// </summary> /// <param name="source">A span containing a sequence of UTF-8 bytes.</param> /// <param name="bytesNeeded">On exit, contains the number of bytes required for encoding from the <paramref name="source"/>.</param> /// <returns>A <see cref="OperationStatus"/> value representing the expected state of the conversion.</returns> public static OperationStatus FromUtf8Length(ReadOnlySpan <byte> source, out int bytesNeeded) => Utf8.ToUtf32Length(source, out bytesNeeded);
public SignatureKeyData(string name, byte[] signature)
{
AlgorithmName = Utf8.GetBytes(name);
Signature = signature;
}