static unsafe void WriteUtf8String(string str, IntPtr dest, int length = 256)
{
Span <byte> buffer = new Span <byte>(dest.ToPointer(), length);
Utf8.FromUtf16(str, buffer, out _, out var count);
buffer[count] = 0;
}
public static Result GetCaseSensitivePath(out int bytesWritten, Span <byte> buffer, U8Span path,
U8Span workingDirectoryPath)
{
UnsafeHelpers.SkipParamInit(out bytesWritten);
string pathUtf16 = StringUtils.Utf8ZToString(path);
string workingDirectoryPathUtf16 = StringUtils.Utf8ZToString(workingDirectoryPath);
Result rc = GetCaseSensitivePathFull(out string caseSensitivePath, out int rootPathLength, pathUtf16,
workingDirectoryPathUtf16);
if (rc.IsFailure())
{
return(rc);
}
OperationStatus status = Utf8.FromUtf16(caseSensitivePath.AsSpan(rootPathLength),
buffer.Slice(0, buffer.Length - 1), out _, out int utf8BytesWritten);
if (status == OperationStatus.DestinationTooSmall)
{
return(ResultFs.TooLongPath.Log());
}
if (status == OperationStatus.InvalidData || status == OperationStatus.NeedMoreData)
{
return(ResultFs.InvalidCharacter.Log());
}
buffer[utf8BytesWritten] = NullTerminator;
bytesWritten = utf8BytesWritten;
return(Result.Success);
}
/// <summary>
/// Writes byte blocks of UTF-8 characters from a span of characters.
/// </summary>
public OperationStatus Write(ReadOnlySpan <char> source, out int charsRead, bool replaceInvalidSequences = true)
{
if (Buffer == null)
{
throw new InvalidOperationException();
}
charsRead = 0;
if (source.IsEmpty)
{
return(OperationStatus.Done);
}
Span <byte> utf8Buffer = stackalloc byte[4096];
do
{
var status = Utf8.FromUtf16(source, utf8Buffer, out int read, out int written, replaceInvalidSequences);
if (status == OperationStatus.InvalidData)
{
return(status);
}
Buffer.WriteVar(written);
Buffer.Write(utf8Buffer.Slice(0, written));
charsRead += read;
source = source[read..];
/// <summary>
/// This is basically the same algorithm as in the t4 template to create the methods
/// It's necessary to update both
/// </summary>
private static ConstantExpression[] GetIntegersForMemberName(string formattedName)
{
var result = new List <ConstantExpression>();
var length = Encoding.UTF8.GetByteCount(formattedName);
if (length > 23)
{
return(null); // fallback
}
var bytes = new byte[length + 1];
bytes[0] = CborWriter.EncodeType(CborType.TextString, (byte)length);
var opStatus = Utf8.FromUtf16(formattedName, bytes.AsSpan(1), out _, out _);
Debug.Assert(opStatus == OperationStatus.Done);
var remaining = bytes.Length;
var ulongCount = Math.DivRem(remaining, 8, out remaining);
var offset = 0;
for (var j = 0; j < ulongCount; j++)
{
result.Add(Expression.Constant(BitConverter.ToUInt64(bytes, offset)));
offset += sizeof(ulong);
}
var uintCount = Math.DivRem(remaining, 4, out remaining);
for (var j = 0; j < uintCount; j++)
{
result.Add(Expression.Constant(BitConverter.ToUInt32(bytes, offset)));
offset += sizeof(uint);
}
var ushortCount = Math.DivRem(remaining, 2, out remaining);
for (var j = 0; j < ushortCount; j++)
{
result.Add(Expression.Constant(BitConverter.ToUInt16(bytes, offset)));
offset += sizeof(ushort);
}
var byteCount = Math.DivRem(remaining, 1, out remaining);
for (var j = 0; j < byteCount; j++)
{
result.Add(Expression.Constant(bytes[offset]));
offset++;
}
Debug.Assert(remaining == 0);
Debug.Assert(offset == bytes.Length);
return(result.ToArray());
}
static unsafe Map()
{
var userInterfaceUtf8 = new byte[UserInterface.Length + 1];
Utf8.FromUtf16(UserInterface, userInterfaceUtf8, out _, out _);
UserInterfaceUtf8 = userInterfaceUtf8;
var instructorOperatorStationUtf8 = new byte[InstructorOperatorStationUtf8.Length + 1];
Utf8.FromUtf16(InstructorOperatorStation, instructorOperatorStationUtf8, out _, out _);
InstructorOperatorStationUtf8 = instructorOperatorStationUtf8;
_mapCreatedCallback = OnMapCreated;
public static string Digest(IncrementalHash hasher, string input)
{
Span <byte> buffer = stackalloc byte[1024];
var src = input.AsSpan();
while (src.Length > 0)
{
if (Utf8.FromUtf16(src, buffer, out int read, out int written) != OperationStatus.Done)
{
throw new Exception();
}
hasher.AppendData(buffer.Slice(0, written));
src = src[read..];
public static byte[]? ToUTF8NullTerminated(ReadOnlySpan <char> str)
{
if (str.IsEmpty)
{
return(null);
}
int maxByteCount = str.Length * sizeof(char);
byte[] data = new byte[maxByteCount + 1]; //Extra charactor acts as the null terminator
var res = Utf8.FromUtf16(str, data.AsSpan(0, maxByteCount), out _, out _);
Debug.Assert(res == OperationStatus.Done);
return(data);
}
public Header(ReadOnlySpan <byte> name, string value)
{
int length = name.Length + value.Length + 3;
_utf8 = new byte[length];
var utf8 = _utf8.AsSpan();
name.CopyTo(_utf8);
_utf8[name.Length] = (byte)':';
utf8 = utf8.Slice(name.Length + 1);
if (Utf8.FromUtf16(MemoryMarshal.AsBytes(value.AsSpan()), utf8, out var consumed, out int written) != OperationStatus.Done)
{
throw new Exception("value is not ASCII");
}
_utf8[length - 2] = (byte)'\r';
_utf8[length - 1] = (byte)'\n';
}
private static void ToBytes_Test_Core(ReadOnlySpan <char> utf16Input, int destinationSize, bool replaceInvalidSequences, bool isFinalChunk, OperationStatus expectedOperationStatus, int expectedNumCharsRead, ReadOnlySpan <byte> expectedUtf8Transcoding)
{
// Arrange
using (BoundedMemory <char> boundedSource = BoundedMemory.AllocateFromExistingData(utf16Input))
using (BoundedMemory <byte> boundedDestination = BoundedMemory.Allocate <byte>(destinationSize))
{
boundedSource.MakeReadonly();
// Act
OperationStatus actualOperationStatus = Utf8.FromUtf16(boundedSource.Span, boundedDestination.Span, out int actualNumCharsRead, out int actualNumBytesWritten, replaceInvalidSequences, isFinalChunk);
// Assert
Assert.Equal(expectedOperationStatus, actualOperationStatus);
Assert.Equal(expectedNumCharsRead, actualNumCharsRead);
Assert.Equal(expectedUtf8Transcoding.Length, actualNumBytesWritten);
Assert.Equal(expectedUtf8Transcoding.ToArray(), boundedDestination.Span.Slice(0, actualNumBytesWritten).ToArray());
}
}
public void Log(string sawmillName, LogEvent message)
{
var robustLevel = message.Level.ToRobust();
lock (_stream)
{
_line
.Clear()
.Append(LogLevelToString(robustLevel))
.Append(sawmillName)
.Append(": ")
.AppendLine(message.RenderMessage());
if (message.Exception != null)
{
_line.AppendLine(message.Exception.ToString());
}
// ReSharper disable once SuggestVarOrType_Elsewhere
if (!_isUtf16Out)
{
Span <byte> buf = stackalloc byte[1024];
var totalChars = _line.Length;
foreach (var chunk in _line.GetChunks())
{
var chunkSize = chunk.Length;
var totalRead = 0;
var span = chunk.Span;
for (;;)
{
var finalChunk = totalRead + chunkSize >= totalChars;
Utf8.FromUtf16(span, buf, out var read, out var wrote, isFinalBlock: finalChunk);
_stream.Write(buf.Slice(0, wrote));
totalRead += read;
if (read >= chunkSize)
{
break;
}
span = span[read..];
public static void WritePrimitiveCore(Stream stream, string value)
{
if (value == null)
{
Primitives.WritePrimitive(stream, (uint)0);
return;
}
if (value.Length == 0)
{
Primitives.WritePrimitive(stream, (uint)1);
return;
}
Span <byte> buf = stackalloc byte[StringByteBufferLength];
var totalChars = value.Length;
var totalBytes = Encoding.UTF8.GetByteCount(value);
Primitives.WritePrimitive(stream, (uint)totalBytes + 1);
Primitives.WritePrimitive(stream, (uint)totalChars);
var totalRead = 0;
ReadOnlySpan <char> span = value;
for (;;)
{
var finalChunk = totalRead + totalChars >= totalChars;
Utf8.FromUtf16(span, buf, out var read, out var wrote, isFinalBlock: finalChunk);
stream.Write(buf.Slice(0, wrote));
totalRead += read;
if (read >= totalChars)
{
break;
}
span = span[read..];
public static ref byte ToUtf8(ReadOnlySpan <char> utf16, Span <byte> utf8, out int count)
{
Utf8.FromUtf16(utf16, utf8, out _, out count);
utf8[count] = 0;
return(ref utf8.GetPinnableReference());
}
public static int ToUtf8(string s, Span <byte> buffer)
{
_ = Utf8.FromUtf16(s, buffer, out _, out var count);
return(count);
}
public static int ToUtf8(ReadOnlySpan <char> s, Span <byte> buffer)
{
_ = Utf8.FromUtf16(s, buffer, out _, out var count);
return(count);
}
public static unsafe byte *ToUtf8Unsafe(ReadOnlySpan <char> utf16, Span <byte> utf8, out int count)
{
Utf8.FromUtf16(utf16, utf8, out _, out count);
utf8[count] = 0;
return((byte *)Unsafe.AsPointer(ref utf8.GetPinnableReference()));
}
// Returns 'null' if the input buffer does not represent well-formed UTF-16 data and 'replaceInvalidSequences' is false.
private static byte[]? CreateBufferFromUtf16Common(ReadOnlySpan <char> value, bool replaceInvalidSequences)
{
// Shortcut: Since we expect most strings to be small-ish, first try a one-pass
// operation where we transcode directly on to the stack and then copy the validated
// data into the new Utf8String instance. It's still O(n), but it should have a smaller
// constant factor than a typical "count + transcode" combo.
OperationStatus status;
byte[] newBuffer;
if (value.Length <= MAX_STACK_TRANSCODE_CHAR_COUNT /* in chars */)
{
if (value.IsEmpty)
{
return(Utf8String.Empty._bytes);
}
Span <byte> scratch = stackalloc byte[MAX_STACK_TRANSCODE_CHAR_COUNT * MAX_UTF8_BYTES_PER_UTF16_CHAR]; // largest possible expansion, as explained below
status = Utf8.FromUtf16(value, scratch, out _, out int scratchBytesWritten, replaceInvalidSequences);
Debug.Assert(status == OperationStatus.Done || status == OperationStatus.InvalidData);
if (status == OperationStatus.InvalidData)
{
return(null);
}
// At this point we know transcoding succeeded, so the original input data was well-formed.
// We'll memcpy the scratch buffer into the new Utf8String instance, which is very fast.
newBuffer = new byte[scratchBytesWritten + 1]; // null-terminated
scratch.Slice(0, scratchBytesWritten).CopyTo(newBuffer);
return(newBuffer);
}
// First, determine how many UTF-8 bytes we'll need in order to represent this data.
// This also checks the input data for well-formedness.
long utf8CodeUnitCountAdjustment;
unsafe
{
fixed(char *pChars = &MemoryMarshal.GetReference(value))
{
if (Utf16Utility.GetPointerToFirstInvalidChar(pChars, value.Length, out utf8CodeUnitCountAdjustment, out int _) != (pChars + (uint)value.Length))
{
return(null);
}
}
}
// The max possible expansion transcoding UTF-16 to UTF-8 is that each input char corresponds
// to 3 UTF-8 bytes. This is most common in CJK languages. Since the input buffer could be
// up to int.MaxValue elements in length, we need to use a 64-bit value to hold the total
// required UTF-8 byte length. However, the VM places restrictions on how large a Utf8String
// instance can be, and the maximum allowed element count is just under int.MaxValue. (This
// mirrors the restrictions already in place for System.String.) The VM will throw an
// OutOfMemoryException if anybody tries to create a Utf8String instance larger than that,
// so if we detect any sort of overflow we'll end up passing int.MaxValue down to the allocation
// routine. This normalizes the OutOfMemoryException the caller sees.
long totalUtf8BytesRequired = (uint)value.Length + utf8CodeUnitCountAdjustment;
if (totalUtf8BytesRequired >= int.MaxValue)
{
totalUtf8BytesRequired = int.MaxValue - 1;
}
// We can get away with FastAllocateSkipZeroInit here because we're not going to return the
// new Utf8String instance to the caller if we don't overwrite every byte of the buffer.
newBuffer = new byte[(int)totalUtf8BytesRequired + 1]; // null-terminated
// Now transcode the UTF-16 input into the newly allocated Utf8String's buffer. We can't call the
// "skip validation" transcoder because the caller could've mutated the input buffer between the
// initial counting step and the transcoding step below.
status = Utf8.FromUtf16(value, newBuffer.AsSpan(0, newBuffer.Length - 1), out _, out int bytesWritten, replaceInvalidSequences: false);
if (status != OperationStatus.Done || bytesWritten != newBuffer.Length - 1)
{
// Did somebody mutate our input buffer? Shouldn't be any other way this could happen.
return(null);
}
return(newBuffer);
}
public override async Task <int> ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)
{
if (count < 0)
{
throw new ArgumentOutOfRangeException(nameof(count));
}
if (offset < 0 || offset >= buffer.Length)
{
throw new ArgumentOutOfRangeException(nameof(offset));
}
if (buffer.Length - offset < count)
{
throw new ArgumentOutOfRangeException(nameof(count));
}
if (count == 0)
{
return(0);
}
var readBuffer = new ArraySegment <byte>(buffer, offset, count);
if (_overflowBuffer.Count > 0)
{
var bytesToCopy = Math.Min(count, _overflowBuffer.Count);
_overflowBuffer.Slice(0, bytesToCopy).CopyTo(readBuffer);
_overflowBuffer = _overflowBuffer.Slice(bytesToCopy);
return(bytesToCopy);
}
if (_charBuffer.Count == 0)
{
await ReadInputChars(cancellationToken);
}
var operationStatus = Utf8.FromUtf16(_charBuffer, readBuffer, out var charsRead, out var bytesWritten, isFinalBlock: false);
_charBuffer = _charBuffer.Slice(charsRead);
switch (operationStatus)
{
case OperationStatus.Done:
return(bytesWritten);
case OperationStatus.DestinationTooSmall:
if (bytesWritten != 0)
{
return(bytesWritten);
}
Utf8.FromUtf16(_charBuffer, _overflowBuffer.Array, out var overFlowChars, out var overflowBytes, isFinalBlock: false);
Debug.Assert(overflowBytes > 0 && overFlowChars > 0, "We expect writes to the overflow buffer to always succeed since it is large enough to accomodate at least one char.");
_charBuffer = _charBuffer.Slice(overFlowChars);
Debug.Assert(readBuffer.Count < overflowBytes);
_overflowBuffer.Array.AsSpan(0, readBuffer.Count).CopyTo(readBuffer);
_overflowBuffer = new ArraySegment <byte>(
_overflowBuffer.Array,
readBuffer.Count,
overflowBytes - readBuffer.Count);
Debug.Assert(_overflowBuffer.Count != 0);
return(readBuffer.Count);
default:
Debug.Fail("We should never see this");
throw new InvalidOperationException();
}
}
