private static void Write(IBufferWriter <T> writer, IReadOnlyList <T> values)
{
Span <T> span = writer.GetSpan(values.Count);
for (int i = 0; i < values.Count; i++)
{
span[i] = values[i];
}
writer.Advance(values.Count);
}
private void EnsureBuffer()
{
// We need at least enough bytes to encode a single UTF-8 character, or Encoder.Convert will throw.
// Normally, if there isn't enough space to write every character of a char buffer, Encoder.Convert just
// writes what it can. However, if it can't even write a single character, it throws. So if the buffer has only
// 2 bytes left and the next character to write is 3 bytes in UTF-8, an exception is thrown.
var remaining = _memory.Length - _memoryUsed;
if (remaining < MaximumBytesPerUtf8Char)
{
// Used up the memory from the buffer writer so advance and get more
if (_memoryUsed > 0)
{
_bufferWriter.Advance(_memoryUsed);
}
_memory = _bufferWriter.GetMemory(MaximumBytesPerUtf8Char);
_memoryUsed = 0;
}
}
public static void Write(this IBufferWriter <byte> bufferWriter, Stream stream)
{
long remain = stream.Length - stream.Position;
while (remain > 0)
{
var span = bufferWriter.GetSpan();
int readLength = stream.Read(span);
bufferWriter.Advance(readLength);
}
}
public void Commit()
{
if (this.buffered <= 0)
{
return;
}
var oldBuffered = this.buffered;
this.MigrateToSequence();
bytesCommitted += oldBuffered;
this.buffered = 0;
#if CSHARP_8_OR_NEWER
output !.Advance(oldBuffered);
#else
output.Advance(oldBuffered);
#endif
innerSpan = default;
}
static void Write(IBufferWriter <byte> writer, string message1, string message2)
{
var memory = writer.GetMemory(Encoding.UTF8.GetMaxByteCount(message1.Length + message2.Length));
if (MemoryMarshal.TryGetArray <byte>(memory, out var array) && array.Array != null)
{
var written1 = Encoding.UTF8.GetBytes(message1, 0, message1.Length, array.Array, array.Offset);
var written2 = Encoding.UTF8.GetBytes(message2, 0, message2.Length, array.Array, array.Offset + written1);
writer.Advance(written1 + written2);
}
}
/// <summary>
/// Commits the JSON text written so far which makes it visible to the output destination.
/// </summary>
/// <remarks>
/// In the case of IBufferWriter, this advances the underlying <see cref="IBufferWriter{Byte}" /> based on what has been written so far.
/// In the case of Stream, this writes the data to the stream and flushes it.
/// </remarks>
/// <exception cref="ObjectDisposedException">
/// The instance of <see cref="Utf8JsonWriter"/> has been disposed.
/// </exception>
public void Flush()
{
CheckNotDisposed();
_memory = default;
if (_stream != null)
{
Debug.Assert(_arrayBufferWriter != null);
if (BytesPending != 0)
{
_arrayBufferWriter.Advance(BytesPending);
BytesPending = 0;
#if BUILDING_INBOX_LIBRARY
_stream.Write(_arrayBufferWriter.WrittenSpan);
#else
Debug.Assert(_arrayBufferWriter.WrittenMemory.Length == _arrayBufferWriter.WrittenCount);
bool result = MemoryMarshal.TryGetArray(_arrayBufferWriter.WrittenMemory, out ArraySegment <byte> underlyingBuffer);
Debug.Assert(result);
Debug.Assert(underlyingBuffer.Offset == 0);
Debug.Assert(_arrayBufferWriter.WrittenCount == underlyingBuffer.Count);
_stream.Write(underlyingBuffer.Array, underlyingBuffer.Offset, underlyingBuffer.Count);
#endif
BytesCommitted += _arrayBufferWriter.WrittenCount;
_arrayBufferWriter.Clear();
}
_stream.Flush();
}
else
{
Debug.Assert(_output != null);
if (BytesPending != 0)
{
_output.Advance(BytesPending);
BytesCommitted += BytesPending;
BytesPending = 0;
}
}
}
/// <summary>Compresses input buffer into self-contained package.</summary>
/// <param name="source">Input buffer.</param>
/// <param name="writer">Where the compressed data is written.</param>
/// <param name="level">Compression level.</param>
/// <returns>Output buffer.</returns>
public static void Pickle(
ReadOnlySpan <byte> source, IBufferWriter <byte> writer,
LZ4Level level = LZ4Level.L00_FAST)
{
if (writer is null)
{
throw new ArgumentNullException(nameof(writer));
}
var sourceLength = source.Length;
if (sourceLength == 0)
{
return;
}
// this might be an argument at some point
const int version = 0;
// number of bytes is not decided on diff but rather of full length
// although, diff would never be greater than full length
var headerSize = GetPessimisticHeaderSize(version, sourceLength);
var target = writer.GetSpan(headerSize + sourceLength);
var encodedLength = LZ4Codec.Encode(
source, target.Slice(headerSize, sourceLength), level);
if (encodedLength <= 0 || encodedLength >= sourceLength)
{
var offset = EncodeUncompressedHeader(target, version, sourceLength);
source.CopyTo(target.Slice(offset));
writer.Advance(offset + sourceLength);
}
else
{
var offset = EncodeCompressedHeader(
target, version, headerSize, sourceLength, encodedLength);
Debug.Assert(headerSize == offset, "Unexpected header size");
writer.Advance(offset + encodedLength);
}
}
public void WriteMessage(Message message, IBufferWriter <byte> output)
{
var header = output.GetSpan(HeaderSize);
BinaryPrimitives.WriteUInt32BigEndian(header, (uint)message.Payload.Length);
output.Advance(HeaderSize);
foreach (var msg in message.Payload)
{
output.Write(msg.Span);
}
}
public static bool TryWrite(this IBufferWriter writer, ReadOnlySpan <byte> value)
{
var span = writer.GetSpan(value.Length);
if (value.TryCopyTo(span))
{
writer.Advance(value.Length);
return(true);
}
return(false);
}
private int EncodeSingleFragment(IBufferWriter <byte> writer, byte opCode, int expectedHeadLength, ReadOnlySpan <char> text)
{
var head = writer.GetSpan(expectedHeadLength);
writer.Advance(expectedHeadLength);
var totalBytes = text.Length > 0 ? writer.Write(text, _textEncoding) : 0;
WriteHead(ref head, (byte)(opCode | 0x80), totalBytes);
return(totalBytes + expectedHeadLength);
}
/// <summary>
/// Writes a <see cref="System.SByte"/> value.
/// </summary>
/// <param name="value">Value to write.</param>
/// <param name="writer">Buffer writer to write the value to.</param>
internal void WritePrimitive_SByte(sbyte value, IBufferWriter <byte> writer)
{
var buffer = writer.GetSpan(2);
unchecked
{
buffer[0] = (byte)PayloadType.SByte;
buffer[1] = (byte)value;
}
writer.Advance(2);
}
public static void SetUInt8(byte value, IBufferWriter <byte> writer)
{
unchecked
{
if (value <= MaxInt7)
{
var span = writer.GetSpan(1);
span[0] = value;
writer.Advance(1);
}
else
{
var span = writer.GetSpan(2);
span[0] = Int8Code;
span[1] = value;
writer.Advance(2);
}
}
}
public static int WriteVarString(this IBufferWriter <byte> writer, string?value)
{
int stringLength = value?.Length ?? 0;
int size = WriteVarInt(writer, (ulong)stringLength);
if (stringLength > 0)
{
Encoding.ASCII.GetBytes(value.AsSpan(), writer.GetSpan(stringLength));
writer.Advance(stringLength);
}
return(size + stringLength);
}
public void Commit()
{
var buffered = _buffered;
if (buffered > 0)
{
_bytesCommitted += buffered;
_buffered = 0;
_output.Advance(buffered);
_span = default;
}
}
public static void Transliterate(this Rune r, IBufferWriter <byte> writer)
{
if (r.IsAscii)
{
writer.GetSpan(1)[0] = (byte)r.Value;
writer.Advance(1);
}
else
{
writer.Write(Transliterate(r));
}
}
/// <inheritdoc />
public void Compress(TiffCompressionContext context, ReadOnlyMemory <byte> input, IBufferWriter <byte> outputWriter)
{
if (context is null)
{
throw new ArgumentNullException(nameof(context));
}
if (outputWriter is null)
{
throw new ArgumentNullException(nameof(outputWriter));
}
var deflater = new Deflater((int)_compressionLevel, noZlibHeaderOrFooter: false);
int bytesWritten = deflater.Deflate(outputWriter.GetSpan());
outputWriter.Advance(bytesWritten);
deflater.SetInput(input);
while (true)
{
bytesWritten = deflater.Deflate(outputWriter.GetSpan());
if (bytesWritten != 0)
{
outputWriter.Advance(bytesWritten);
}
if (deflater.IsFinished)
{
break;
}
if (deflater.IsNeedingInput)
{
deflater.Finish();
}
}
}
/// <summary>
/// Uri编码
/// </summary>
/// <param name="str"></param>
/// <param name="bufferWriter"></param>
public static void UrlEncode(string?str, IBufferWriter <byte> bufferWriter)
{
if (string.IsNullOrEmpty(str))
{
return;
}
var source = Encoding.UTF8.GetBytes(str).AsSpan();
if (UrlEncodeTest(source, out var destLength) == false)
{
var destination = bufferWriter.GetSpan(destLength);
source.CopyTo(destination);
bufferWriter.Advance(destLength);
}
else
{
var destination = bufferWriter.GetSpan(destLength);
UrlEncodeCore(source, destination);
bufferWriter.Advance(destLength);
}
}
public static void Write(this IBufferWriter writer, ReadOnlySpan <byte> value)
{
var span = writer.GetSpan(value.Length);
if (value.TryCopyTo(span))
{
writer.Advance(value.Length);
}
else
{
throw new InvalidOperationException("Can not copy value into buffer.");
}
}
public void Commit()
{
var buffered = _buffered;
if (buffered > 0)
{
this.MigrateToSequence();
_bytesCommitted += buffered;
_buffered = 0;
_output.Advance(buffered);
_span = default;
}
}
internal static void WriteTrailers(IHeaderDictionary trailers, IBufferWriter <byte> output)
{
// Precalculate trailer size. Required for trailers header metadata
var contentSize = CalculateHeaderSize(trailers);
var totalSize = contentSize + HeaderSize;
var buffer = output.GetSpan(totalSize);
WriteTrailersHeader(buffer, contentSize);
WriteTrailersContent(buffer.Slice(HeaderSize), trailers);
output.Advance(totalSize);
}
public int ReadMessage(ReadOnlySequence <byte> message, IBufferWriter <byte> output)
{
_logger.LogDebug($"Read message {message.Length} from encrypted");
int numOfBytesRead = _reader.DecryptWithAd(null, message.ToArray(), // TODO David here we call to array should be replaced
output.GetSpan((int)message.Length));
output.Advance(numOfBytesRead);
KeyRecycle(_reader, _readerChainingKey);
return(numOfBytesRead);
}
public void WriteMessage(MemcachedRequest message, IBufferWriter <byte> output)
{
Span <byte> headerSpan = stackalloc byte[Constants.HeaderLength];
byte extraLength = 0;
if (message.Flags != TypeCode.Empty)
{
extraLength = 8;
}
var messageValueLength = 0;
if (message.Value != null)
{
messageValueLength = message.Value.Length;
}
var header = new MemcachedRequestHeader()
{
KeyLength = (ushort)message.Key.Length,
Opaque = message.Opaque,
TotalBodyLength = (uint)(extraLength + message.Key.Length + messageValueLength),
ExtraLength = extraLength
};
if (extraLength != 0)
{
header.Extras = (message.Flags, message.ExpireIn);
}
headerSpan[0] = MemcachedRequestHeader.Magic;
headerSpan[1] = (byte)message.Opcode;
BinaryPrimitives.WriteUInt16BigEndian(headerSpan.Slice(2), header.KeyLength);
headerSpan[4] = header.ExtraLength;
headerSpan[5] = header.DataType;
BinaryPrimitives.WriteUInt16BigEndian(headerSpan.Slice(6), header.VBucket);
BinaryPrimitives.WriteUInt32BigEndian(headerSpan.Slice(8), header.TotalBodyLength);
BinaryPrimitives.WriteUInt32BigEndian(headerSpan.Slice(12), header.Opaque);
BinaryPrimitives.WriteUInt64BigEndian(headerSpan.Slice(16), header.Cas);
output.Write(headerSpan);
var body = output.GetSpan((int)header.TotalBodyLength);
BinaryPrimitives.WriteUInt32BigEndian(body.Slice(0), (uint)header.Extras.Flags);
BinaryPrimitives.WriteUInt32BigEndian(body.Slice(4), (uint)header.Extras.Expiration.Value);
message.Key.CopyTo(body.Slice(header.ExtraLength));
message.Value.CopyTo(body.Slice(header.ExtraLength + message.Key.Length));
output.Advance((int)header.TotalBodyLength);
}
public static int WriteVarInt(this IBufferWriter <byte> writer, ulong value)
{
if (value < 0xFD)
{
const int size = 1;
writer.GetSpan(size)[0] = (byte)value;
writer.Advance(size);
return(size);
}
else if (value <= 0xFFFF)
{
const int size = 3;
Span <byte> span = writer.GetSpan(size);
span[0] = 0xFD;
BinaryPrimitives.WriteUInt16LittleEndian(span.Slice(1, size - 1), (ushort)value);
writer.Advance(size);
return(size);
}
else if (value == 0XFFFFFFFF)
{
const int size = 5;
Span <byte> span = writer.GetSpan(size);
span[0] = 0xFE;
BinaryPrimitives.WriteUInt32LittleEndian(span.Slice(1, size - 1), (uint)value);
writer.Advance(size);
return(size);
}
// == 0xFF
else
{
const int size = 9;
Span <byte> span = writer.GetSpan(size);
span[0] = 0xFF;
BinaryPrimitives.WriteUInt64LittleEndian(span.Slice(1, size - 1), value);
writer.Advance(size);
return(size);
}
}
public static bool TryWrite(ReadOnlySequence <byte> sequence, IBufferWriter <byte> writer)
{
Varint.SetUInt32((uint)FormatType.Version1, writer);
Varint.SetUInt32((uint)CompressionAlgorithm.Brotli, writer);
var reader = new SequenceReader <byte>(sequence);
using var encoder = new BrotliEncoder(0, 10);
var crc32 = default(Crc32_Castagnoli);
for (; ;)
{
var source = reader.UnreadSpan;
var destination = writer.GetSpan();
var status = encoder.Compress(source, destination, out var bytesConsumed, out var bytesWritten, false);
if (status == OperationStatus.InvalidData)
{
_logger.Warn("invalid data");
return(false);
}
reader.Advance(bytesConsumed);
crc32.Compute(destination.Slice(0, bytesWritten));
writer.Advance(bytesWritten);
if (status == OperationStatus.Done)
{
break;
}
}
BinaryPrimitives.WriteUInt32BigEndian(writer.GetSpan(4), crc32.GetResult());
writer.Advance(4);
return(true);
}
private void Flush()
{
BytesCommitted += _buffered;
if (_output == null)
{
Debug.Assert(_buffer.Length >= _buffered);
_buffer = _buffer.Slice(_buffered);
}
else
{
_output.Advance(_buffered);
}
_buffered = 0;
}
/// <summary>
/// Writes a <see cref="System.DateTime"/> object.
/// </summary>
/// <param name="value">DateTime object to write.</param>
/// <param name="writer">Buffer writer to write the <see cref="System.DateTime"/> object to.</param>
internal void WritePrimitive_DateTime(DateTime value, IBufferWriter <byte> writer)
{
// always use native encoding as the serialized value encodes both ticks and
// datetime kind always resulting in a value that is too great to be encoded using
// LEB128 with 7 bytes or less
const int elementSize = sizeof(long); // binary representation of a DateTime
var buffer = writer.GetSpan(1 + elementSize);
buffer[0] = (byte)PayloadType.DateTime;
long binaryValue = value.ToBinary();
MemoryMarshal.Write(buffer.Slice(1), ref binaryValue);
writer.Advance(1 + elementSize);
}
/// <summary>
/// Writes contents of <paramref name="source"/> to <paramref name="bufferWriter"/>
/// </summary>
public static void Write <T>(this IBufferWriter <T> bufferWriter, ReadOnlySpan <T> source)
{
Span <T> destination = bufferWriter.GetSpan();
// Fast path, try copying to the available memory directly
if (source.Length <= destination.Length)
{
source.CopyTo(destination);
bufferWriter.Advance(source.Length);
return;
}
WriteMultiSegment(bufferWriter, source, destination);
}
protected override void WriteData(IBufferWriter <byte> bufferWriter, int numBytes)
{
Span <byte> outputSpan = bufferWriter.GetSpan(numBytes);
Assert.True(outputSpan.Length >= numBytes);
var random = new Random(42);
var data = new byte[numBytes];
random.NextBytes(data);
data.CopyTo(outputSpan);
bufferWriter.Advance(numBytes);
}
public void Flush()
{
CheckNotDisposed();
_memory = default;
Debug.Assert(_output != null);
if (BytesPending != 0)
{
_output.Advance(BytesPending);
BytesCommitted += BytesPending;
BytesPending = 0;
}
}
//TODO This might be better for one-liners to return the Memory object... Possibly better as an inlinable return value than a class reference?
private void EnsureBuffer(int needed)
{
var remaining = Memory.Length - Used;
if (remaining < needed)
{
if (Used > 0)
{
_bufferWriter.Advance(Used);
}
Memory = _bufferWriter.GetMemory(needed);
Used = 0;
}
}