public static async IAsyncEnumerable <string> ReadAllLinesAsync(this StreamReader reader,
[EnumeratorCancellation] CancellationToken cancellationToken = default)
{
var stringBuilder = new StringBuilder();
using var buffer = new PooledBuffer <char>(BufferSizes.StreamReader);
int charsRead;
while ((charsRead = await reader.ReadAsync(buffer.Array, cancellationToken)) > 0)
{
for (var i = 0; i < charsRead; i++)
{
if (buffer.Array[i] == '\n')
{
// Trigger on buffered input (even if it's empty)
yield return(stringBuilder.ToString());
stringBuilder.Clear();
}
else if (buffer.Array[i] != '\r')
{
stringBuilder.Append(buffer.Array[i]);
}
}
}
// Yield what's remaining
if (stringBuilder.Length > 0)
{
yield return(stringBuilder.ToString());
}
}
/// <summary>
/// Read multiple records, where some of them might be null.
/// </summary>
/// <param name="buf">Buffer.</param>
/// <param name="schema">Schema or null when there is no value.</param>
/// <param name="keyOnly">Key only mode.</param>
/// <returns>List of records.</returns>
public IList <T?> ReadMultipleNullable(PooledBuffer buf, Schema?schema, bool keyOnly = false)
{
if (schema == null)
{
// Null schema means empty collection.
return(Array.Empty <T?>());
}
// Skip schema version.
var r = buf.GetReader();
r.Skip();
var count = r.ReadInt32();
var res = new List <T?>(count);
for (var i = 0; i < count; i++)
{
var hasValue = r.ReadBoolean();
res.Add(hasValue ? _handler.Read(ref r, schema, keyOnly) : null);
}
return(res);
}
public void SendFromServerToClient()
{
RunWith5SecTimeout(async() =>
{
var testMsg = new byte[] { 1, 2, 3, 4, 5 };
IPooledBuffer receivedMsg;
WriteLog("Starting server");
using (var server = CreateServer())
{
await server.StartAsync();
var serverConnectionTask = server.In.ReadAsync();
WriteLog("Connecting client");
var client = CreateClient();
using (var clientConnection = await client.ConnectAsync())
{
WriteLog("Client connected");
using (var serverConection = await serverConnectionTask)
{
await serverConection.Out.WriteAsync(PooledBuffer.Get(testMsg));
receivedMsg = await clientConnection.In.ReadAsync();
WriteLog("Disposing connections");
}
}
WriteLog("Disposing server");
}
receivedMsg.ToArray().ShouldBe(testMsg);
});
}
public void StructPackerBench()
{
SmallValueMsg msg = _smallValueMsg;
using PooledBuffer slice = msg.PackToBuffer();
msg.Unpack(slice.Data);
}
public void MessageSerializationTests()
{
var msgPack = new BigMessage(true);
int size = msgPack.GetSize();
size.TestIsGreater(0);
using PooledBuffer buffer = Tools.PackMsgToBuffer(ref msgPack, size, BigMessage.Pack);
byte[] array = Tools.PackMsgToArray(ref msgPack, size, BigMessage.Pack);
using var memStr = new MemoryStream();
Tools.PackMsgToStream(ref msgPack, memStr, size, BigMessage.Pack);
memStr.Position = 0;
var msgUnpack = new BigMessage();
Tools.UnpackMsg(ref msgUnpack, memStr, BigMessage.Unpack);
msgPack.TestEqual(msgUnpack);
msgUnpack = new BigMessage();
int index = 0;
Tools.UnpackMsg(ref msgUnpack, array, ref index, BigMessage.Unpack);
msgPack.TestEqual(msgUnpack);
}
public static async IAsyncEnumerable <string> ReadAllLinesAsync(
this StreamReader reader,
[EnumeratorCancellation] CancellationToken cancellationToken = default)
{
var stringBuilder = new StringBuilder();
using var buffer = PooledBuffer.ForStreamReader();
// Following sequences are treated as individual linebreaks:
// - \r
// - \n
// - \r\n
// Even though \r and \n are linebreaks on their own, \r\n together
// should not yield two lines. To ensure that, we keep track of the
// previous char and check if it's part of a sequence.
var prevSeqChar = (char?)null;
int charsRead;
while ((charsRead = await reader.ReadAsync(buffer.Array, cancellationToken).ConfigureAwait(false)) > 0)
{
for (var i = 0; i < charsRead; i++)
{
var curChar = buffer.Array[i];
// If current char and last char are part of a line break sequence,
// skip over the current char and move on.
// The buffer was already yielded in the previous iteration, so there's
// nothing left to do.
if (prevSeqChar == '\r' && curChar == '\n')
{
prevSeqChar = null;
continue;
}
// If current char is \n or \r, yield the buffer (even if it is empty)
if (curChar == '\n' || curChar == '\r')
{
yield return(stringBuilder.ToString());
stringBuilder.Clear();
}
// For any other char, just append it to the buffer
else
{
stringBuilder.Append(curChar);
}
prevSeqChar = curChar;
}
}
// Yield what's remaining in the buffer
if (stringBuilder.Length > 0)
{
yield return(stringBuilder.ToString());
}
}
/// <summary>
/// 메모리 반납하기
/// </summary>
/// <param name="buffer">반납할 buffer</param>
public void Return(byte[] buffer)
{
PooledBuffer pooled = FindPool(buffer.Length);
if (pooled != null)
{
pooled.Return(buffer);
}
}
public static async IAsyncEnumerable <string> ReadAllLinesAsync(this StreamReader reader,
[EnumeratorCancellation] CancellationToken cancellationToken = default)
{
var stringBuilder = new StringBuilder();
using var buffer = PooledBuffer.ForStreamReader();
// contains the first char of the line break string in a series of line breaks
// this enables us to convert all of these: "A\r\rB", "A\n\nB" and "A\r\n\r\nB" into 3 lines: "A", "" and "B"
// if we didn't do this the last example would be converted into 5 lines
char?lineSeparator = null;
bool firstChar = true;
int charsRead;
while ((charsRead = await reader.ReadAsync(buffer.Array, cancellationToken)) > 0)
{
for (var i = 0; i < charsRead; i++)
{
char current = buffer.Array[i];
// if the first char we read is a '\r' we don't return an empty line
if (current == '\r' && firstChar)
{
firstChar = false;
continue;
}
if (current == '\n' || current == '\r')
{
lineSeparator ??= current;
if (current == lineSeparator)
{
// Trigger on buffered input (even if it's empty)
yield return(stringBuilder.ToString());
stringBuilder.Clear();
}
}
else
{
stringBuilder.Append(current);
lineSeparator = null;
}
firstChar = false;
}
}
// Yield what's remaining
if (stringBuilder.Length > 0)
{
yield return(stringBuilder.ToString());
}
}
/// <summary>
/// 생성자
/// </summary>
/// <param name="sizeArray">사이즈에 해당하는 풀을 생성함</param>
public PooledBufferManager(int[] sizeArray)
{
Array.Sort(sizeArray);
_pools = new PooledBuffer[sizeArray.Length];
for (int i = 0; i < sizeArray.Length; ++i)
{
_pools[i] = new PooledBuffer(sizeArray[i]);
}
}
public static IPooledBuffer Serialize(this IMessage message)
{
var buffer = PooledBuffer.Rent();
var stream = new CodedOutputStream(buffer.Array);
message.WriteTo(stream);
stream.Flush();
buffer.Count = (int)stream.Position;
return(buffer);
}
/// <summary>
/// 생성자
/// </summary>
/// <param name="sizeArray">사이즈에 해당하는 풀을 생성함</param>
public PooledBufferManager(int[] sizeArray)
{
Array.Sort(sizeArray);
_pools = new PooledBuffer[sizeArray.Length];
for (int i = 0; i < sizeArray.Length; ++i)
{
_pools[i] = new PooledBuffer(sizeArray[i]);
}
}
public static async Task WriteByteAsync(
this Stream stream,
byte value,
CancellationToken cancellationToken = default)
{
using var buffer = new PooledBuffer <byte>(1);
buffer.Array[0] = value;
await stream.WriteAsync(buffer.Array, 0, 1, cancellationToken).ConfigureAwait(false);
}
/// <summary>
/// 메모리 획득하기
/// </summary>
/// <param name="size">획득할 메모리 사이즈, 만약 풀에 존재하지 않는 값이면 일반 Alloctor로 할당해서 넘김</param>
/// <returns>할당한 buffer </returns>
public byte[] Take(int size)
{
PooledBuffer pooled = FindPool(size);
if (pooled == null)
{
return(PooledBuffer.AllocNewBuffer(size));
}
return(pooled.Take(size));
}
private async Task ReceiveInternalAsync()
{
int curMessageLength = 0;
while (!_receiveQueue.Out.IsCompleted())
{
var result = await _webSocket.ReceiveAsync(new ArraySegment <byte>(_receiveBuffer, curMessageLength, _receiveBuffer.Length - curMessageLength), CancellationToken.None).ConfigureAwait(false);
_log.Trace("Received websocket message: {0}", result.MessageType);
if (result.MessageType == WebSocketMessageType.Close)
{
_log.Trace("Received close message. Current state: {0}", _webSocket.State);
_receiveQueue.Out.TryComplete();
break;
}
curMessageLength += result.Count;
if (!result.EndOfMessage)
{
continue;
}
switch (result.MessageType)
{
case WebSocketMessageType.Binary:
_log.Trace("Received binary message of length {0}", result.Count);
var msg = PooledBuffer.Get(new ArraySegment <byte>(_receiveBuffer, 0, curMessageLength));
try
{
await _receiveQueue.Out.WriteAsync(msg).ConfigureAwait(false);
}
catch
{
msg.Dispose();
throw;
}
break;
case WebSocketMessageType.Text:
_log.Trace("Received text of length {0}", result.Count);
var text = Encoding.UTF8.GetString(_receiveBuffer, 0, result.Count);
if (string.Equals(text, "<END>"))
{
_log.Trace("Received <END> message");
_receiveQueue.Out.TryComplete();
}
else if (string.Equals(text, "<PING>"))
{
_log.Trace("Received <PING> message");
}
break;
}
curMessageLength = 0;
}
_receiveQueue.Out.TryComplete();
}
public static async IAsyncEnumerable <byte> ReadAllBytesAsync(
this Stream stream,
[EnumeratorCancellation] CancellationToken cancellationToken = default)
{
using var buffer = new PooledBuffer <byte>(1);
while (await stream.ReadAsync(buffer.Array, 0, 1, cancellationToken).ConfigureAwait(false) > 0)
{
yield return(buffer.Array[0]);
}
}
public override async Task CopyFromAsync(Stream source, CancellationToken cancellationToken = default)
{
using var reader = new StreamReader(source, _encoding, false, BufferSizes.StreamReader, true);
using var buffer = new PooledBuffer <char>(BufferSizes.StreamReader);
int charsRead;
while ((charsRead = await reader.ReadAsync(buffer.Array, cancellationToken)) > 0)
{
_stringBuilder.Append(buffer.Array, 0, charsRead);
}
}
/// <inheritdoc />
public async Task ExtractPackageAsync(string sourceFilePath, string destDirPath,
IProgress <double>?progress = null, CancellationToken cancellationToken = default)
{
// Read the zip
using var archive = ZipFile.OpenRead(sourceFilePath);
// Get entries in the content directory
var entries = archive.Entries
.Where(e => e.FullName.StartsWith(_rootDirPath, StringComparison.OrdinalIgnoreCase))
.ToArray();
// For progress reporting
var totalBytes = entries.Sum(e => e.Length);
var totalBytesCopied = 0L;
// Loop through entries
foreach (var entry in entries)
{
// Get relative entry path
var relativeEntryPath = entry.FullName.Substring(_rootDirPath.Length).TrimStart('/', '\\');
// Get destination paths
var entryDestFilePath = Path.Combine(destDirPath, relativeEntryPath);
var entryDestDirPath = Path.GetDirectoryName(entryDestFilePath);
// Create directory
if (!string.IsNullOrWhiteSpace(entryDestDirPath))
{
Directory.CreateDirectory(entryDestDirPath);
}
// If the entry is a directory - continue
if (entry.FullName.Last() == Path.DirectorySeparatorChar || entry.FullName.Last() == Path.AltDirectorySeparatorChar)
{
continue;
}
// Extract entry
using var input = entry.Open();
using var output = File.Create(entryDestFilePath);
using var buffer = PooledBuffer.ForStream();
int bytesCopied;
do
{
bytesCopied = await input.CopyBufferedToAsync(output, buffer.Array, cancellationToken);
totalBytesCopied += bytesCopied;
progress?.Report(1.0 * totalBytesCopied / totalBytes);
} while (bytesCopied > 0);
}
}
public static async Task CopyToAsync(this Stream source, Stream destination,
IProgress<double>? progress = null, CancellationToken cancellationToken = default)
{
using var buffer = PooledBuffer.ForStream();
var totalBytesCopied = 0L;
int bytesCopied;
do
{
bytesCopied = await source.CopyBufferedToAsync(destination, buffer.Array, cancellationToken);
totalBytesCopied += bytesCopied;
progress?.Report(1.0 * totalBytesCopied / source.Length);
} while (bytesCopied > 0);
}
/// <summary>
/// Reads the value part.
/// </summary>
/// <param name="buf">Buffer.</param>
/// <param name="schema">Schema or null when there is no value.</param>
/// <param name="key">Key part.</param>
/// <returns>Resulting record with key and value parts.</returns>
public T?ReadValue(PooledBuffer buf, Schema?schema, T key)
{
if (schema == null)
{
// Null schema means null record.
return(null);
}
// Skip schema version.
var r = buf.GetReader();
r.Skip();
return(_handler.ReadValuePart(ref r, schema, key));
}
private void OnDataReceived(byte[] data)
{
_log.Trace("Received message of length={0}", data.Length);
var msg = PooledBuffer.Get(data);
try
{
_buffer.Out.WriteAsync(msg, _cancellationToken).GetResult();
}
catch (Exception ex)
{
_log.Trace(ex, "Exception in OnDataReceived callback");
msg.Dispose();
_buffer.Out.TryTerminate(ex);
}
}
private async Task <IPooledBuffer> ReadDatagram(int length)
{
try
{
var datagram = await PooledBuffer
.Get(_stream, length, _cancellationToken)
.ConfigureAwait(false);
return(datagram);
}
catch (Exception ex)
{
_log.Warn(ex, $"Caught exception during attempt to read datagram of length {length}");
throw;
}
}
private void OnDataReceived(object sender, DataReceivedEventArgs e)
{
try
{
_log.Trace("Received message of length={0}", e.Data.Length);
if (!_receiveQueue.Out.TryWriteSafeAsync(PooledBuffer.Get(e.Data)).GetAwaiter().GetResult())
{
_log.Trace("Failed to add received message to receive queueu");
}
}
catch (Exception ex)
{
_log.Trace(ex, "Exception in OnDataReceived callback");
_receiveQueue.Out.TryTerminate(ex);
}
}
public static async Task CopyToAsync(this Stream source, Stream destination, bool autoFlush,
CancellationToken cancellationToken = default)
{
using var buffer = PooledBuffer.ForStream();
int bytesRead;
while ((bytesRead = await source.ReadAsync(buffer.Array, cancellationToken).ConfigureAwait(false)) != 0)
{
await destination.WriteAsync(buffer.Array, 0, bytesRead, cancellationToken).ConfigureAwait(false);
if (autoFlush)
{
await destination.FlushAsync(cancellationToken).ConfigureAwait(false);
}
}
}
public static async Task CopyToAsync(this Stream source, Stream destination, bool autoFlush,
CancellationToken cancellationToken = default)
{
using var buffer = new PooledBuffer <byte>(BufferSizes.Stream);
int bytesRead;
while ((bytesRead = await source.ReadAsync(buffer.Array, cancellationToken)) != 0)
{
await destination.WriteAsync(buffer.Array, 0, bytesRead, cancellationToken);
if (autoFlush)
{
await destination.FlushAsync(cancellationToken);
}
}
}
public static async Task <byte[]> ReadByteChunkAsync(
this Stream stream,
int expectedLength,
CancellationToken cancellationToken = default)
{
using var buffer = new PooledBuffer <byte>(expectedLength);
var bytesRead = await stream.ReadAsync(buffer.Array, 0, expectedLength, cancellationToken)
.ConfigureAwait(false);
// The buffer is rented so we can't return it, plus it may be larger than needed.
// So we copy everything to a new buffer.
var result = new byte[bytesRead];
Array.Copy(buffer.Array, result, bytesRead);
return(result);
}
private ManualResetEventSlim Send(PooledBuffer packet)
{
if (packet.Length > MaximumPacketSize)
{
throw new Exception("This packet is too large to send, if this is a legitimate size, increase the MaxPacketSize.");
}
switch (m_compressionMode)
{
case CtpCompressionMode.None:
return(m_send(packet));
break;
case CtpCompressionMode.Deflate:
using (var ms = new MemoryStream())
{
using (var comp = new DeflateStream(ms, CompressionMode.Compress, true))
{
packet.CopyTo(comp);
}
return(m_send(PacketMethods.CreatePacket(PacketContents.CompressedDeflate, packet.Length, ms.ToArray())));
}
break;
case CtpCompressionMode.Zlib:
if (m_stream == null)
{
m_stream = new MemoryStream();
m_deflate = new Ionic.Zlib.DeflateStream(m_stream, Ionic.Zlib.CompressionMode.Compress);
m_deflate.FlushMode = FlushType.Sync;
}
m_stream.Write(BigEndian.GetBytes(packet.Length), 0, 4);
packet.CopyTo(m_deflate);
m_deflate.Flush();
byte[] rv = m_stream.ToArray();
m_stream.SetLength(0);
return(m_send(PacketMethods.CreatePacket(PacketContents.CompressedZlib, packet.Length, rv)));
break;
default:
throw new ArgumentOutOfRangeException();
}
}
private async Task HandleReceiveMessageAsync(WebSocketReceiveResult result, int curMessageLength)
{
switch (result.MessageType)
{
case WebSocketMessageType.Binary:
_log.Trace("Received binary message of length {0}", result.Count);
var msg = PooledBuffer.Get(new ArraySegment <byte>(_receiveBuffer, 0, curMessageLength));
try
{
await _buffer.Out.WriteAsync(msg, _cancellationToken).ConfigureAwait(false);
_log.Trace("Received binary message of length {0} added to buffer", result.Count);
}
catch
{
msg.Dispose();
throw;
}
break;
case WebSocketMessageType.Text:
_log.Trace("Received text of length {0}", result.Count);
var text = Encoding.UTF8.GetString(_receiveBuffer, 0, result.Count);
if (string.Equals(text, "<END>"))
{
_log.Trace("Received <END> message");
_buffer.Out.TryComplete();
}
else if (string.Equals(text, "<PING>"))
{
_log.Trace("Received <PING> message");
}
break;
case WebSocketMessageType.Close:
_buffer.Out.TryTerminate();
break;
default:
throw new ArgumentOutOfRangeException();
}
}
private async Task ProcessAsync()
{
try
{
while (true)
{
_log.Trace("Awaiting next message {0}", _count);
var length = await ReadLengthAsync().ConfigureAwait(false);
if (length == EndMessage)
{
_log.Trace("Completing receiving datagrams because <END> message received");
break;
}
_log.Trace("Reading message {0} of length {1}", _count, length);
var datagram = await PooledBuffer
.Get(_stream, length, _cancellationToken)
.ConfigureAwait(false);
try
{
await _buffer.Out.WriteAsync(datagram, _cancellationToken).ConfigureAwait(false);
}
catch
{
datagram.Dispose();
throw;
}
_log.Trace("Received message {0} of length {1}", _count, length);
_count++;
}
_buffer.Out.TryComplete();
}
catch (Exception ex)
{
_log.Trace("Process failed: {0}", ex.FormatTypeAndMessage());
_buffer.Out.TryTerminate(ex);
_buffer.In.DisposeBufferedItems();
throw;
}
}
private static IEnumerable <TransportMessage> GenerateAllTransportMessages()
{
yield return(new TransportMessage(TransportHeaderPool.Instance.CreateConnectionOpenHeader(UniqueId.Generate())));
yield return(new TransportMessage(TransportHeaderPool.Instance.CreateChannelOpenHeader(UniqueId.Generate())));
yield return(new TransportMessage(
TransportHeaderPool.Instance.CreateFrameHeader(UniqueId.Generate(), true, 3),
new Maybe <IPooledBuffer>(PooledBuffer.Get(new byte[] { 1, 2, 3 }))));
yield return(new TransportMessage(
TransportHeaderPool.Instance.CreateChannelCloseHeader(
UniqueId.Generate(),
new CompletionHeader(CompletionStatusHeader.Canceled, Nothing.Instance))));
yield return(new TransportMessage(
TransportHeaderPool.Instance.CreateConnectionCloseHeader(
new CompletionHeader(
CompletionStatusHeader.Failed,
new ErrorHeader("Error message", "Error details")))));
}
private async Task ReceiveLoopAsync(IWriteOnlyChannel <IPooledBuffer> received, CancellationToken cancellationToken)
{
try
{
while (true)
{
_log.Trace("Awaiting next message {0}", _receiveCount);
var length = await ReadLengthAsync(cancellationToken).ConfigureAwait(false);
if (length == EndMessage)
{
_log.Trace("Completing receiving datagrams because <END> message received");
break;
}
_log.Trace("Reading message {0} of length {1}", _receiveCount, length);
var datagram = await PooledBuffer
.Get(_stream, length, cancellationToken)
.ConfigureAwait(false);
try
{
await received.WriteAsync(datagram).ConfigureAwait(false);
}
catch
{
datagram.Dispose();
throw;
}
_log.Trace("Received message {0} of length {1}", _receiveCount, length);
_receiveCount++;
}
}
catch
{
Out.TryTerminate();
throw;
}
}
private async Task CopyToStreamAsync(HttpContent content, Stream destination, IProgress <double> progress = null, CancellationToken cancellationToken = default)
{
long?length = content.Headers.ContentLength;
using Stream source = await content.ReadAsStreamAsync();
using (PooledBuffer <byte> buffer = new PooledBuffer <byte>(81920)) {
long totalBytesCopied = 0L;
int bytesCopied;
do
{
bytesCopied = await CopyBufferedToAsync(source, destination, buffer.Array, cancellationToken);
totalBytesCopied += bytesCopied;
if (length != null)
{
progress?.Report(1.0 * totalBytesCopied / length.Value);
}
} while (bytesCopied > 0);
}
}
