public async Task WriteLargeDataBinary(int length)
{
byte[] data = new byte[length];
new Random(length).NextBytes(data);
using (var memoryPool = new MemoryPool())
{
var pipe = new Pipe(memoryPool);
var output = pipe.Alloc();
output.Write(data);
var foo = output.Memory.IsEmpty; // trying to see if .Memory breaks
await output.FlushAsync();
pipe.CompleteWriter();
int offset = 0;
while (true)
{
var result = await pipe.ReadAsync();
var input = result.Buffer;
if (input.Length == 0)
{
break;
}
Assert.True(input.Equals(new Span <byte>(data, offset, input.Length)));
offset += input.Length;
pipe.Advance(input.End);
}
Assert.Equal(data.Length, offset);
}
}
public async Task WriteLargeDataTextUtf8(int length)
{
string data = new string('#', length);
FillRandomStringData(data, length);
using (var memoryPool = new MemoryPool())
{
var pipe = new Pipe(memoryPool);
var output = pipe.Alloc();
output.Append(data, EncodingData.InvariantUtf8);
var foo = output.Memory.IsEmpty; // trying to see if .Memory breaks
await output.FlushAsync();
pipe.CompleteWriter();
int offset = 0;
while (true)
{
var result = await pipe.ReadAsync();
var input = result.Buffer;
if (input.Length == 0)
{
break;
}
string s = ReadableBufferExtensions.GetUtf8String(input);
Assert.Equal(data.Substring(offset, input.Length), s);
offset += input.Length;
pipe.Advance(input.End);
}
Assert.Equal(data.Length, offset);
}
}
public async Task CanAppendSelfWhileNotEmpty()
{
byte[] chunk = new byte[512];
new Random().NextBytes(chunk);
using (var memoryPool = new MemoryPool())
{
var pipe = new Pipe(memoryPool);
var output = pipe.Alloc();
for (int i = 0; i < 20; i++)
{
output.Write(chunk);
}
var readable = output.AsReadableBuffer();
Assert.Equal(512 * 20, readable.Length);
output.Append(readable);
Assert.Equal(512 * 20, readable.Length);
readable = output.AsReadableBuffer();
Assert.Equal(2 * 512 * 20, readable.Length);
await output.FlushAsync();
}
}
private void ProcessReceives()
{
_buffer = _input.Alloc(2048);
var receiveBufferSeg = GetSegmentFromMemory(_buffer.Memory);
if (!_rio.RioReceive(_requestQueue, ref receiveBufferSeg, 1, RioReceiveFlags.None, 0))
{
ThrowError(ErrorType.Receive);
}
}
public async Task CanWriteNothingToBuffer()
{
using (var memoryPool = new MemoryPool())
{
var pipe = new Pipe(memoryPool);
var buffer = pipe.Alloc();
buffer.Advance(0); // doing nothing, the hard way
await buffer.FlushAsync();
}
}
private async void HandshakeWriting()
{
var writer = _handshakeOutpipe.Alloc();
_state.StartHandshake(ref writer);
if (writer.BytesWritten > 0)
{
await writer.FlushAsync();
}
else
{
writer.Commit();
}
while (true)
{
while (true)
{
var result = await _handshakeOutpipe.ReadAsync();
var buffer = result.Buffer;
if (result.IsCompleted && result.Buffer.IsEmpty)
{
break;
}
try
{
while (buffer.Length > 0)
{
ReadableBuffer messageBuffer;
if (buffer.Length <= RecordProcessor.PlainTextMaxSize)
{
messageBuffer = buffer;
buffer = buffer.Slice(buffer.End);
}
else
{
messageBuffer = buffer.Slice(0, RecordProcessor.PlainTextMaxSize);
buffer = buffer.Slice(RecordProcessor.PlainTextMaxSize);
}
writer = _lowerConnection.Output.Alloc();
RecordProcessor.WriteRecord(ref writer, RecordType.Handshake, messageBuffer, _state);
await writer.FlushAsync();
}
_state.DataForCurrentScheduleSent.Set();
}
finally
{
_handshakeOutpipe.AdvanceReader(buffer.Start, buffer.End);
}
}
}
}
private unsafe Uv.uv_buf_t OnAlloc(UvStreamHandle handle, int status)
{
_inputBuffer = _input.Alloc(2048);
void *pointer;
if (!_inputBuffer.Memory.TryGetPointer(out pointer))
{
throw new InvalidOperationException("Pointer must be pinned");
}
return(handle.Libuv.buf_init((IntPtr)pointer, _inputBuffer.Memory.Length));
}
public async Task CanAppendSelfWhileEmpty()
{ // not really an expectation; just an accepted caveat
using (var memoryPool = new MemoryPool())
{
var pipe = new Pipe(memoryPool);
var output = pipe.Alloc();
var readable = output.AsReadableBuffer();
output.Append(readable);
Assert.Equal(0, output.AsReadableBuffer().Length);
await output.FlushAsync();
}
}
public async Task CanWriteUInt64ToBuffer(ulong value, string valueAsString)
{
using (var memoryPool = new MemoryPool())
{
var pipe = new Pipe(memoryPool);
var buffer = pipe.Alloc();
buffer.Append(value, EncodingData.InvariantUtf8);
await buffer.FlushAsync();
var result = await pipe.ReadAsync();
var inputBuffer = result.Buffer;
Assert.Equal(valueAsString, inputBuffer.GetUtf8String());
}
}
public void CanReReadDataThatHasNotBeenCommitted_LargeData()
{
using (var memoryPool = new MemoryPool())
{
var pipe = new Pipe(memoryPool);
var output = pipe.Alloc();
byte[] predictablyGibberish = new byte[512];
const int SEED = 1235412;
Random random = new Random(SEED);
for (int i = 0; i < 50; i++)
{
for (int j = 0; j < predictablyGibberish.Length; j++)
{
// doing it this way to be 100% sure about repeating the PRNG order
predictablyGibberish[j] = (byte)random.Next(0, 256);
}
output.Write(predictablyGibberish);
}
var readable = output.AsReadableBuffer();
Assert.False(readable.IsSingleSpan);
Assert.False(readable.IsEmpty);
Assert.Equal(50 * 512, readable.Length);
random = new Random(SEED);
int correctCount = 0;
foreach (var memory in readable)
{
var span = memory.Span;
for (int i = 0; i < span.Length; i++)
{
if (span[i] == (byte)random.Next(0, 256))
{
correctCount++;
}
}
}
Assert.Equal(50 * 512, correctCount);
}
}
public void CanReReadDataThatHasNotBeenCommitted_SmallData()
{
using (var memoryPool = new MemoryPool())
{
var pipe = new Pipe(memoryPool);
var output = pipe.Alloc();
Assert.True(output.AsReadableBuffer().IsEmpty);
Assert.Equal(0, output.AsReadableBuffer().Length);
output.Append("hello world", EncodingData.InvariantUtf8);
var readable = output.AsReadableBuffer();
// check that looks about right
Assert.False(readable.IsEmpty);
Assert.Equal(11, readable.Length);
Assert.True(readable.Equals(Encoding.UTF8.GetBytes("hello world")));
Assert.True(readable.Slice(1, 3).Equals(Encoding.UTF8.GetBytes("ell")));
// check it all works after we write more
output.Append("more data", EncodingData.InvariantUtf8);
// note that the snapshotted readable should not have changed by this
Assert.False(readable.IsEmpty);
Assert.Equal(11, readable.Length);
Assert.True(readable.Equals(Encoding.UTF8.GetBytes("hello world")));
Assert.True(readable.Slice(1, 3).Equals(Encoding.UTF8.GetBytes("ell")));
// if we fetch it again, we can see everything
readable = output.AsReadableBuffer();
Assert.False(readable.IsEmpty);
Assert.Equal(20, readable.Length);
Assert.True(readable.Equals(Encoding.UTF8.GetBytes("hello worldmore data")));
}
}
private async void StartReading()
{
try
{
while (true)
{
var result = await _lowerConnection.Input.ReadAsync();
var buffer = result.Buffer;
try
{
ReadableBuffer messageBuffer;
while (RecordProcessor.TryGetFrame(ref buffer, out messageBuffer))
{
var recordType = RecordProcessor.ReadRecord(ref messageBuffer, _state);
if (_state == null)
{
if (recordType != RecordType.Handshake)
{
Alerts.AlertException.ThrowAlert(Alerts.AlertLevel.Fatal, Alerts.AlertDescription.unexpected_message, "Requre a handshake for first message");
}
_state = VersionStateFactory.GetNewStateMachine(messageBuffer, _listener);
HandshakeWriting();
}
Console.WriteLine($"Received TLS frame {recordType}");
if (recordType == RecordType.Handshake)
{
var writer = _handshakePipe.Alloc();
writer.Append(messageBuffer);
await writer.FlushAsync();
await HandshakeReading();
if (_state.State == StateType.HandshakeComplete && !_startedApplicationWrite)
{
ApplicationWriting();
_startedApplicationWrite = true;
}
continue;
}
if (recordType == RecordType.Alert)
{
_state.HandleAlertMessage(messageBuffer);
continue;
}
if (recordType == RecordType.Application)
{
Console.WriteLine("Writing Application Data");
var writer = _outputPipe.Alloc();
writer.Append(messageBuffer);
await writer.FlushAsync();
continue;
}
Alerts.AlertException.ThrowAlert(Alerts.AlertLevel.Fatal, Alerts.AlertDescription.unexpected_message, $"Unknown message type {recordType}");
}
if (result.IsCompleted)
{
return;
}
}
finally
{
_lowerConnection.Input.Advance(buffer.Start, buffer.End);
}
}
}
catch
{
//nom nom
Dispose();
}
}
private async void ReceiveFromSocketAndPushToWriterAsync()
{
SocketAsyncEventArgs args = null;
try
{
// if the consumer says they don't want the data, we need to shut down the receive
GC.KeepAlive(_input.Writing.ContinueWith(delegate
{// GC.KeepAlive here just to shut the compiler up
try { Socket.Shutdown(SocketShutdown.Receive); } catch { }
}));
// wait for someone to be interested in data before we
// start allocating buffers and probing the socket
await _input.ReadingStarted;
args = GetOrCreateSocketAsyncEventArgs();
while (!_input.Writing.IsCompleted)
{
bool haveWriteBuffer = false;
WritableBuffer buffer = default(WritableBuffer);
var initialSegment = default(ArraySegment <byte>);
try
{
int bytesFromInitialDataBuffer = 0;
if (Socket.Available == 0)
{
// now, this gets a bit messy unfortunately, because support for the ideal option
// (zero-length reads) is platform dependent
switch (_bufferStyle)
{
case BufferStyle.Unknown:
try
{
initialSegment = await ReceiveInitialDataUnknownStrategyAsync(args);
}
catch
{
initialSegment = default(ArraySegment <byte>);
}
if (initialSegment.Array == null)
{
continue; // redo from start
}
break;
case BufferStyle.UseZeroLengthBuffer:
// if we already have a buffer, use that (but: zero count); otherwise use a shared
// zero-length; this avoids constantly changing the buffer that the args use, which
// avoids some overheads
args.SetBuffer(args.Buffer ?? _zeroLengthBuffer, 0, 0);
// await async for the io work to be completed
await Socket.ReceiveSignalAsync(args);
break;
case BufferStyle.UseSmallBuffer:
// We need to do a speculative receive with a *cheap* buffer while we wait for input; it would be *nice* if
// we could do a zero-length receive, but this is not supported equally on all platforms (fine on Windows, but
// linux hates it). The key aim here is to make sure that we don't tie up an entire block from the memory pool
// waiting for input on a socket; fine for 1 socket, not so fine for 100,000 sockets
// do a short receive while we wait (async) for data
initialSegment = LeaseSmallBuffer();
args.SetBuffer(initialSegment.Array, initialSegment.Offset, initialSegment.Count);
// await async for the io work to be completed
await Socket.ReceiveSignalAsync(args);
break;
}
if (args.SocketError != SocketError.Success)
{
throw new SocketException((int)args.SocketError);
}
// note we can't check BytesTransferred <= 0, as we always
// expect 0; but if we returned, we expect data to be
// buffered *on the socket*, else EOF
if ((bytesFromInitialDataBuffer = args.BytesTransferred) <= 0)
{
if (ReferenceEquals(initialSegment.Array, _zeroLengthBuffer))
{
// sentinel value that means we should just
// consume sync (we expect there to be data)
initialSegment = default(ArraySegment <byte>);
}
else
{
// socket reported EOF
RecycleSmallBuffer(ref initialSegment);
}
if (Socket.Available == 0)
{
// yup, definitely an EOF
break;
}
}
}
// note that we will try to coalesce things here to reduce the number of flushes; we
// certainly want to coalesce the initial buffer (from the speculative receive) with the initial
// data, but we probably don't want to buffer indefinitely; for now, it will buffer up to 4 pages
// before flushing (entirely arbitrarily) - might want to make this configurable later
buffer = _input.Alloc(SmallBufferSize * 2);
haveWriteBuffer = true;
const int FlushInputEveryBytes = 4 * MemoryPool.MaxPooledBlockLength;
if (initialSegment.Array != null)
{
// need to account for anything that we got in the speculative receive
if (bytesFromInitialDataBuffer != 0)
{
buffer.Write(new Span <byte>(initialSegment.Array, initialSegment.Offset, bytesFromInitialDataBuffer));
}
// make the small buffer available to other consumers
RecycleSmallBuffer(ref initialSegment);
}
bool isEOF = false;
while (Socket.Available != 0 && buffer.BytesWritten < FlushInputEveryBytes)
{
buffer.Ensure(); // ask for *something*, then use whatever is available (usually much much more)
SetBuffer(buffer.Memory, args);
// await async for the io work to be completed
await Socket.ReceiveSignalAsync(args);
// either way, need to validate
if (args.SocketError != SocketError.Success)
{
throw new SocketException((int)args.SocketError);
}
int len = args.BytesTransferred;
if (len <= 0)
{
// socket reported EOF
isEOF = true;
break;
}
// record what data we filled into the buffer
buffer.Advance(len);
}
if (isEOF)
{
break;
}
}
finally
{
RecycleSmallBuffer(ref initialSegment);
if (haveWriteBuffer)
{
await buffer.FlushAsync();
}
}
}
_input.CompleteWriter();
}
catch (Exception ex)
{
// don't trust signal after an error; someone else could
// still have it and invoke Set
if (args != null)
{
args.UserToken = null;
}
_input?.CompleteWriter(ex);
}
finally
{
RecycleSocketAsyncEventArgs(args);
}
}
public WritableBuffer Alloc(int minimumSize = 0)
{
return(_pipe.Alloc(minimumSize));
}