public async Task SerializeAsync(Stream stream, DataSegment segment, Random?random = null, CancellationToken token = default)
{
// length
int numsize = s_encoding.GetBytes(segment.Length.ToString(), _buffer);
await stream.WriteAsync(_buffer.AsMemory(0, numsize), token);
stream.WriteByte((byte)',');
// checksum
numsize = s_encoding.GetBytes(segment.Checksum.ToString(), _buffer);
await stream.WriteAsync(_buffer.AsMemory(0, numsize), token);
stream.WriteByte((byte)',');
// payload
Memory <byte> source = segment.AsMemory();
// write the entire segment outright if not given random instance
if (random == null)
{
await stream.WriteAsync(source, token);
return;
}
// randomize chunking otherwise
while (source.Length > 0)
{
if (random.NextBoolean(probability: 0.05))
{
stream.WriteByte(source.Span[0]);
source = source.Slice(1);
}
else
{
// TODO consider non-uniform distribution for chunk sizes
int chunkSize = random.Next(source.Length);
Memory <byte> chunk = source.Slice(0, chunkSize);
source = source.Slice(chunkSize);
if (random.NextBoolean(probability: 0.9))
{
await stream.WriteAsync(chunk, token);
}
else
{
stream.Write(chunk.Span);
}
}
if (random.NextBoolean(probability: 0.3))
{
await stream.FlushAsync(token);
}
// randomized delay
if (random.NextBoolean(probability: 0.05))
{
if (random.NextBoolean(probability: 0.7))
{
await Task.Delay(random.Next(60));
}
else
{
Thread.SpinWait(random.Next(1000));
}
}
}
}
protected override async Task HandleConnection(int workerId, long jobId, SslStream stream, TcpClient client, Random random, TimeSpan duration, CancellationToken token)
{
// token used for signalling cooperative cancellation; do not pass this to SslStream methods
using var connectionLifetimeToken = new CancellationTokenSource(duration);
long messagesInFlight = 0;
DateTime lastWrite = DateTime.Now;
DateTime lastRead = DateTime.Now;
await StressTaskExtensions.WhenAllThrowOnFirstException(token, Sender, Receiver, Monitor);
async Task Sender(CancellationToken token)
{
var serializer = new DataSegmentSerializer();
while (!token.IsCancellationRequested && !connectionLifetimeToken.IsCancellationRequested)
{
await ApplyBackpressure();
DataSegment chunk = DataSegment.CreateRandom(random, _config.MaxBufferLength);
try
{
await serializer.SerializeAsync(stream, chunk, random, token);
stream.WriteByte((byte)'\n');
await stream.FlushAsync(token);
Interlocked.Increment(ref messagesInFlight);
lastWrite = DateTime.Now;
}
finally
{
chunk.Return();
}
}
// write an empty line to signal completion to the server
stream.WriteByte((byte)'\n');
await stream.FlushAsync(token);
/// Polls until number of in-flight messages falls below threshold
async Task ApplyBackpressure()
{
if (Volatile.Read(ref messagesInFlight) > 5000)
{
Stopwatch stopwatch = Stopwatch.StartNew();
bool isLogged = false;
while (!token.IsCancellationRequested && !connectionLifetimeToken.IsCancellationRequested && Volatile.Read(ref messagesInFlight) > 2000)
{
// only log if tx has been suspended for a while
if (!isLogged && stopwatch.ElapsedMilliseconds >= 1000)
{
isLogged = true;
lock (Console.Out)
{
Console.ForegroundColor = ConsoleColor.Yellow;
Console.WriteLine($"worker #{workerId}: applying backpressure");
Console.WriteLine();
Console.ResetColor();
}
}
await Task.Delay(20);
}
if (isLogged)
{
Console.WriteLine($"worker #{workerId}: resumed tx after {stopwatch.Elapsed}");
}
}
}
}
async Task Receiver(CancellationToken token)
{
var serializer = new DataSegmentSerializer();
using var cts = CancellationTokenSource.CreateLinkedTokenSource(token);
await stream.ReadLinesUsingPipesAsync(Callback, cts.Token, separator : '\n');
Task Callback(ReadOnlySequence <byte> buffer)
{
if (buffer.Length == 0 && connectionLifetimeToken.IsCancellationRequested)
{
// server echoed back empty buffer sent by client,
// signal cancellation and complete the connection.
cts.Cancel();
return(Task.CompletedTask);
}
// deserialize to validate the checksum, then discard
DataSegment chunk = serializer.Deserialize(buffer);
chunk.Return();
Interlocked.Decrement(ref messagesInFlight);
lastRead = DateTime.Now;
return(Task.CompletedTask);
}
}
async Task Monitor(CancellationToken token)
{
do
{
await Task.Delay(500);
if ((DateTime.Now - lastWrite) >= TimeSpan.FromSeconds(10))
{
throw new Exception($"worker #{workerId} job #{jobId} has stopped writing bytes to server");
}
if ((DateTime.Now - lastRead) >= TimeSpan.FromSeconds(10))
{
throw new Exception($"worker #{workerId} job #{jobId} has stopped receiving bytes from server");
}
}while(!token.IsCancellationRequested && !connectionLifetimeToken.IsCancellationRequested);
}
}
