public async Task RunIndividualTestCase(H2SpecTestCase testCase)
{
var memoryPoolFactory = new DiagnosticMemoryPoolFactory(allowLateReturn: true);
var hostBuilder = TransportSelector.GetWebHostBuilder(memoryPoolFactory.Create)
.UseKestrel(options =>
{
options.Listen(IPAddress.Loopback, 0, listenOptions =>
{
listenOptions.Protocols = HttpProtocols.Http2;
if (testCase.Https)
{
listenOptions.UseHttps(TestResources.TestCertificatePath, "testPassword");
}
});
})
.ConfigureServices(AddTestLogging)
.Configure(ConfigureHelloWorld);
using (var host = hostBuilder.Build())
{
await host.StartAsync();
H2SpecCommands.RunTest(testCase.Id, host.GetPort(), testCase.Https, Logger);
}
}
public async Task GracefulTurnsAbortiveIfRequestsDoNotFinish()
{
var requestStarted = new TaskCompletionSource <object>(TaskCreationOptions.RunContinuationsAsynchronously);
var requestUnblocked = new TaskCompletionSource <object>(TaskCreationOptions.RunContinuationsAsynchronously);
var memoryPoolFactory = new DiagnosticMemoryPoolFactory(allowLateReturn: true);
var testContext = new TestServiceContext(LoggerFactory)
{
MemoryPoolFactory = memoryPoolFactory.Create,
ExpectedConnectionMiddlewareCount = 1
};
TestApplicationErrorLogger.ThrowOnUngracefulShutdown = false;
// Abortive shutdown leaves one request hanging
using (var server = new TestServer(async context =>
{
requestStarted.SetResult(null);
await requestUnblocked.Task.DefaultTimeout();
await context.Response.WriteAsync("hello world " + context.Request.Protocol);
},
testContext,
kestrelOptions =>
{
kestrelOptions.Listen(IPAddress.Loopback, 0, listenOptions =>
{
listenOptions.Protocols = HttpProtocols.Http2;
listenOptions.UseHttps(_x509Certificate2);
});
},
_ => { }))
{
var requestTask = Client.GetStringAsync($"https://localhost:{server.Port}/");
Assert.False(requestTask.IsCompleted);
await requestStarted.Task.DefaultTimeout();
// Wait for the graceful shutdown log before canceling the token passed to StopAsync and triggering an ungraceful shutdown.
// Otherwise, graceful shutdown might be skipped causing there to be no corresponding log. https://github.com/aspnet/AspNetCore/issues/6556
var closingMessageTask = TestApplicationErrorLogger.WaitForMessage(m => m.Message.Contains("is closing.")).DefaultTimeout();
var cts = new CancellationTokenSource();
var stopServerTask = server.StopAsync(cts.Token).DefaultTimeout();
await closingMessageTask;
cts.Cancel();
await stopServerTask;
}
Assert.Contains(TestApplicationErrorLogger.Messages, m => m.Message.Contains("is closing."));
Assert.Contains(TestApplicationErrorLogger.Messages, m => m.Message.Contains("is closed. The last processed stream ID was 1."));
Assert.Contains(TestApplicationErrorLogger.Messages, m => m.Message.Contains("Some connections failed to close gracefully during server shutdown."));
Assert.DoesNotContain(TestApplicationErrorLogger.Messages, m => m.Message.Contains("Request finished in"));
requestUnblocked.SetResult(null);
await memoryPoolFactory.WhenAllBlocksReturned(TestConstants.DefaultTimeout);
}
public async Task GracefulTurnsAbortiveIfRequestsDoNotFinish()
{
var requestStarted = new TaskCompletionSource <object>(TaskCreationOptions.RunContinuationsAsynchronously);
var requestUnblocked = new TaskCompletionSource <object>(TaskCreationOptions.RunContinuationsAsynchronously);
var memoryPoolFactory = new DiagnosticMemoryPoolFactory(allowLateReturn: true);
var testContext = new TestServiceContext(LoggerFactory)
{
MemoryPoolFactory = memoryPoolFactory.Create
};
TestApplicationErrorLogger.ThrowOnUngracefulShutdown = false;
// Abortive shutdown leaves one request hanging
using (var server = new TestServer(async context =>
{
requestStarted.SetResult(null);
await requestUnblocked.Task.DefaultTimeout();
await context.Response.WriteAsync("hello world " + context.Request.Protocol);
},
testContext,
kestrelOptions =>
{
kestrelOptions.Listen(IPAddress.Loopback, 0, listenOptions =>
{
listenOptions.Protocols = HttpProtocols.Http2;
listenOptions.UseHttps(_x509Certificate2);
});
},
_ => { }))
{
var requestTask = Client.GetStringAsync($"https://localhost:{server.Port}/");
Assert.False(requestTask.IsCompleted);
await requestStarted.Task.DefaultTimeout();
await server.StopAsync(new CancellationToken(true)).DefaultTimeout();
}
Assert.Contains(TestApplicationErrorLogger.Messages, m => m.Message.Contains("is closing."));
Assert.Contains(TestApplicationErrorLogger.Messages, m => m.Message.Contains("is closed. The last processed stream ID was 1."));
Assert.Contains(TestApplicationErrorLogger.Messages, m => m.Message.Contains("Some connections failed to close gracefully during server shutdown."));
Assert.DoesNotContain(TestApplicationErrorLogger.Messages, m => m.Message.Contains("Request finished in"));
requestUnblocked.SetResult(null);
await memoryPoolFactory.WhenAllBlocksReturned(TestConstants.DefaultTimeout);
}
public async Task ServerShutsDownGracefullyWhenMaxRequestBufferSizeExceeded()
{
// Parameters
var data = new byte[_dataLength];
var bytesWrittenTimeout = TimeSpan.FromMilliseconds(100);
var bytesWrittenPollingInterval = TimeSpan.FromMilliseconds(bytesWrittenTimeout.TotalMilliseconds / 10);
var maxSendSize = 4096;
var startReadingRequestBody = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var clientFinishedSendingRequestBody = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var lastBytesWritten = DateTime.MaxValue;
var memoryPoolFactory = new DiagnosticMemoryPoolFactory(allowLateReturn: true);
using (var host = await StartHost(16 * 1024, data, false, startReadingRequestBody, clientFinishedSendingRequestBody, memoryPoolFactory.Create))
{
var port = host.GetPort();
using (var socket = CreateSocket(port))
using (var stream = new NetworkStream(socket))
{
await WritePostRequestHeaders(stream, data.Length);
var bytesWritten = 0;
Func <Task> sendFunc = async() =>
{
while (bytesWritten < data.Length)
{
var size = Math.Min(data.Length - bytesWritten, maxSendSize);
await stream.WriteAsync(data, bytesWritten, size).ConfigureAwait(false);
bytesWritten += size;
lastBytesWritten = DateTime.Now;
}
clientFinishedSendingRequestBody.TrySetResult();
};
var ignore = sendFunc();
// The minimum is (maxRequestBufferSize - maxSendSize + 1), since if bytesWritten is
// (maxRequestBufferSize - maxSendSize) or smaller, the client should be able to
// complete another send.
var minimumExpectedBytesWritten = (16 * 1024) - maxSendSize + 1;
// The maximum is harder to determine, since there can be OS-level buffers in both the client
// and server, which allow the client to send more than maxRequestBufferSize before getting
// paused. We assume the combined buffers are smaller than the difference between
// data.Length and maxRequestBufferSize.
var maximumExpectedBytesWritten = data.Length - 1;
// Block until the send task has gone a while without writing bytes AND
// the bytes written exceeds the minimum expected. This indicates the server buffer
// is full.
//
// If the send task is paused before the expected number of bytes have been
// written, keep waiting since the pause may have been caused by something else
// like a slow machine.
while ((DateTime.Now - lastBytesWritten) < bytesWrittenTimeout ||
bytesWritten < minimumExpectedBytesWritten)
{
await Task.Delay(bytesWrittenPollingInterval);
}
// Verify the number of bytes written before the client was paused.
Assert.InRange(bytesWritten, minimumExpectedBytesWritten, maximumExpectedBytesWritten);
// Dispose host prior to closing connection to verify the server doesn't throw during shutdown
// if a connection no longer has alloc and read callbacks configured.
try
{
await host.StopAsync();
}
// Remove when https://github.com/dotnet/runtime/issues/40290 is fixed
catch (OperationCanceledException)
{
}
host.Dispose();
}
}
// Allow appfunc to unblock
startReadingRequestBody.SetResult();
clientFinishedSendingRequestBody.SetResult();
await memoryPoolFactory.WhenAllBlocksReturned(TestConstants.DefaultTimeout);
}
// This is inherently flaky and is relying on helix retry to pass consistently
public async Task LargeUpload(long?maxRequestBufferSize, bool connectionAdapter, bool expectPause)
{
// Parameters
var data = new byte[_dataLength];
var bytesWrittenTimeout = TimeSpan.FromMilliseconds(100);
var bytesWrittenPollingInterval = TimeSpan.FromMilliseconds(bytesWrittenTimeout.TotalMilliseconds / 10);
var maxSendSize = 4096;
var startReadingRequestBody = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var clientFinishedSendingRequestBody = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var lastBytesWritten = DateTime.MaxValue;
var memoryPoolFactory = new DiagnosticMemoryPoolFactory(allowLateReturn: true);
using (var host = await StartHost(maxRequestBufferSize, data, connectionAdapter, startReadingRequestBody, clientFinishedSendingRequestBody, memoryPoolFactory.Create))
{
var port = host.GetPort();
using (var socket = CreateSocket(port))
using (var stream = new NetworkStream(socket))
{
await WritePostRequestHeaders(stream, data.Length);
var bytesWritten = 0;
Func <Task> sendFunc = async() =>
{
while (bytesWritten < data.Length)
{
var size = Math.Min(data.Length - bytesWritten, maxSendSize);
await stream.WriteAsync(data, bytesWritten, size).ConfigureAwait(false);
bytesWritten += size;
lastBytesWritten = DateTime.Now;
}
Assert.Equal(data.Length, bytesWritten);
clientFinishedSendingRequestBody.TrySetResult();
};
var sendTask = sendFunc();
if (expectPause)
{
// The minimum is (maxRequestBufferSize - maxSendSize + 1), since if bytesWritten is
// (maxRequestBufferSize - maxSendSize) or smaller, the client should be able to
// complete another send.
var minimumExpectedBytesWritten = maxRequestBufferSize.Value - maxSendSize + 1;
// The maximum is harder to determine, since there can be OS-level buffers in both the client
// and server, which allow the client to send more than maxRequestBufferSize before getting
// paused. We assume the combined buffers are smaller than the difference between
// data.Length and maxRequestBufferSize.
var maximumExpectedBytesWritten = data.Length - 1;
// Block until the send task has gone a while without writing bytes AND
// the bytes written exceeds the minimum expected. This indicates the server buffer
// is full.
//
// If the send task is paused before the expected number of bytes have been
// written, keep waiting since the pause may have been caused by something else
// like a slow machine.
while ((DateTime.Now - lastBytesWritten) < bytesWrittenTimeout ||
bytesWritten < minimumExpectedBytesWritten)
{
await Task.Delay(bytesWrittenPollingInterval);
}
// Verify the number of bytes written before the client was paused.
Assert.InRange(bytesWritten, minimumExpectedBytesWritten, maximumExpectedBytesWritten);
// Tell server to start reading request body
startReadingRequestBody.TrySetResult();
// Wait for sendTask to finish sending the remaining bytes
await sendTask;
}
else
{
// Ensure all bytes can be sent before the server starts reading
await sendTask;
// Tell server to start reading request body
startReadingRequestBody.TrySetResult();
}
await AssertStreamContains(stream, $"bytesRead: {data.Length}");
}
await host.StopAsync();
}
await memoryPoolFactory.WhenAllBlocksReturned(TestConstants.DefaultTimeout);
}
