public async Task AutobahnTestSuite()
{
// If we're on CI, we want to actually fail if WsTest isn't installed, rather than just skipping the test
// The SkipIfWsTestNotPresent attribute ensures that this test isn't skipped on CI, so we just need to check that Wstest is present
// And we use Assert.True to provide an error message
Assert.True(Wstest.Default != null, $"The 'wstest' executable (Autobahn GameSockets Test Suite) could not be found at '{Wstest.DefaultLocation}'. Run the Build Agent setup scripts to install it or see https://github.com/crossbario/autobahn-testsuite for instructions on manual installation.");
using (StartLog(out var loggerFactory))
{
var logger = loggerFactory.CreateLogger <AutobahnTests>();
var reportDir = Environment.GetEnvironmentVariable("AUTOBAHN_SUITES_REPORT_DIR");
var outDir = !string.IsNullOrEmpty(reportDir) ?
reportDir :
Path.Combine(AppContext.BaseDirectory, "autobahnreports");
if (Directory.Exists(outDir))
{
Directory.Delete(outDir, recursive: true);
}
outDir = outDir.Replace("\\", "\\\\");
// 9.* is Limits/Performance which is VERY SLOW; 12.*/13.* are compression which we don't implement
var spec = new AutobahnSpec(outDir)
.IncludeCase("*")
.ExcludeCase("9.*", "12.*", "13.*");
var cts = new CancellationTokenSource();
cts.CancelAfter(TestTimeout); // These tests generally complete in just over 1 minute.
using (cts.Token.Register(() => logger.LogError("Test run is taking longer than maximum duration of {timeoutMinutes:0.00} minutes. Aborting...", TestTimeout.TotalMinutes)))
{
AutobahnResult result;
using (var tester = new AutobahnTester(loggerFactory, spec))
{
await tester.DeployTestAndAddToSpec(ServerType.Kestrel, ssl : false, environment : "ManagedSockets", cancellationToken : cts.Token);
await tester.DeployTestAndAddToSpec(ServerType.Kestrel, ssl : true, environment : "ManagedSockets", cancellationToken : cts.Token);
// Windows-only GameListener tests
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
if (IsWindows8OrHigher())
{
// GameListener occasionally gives a non-strict response on 3.2. IIS Express seems to have the same behavior. Wonder if it's related to HttpSys?
// For now, just allow the non-strict response, it's not a failure.
await tester.DeployTestAndAddToSpec(ServerType.HttpSys, ssl : false, environment : "ManagedSockets", cancellationToken : cts.Token);
}
}
result = await tester.Run(cts.Token);
tester.Verify(result);
}
}
// If it hasn't been cancelled yet, cancel the token just to be sure
cts.Cancel();
}
}
public AutobahnTester(ILoggerFactory loggerFactory, AutobahnSpec baseSpec)
{
_loggerFactory = loggerFactory;
_logger = _loggerFactory.CreateLogger("AutobahnTester");
Spec = baseSpec;
}
public async Task AutobahnTestSuite()
{
using (StartLog(out var loggerFactory))
{
var logger = loggerFactory.CreateLogger <AutobahnTests>();
var reportDir = Environment.GetEnvironmentVariable("AUTOBAHN_SUITES_REPORT_DIR");
var outDir = !string.IsNullOrEmpty(reportDir) ?
reportDir :
Path.Combine(AppContext.BaseDirectory, "autobahnreports");
if (Directory.Exists(outDir))
{
Directory.Delete(outDir, recursive: true);
}
outDir = outDir.Replace("\\", "\\\\");
// 9.* is Limits/Performance which is VERY SLOW; 12.*/13.* are compression which we don't implement
var spec = new AutobahnSpec(outDir)
.IncludeCase("*")
.ExcludeCase("9.*", "12.*", "13.*");
var cts = new CancellationTokenSource();
cts.CancelAfter(TestTimeout); // These tests generally complete in just over 1 minute.
using (cts.Token.Register(() => logger.LogError("Test run is taking longer than maximum duration of {timeoutMinutes:0.00} minutes. Aborting...", TestTimeout.TotalMinutes)))
{
AutobahnResult result;
using (var tester = new AutobahnTester(loggerFactory, spec))
{
await tester.DeployTestAndAddToSpec(ServerType.Kestrel, ssl : false, environment : "ManagedSockets", cancellationToken : cts.Token);
// Windows-only WebListener tests, and Kestrel SSL tests (due to: https://github.com/aspnet/WebSockets/issues/102)
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
await tester.DeployTestAndAddToSpec(ServerType.Kestrel, ssl : true, environment : "ManagedSockets", cancellationToken : cts.Token);
if (IsWindows8OrHigher())
{
// WebListener occasionally gives a non-strict response on 3.2. IIS Express seems to have the same behavior. Wonder if it's related to HttpSys?
// For now, just allow the non-strict response, it's not a failure.
await tester.DeployTestAndAddToSpec(ServerType.WebListener, ssl : false, environment : "ManagedSockets", cancellationToken : cts.Token);
}
}
result = await tester.Run(cts.Token);
tester.Verify(result);
}
}
// If it hasn't been cancelled yet, cancel the token just to be sure
cts.Cancel();
}
}
public async Task AutobahnTestSuite()
{
var reportDir = Environment.GetEnvironmentVariable("AUTOBAHN_SUITES_REPORT_DIR");
var outDir = !string.IsNullOrEmpty(reportDir) ?
reportDir :
Path.Combine(PlatformServices.Default.Application.ApplicationBasePath, "autobahnreports");
if (Directory.Exists(outDir))
{
Directory.Delete(outDir, recursive: true);
}
outDir = outDir.Replace("\\", "\\\\");
// 9.* is Limits/Performance which is VERY SLOW; 12.*/13.* are compression which we don't implement
var spec = new AutobahnSpec(outDir)
.IncludeCase("*")
.ExcludeCase("9.*", "12.*", "13.*");
var loggerFactory = new LoggerFactory(); // No logging by default! It's very loud...
if (string.Equals(Environment.GetEnvironmentVariable("AUTOBAHN_SUITES_LOG"), "1", StringComparison.Ordinal))
{
loggerFactory.AddConsole();
}
AutobahnResult result;
using (var tester = new AutobahnTester(loggerFactory, spec))
{
await tester.DeployTestAndAddToSpec(ServerType.Kestrel, ssl : false, expectationConfig : expect => expect
.NonStrict("6.4.3", "6.4.4"));
result = await tester.Run();
tester.Verify(result);
}
}
