Beispiel #1
0
        public async Task AutobahnTestSuite()
        {
            // If we're on CI, we want to actually fail if WsTest isn't installed, rather than just skipping the test
            // The SkipIfWsTestNotPresent attribute ensures that this test isn't skipped on CI, so we just need to check that Wstest is present
            // And we use Assert.True to provide an error message
            Assert.True(Wstest.Default != null, $"The 'wstest' executable (Autobahn GameSockets Test Suite) could not be found at '{Wstest.DefaultLocation}'. Run the Build Agent setup scripts to install it or see https://github.com/crossbario/autobahn-testsuite for instructions on manual installation.");

            using (StartLog(out var loggerFactory))
            {
                var logger    = loggerFactory.CreateLogger <AutobahnTests>();
                var reportDir = Environment.GetEnvironmentVariable("AUTOBAHN_SUITES_REPORT_DIR");
                var outDir    = !string.IsNullOrEmpty(reportDir) ?
                                reportDir :
                                Path.Combine(AppContext.BaseDirectory, "autobahnreports");

                if (Directory.Exists(outDir))
                {
                    Directory.Delete(outDir, recursive: true);
                }

                outDir = outDir.Replace("\\", "\\\\");

                // 9.* is Limits/Performance which is VERY SLOW; 12.*/13.* are compression which we don't implement
                var spec = new AutobahnSpec(outDir)
                           .IncludeCase("*")
                           .ExcludeCase("9.*", "12.*", "13.*");

                var cts = new CancellationTokenSource();
                cts.CancelAfter(TestTimeout); // These tests generally complete in just over 1 minute.

                using (cts.Token.Register(() => logger.LogError("Test run is taking longer than maximum duration of {timeoutMinutes:0.00} minutes. Aborting...", TestTimeout.TotalMinutes)))
                {
                    AutobahnResult result;
                    using (var tester = new AutobahnTester(loggerFactory, spec))
                    {
                        await tester.DeployTestAndAddToSpec(ServerType.Kestrel, ssl : false, environment : "ManagedSockets", cancellationToken : cts.Token);

                        await tester.DeployTestAndAddToSpec(ServerType.Kestrel, ssl : true, environment : "ManagedSockets", cancellationToken : cts.Token);

                        // Windows-only GameListener tests
                        if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
                        {
                            if (IsWindows8OrHigher())
                            {
                                // GameListener occasionally gives a non-strict response on 3.2. IIS Express seems to have the same behavior. Wonder if it's related to HttpSys?
                                // For now, just allow the non-strict response, it's not a failure.
                                await tester.DeployTestAndAddToSpec(ServerType.HttpSys, ssl : false, environment : "ManagedSockets", cancellationToken : cts.Token);
                            }
                        }

                        result = await tester.Run(cts.Token);

                        tester.Verify(result);
                    }
                }

                // If it hasn't been cancelled yet, cancel the token just to be sure
                cts.Cancel();
            }
        }
Beispiel #2
0
    public AutobahnTester(ILoggerFactory loggerFactory, AutobahnSpec baseSpec)
    {
        _loggerFactory = loggerFactory;
        _logger        = _loggerFactory.CreateLogger("AutobahnTester");

        Spec = baseSpec;
    }
Beispiel #3
0
        public async Task AutobahnTestSuite()
        {
            using (StartLog(out var loggerFactory))
            {
                var logger    = loggerFactory.CreateLogger <AutobahnTests>();
                var reportDir = Environment.GetEnvironmentVariable("AUTOBAHN_SUITES_REPORT_DIR");
                var outDir    = !string.IsNullOrEmpty(reportDir) ?
                                reportDir :
                                Path.Combine(AppContext.BaseDirectory, "autobahnreports");

                if (Directory.Exists(outDir))
                {
                    Directory.Delete(outDir, recursive: true);
                }

                outDir = outDir.Replace("\\", "\\\\");

                // 9.* is Limits/Performance which is VERY SLOW; 12.*/13.* are compression which we don't implement
                var spec = new AutobahnSpec(outDir)
                           .IncludeCase("*")
                           .ExcludeCase("9.*", "12.*", "13.*");

                var cts = new CancellationTokenSource();
                cts.CancelAfter(TestTimeout); // These tests generally complete in just over 1 minute.

                using (cts.Token.Register(() => logger.LogError("Test run is taking longer than maximum duration of {timeoutMinutes:0.00} minutes. Aborting...", TestTimeout.TotalMinutes)))
                {
                    AutobahnResult result;
                    using (var tester = new AutobahnTester(loggerFactory, spec))
                    {
                        await tester.DeployTestAndAddToSpec(ServerType.Kestrel, ssl : false, environment : "ManagedSockets", cancellationToken : cts.Token);

                        // Windows-only WebListener tests, and Kestrel SSL tests (due to: https://github.com/aspnet/WebSockets/issues/102)
                        if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
                        {
                            await tester.DeployTestAndAddToSpec(ServerType.Kestrel, ssl : true, environment : "ManagedSockets", cancellationToken : cts.Token);

                            if (IsWindows8OrHigher())
                            {
                                // WebListener occasionally gives a non-strict response on 3.2. IIS Express seems to have the same behavior. Wonder if it's related to HttpSys?
                                // For now, just allow the non-strict response, it's not a failure.
                                await tester.DeployTestAndAddToSpec(ServerType.WebListener, ssl : false, environment : "ManagedSockets", cancellationToken : cts.Token);
                            }
                        }

                        result = await tester.Run(cts.Token);

                        tester.Verify(result);
                    }
                }

                // If it hasn't been cancelled yet, cancel the token just to be sure
                cts.Cancel();
            }
        }
Beispiel #4
0
        public async Task AutobahnTestSuite()
        {
            var reportDir = Environment.GetEnvironmentVariable("AUTOBAHN_SUITES_REPORT_DIR");
            var outDir    = !string.IsNullOrEmpty(reportDir) ?
                            reportDir :
                            Path.Combine(PlatformServices.Default.Application.ApplicationBasePath, "autobahnreports");

            if (Directory.Exists(outDir))
            {
                Directory.Delete(outDir, recursive: true);
            }

            outDir = outDir.Replace("\\", "\\\\");

            // 9.* is Limits/Performance which is VERY SLOW; 12.*/13.* are compression which we don't implement
            var spec = new AutobahnSpec(outDir)
                       .IncludeCase("*")
                       .ExcludeCase("9.*", "12.*", "13.*");

            var loggerFactory = new LoggerFactory(); // No logging by default! It's very loud...

            if (string.Equals(Environment.GetEnvironmentVariable("AUTOBAHN_SUITES_LOG"), "1", StringComparison.Ordinal))
            {
                loggerFactory.AddConsole();
            }

            AutobahnResult result;

            using (var tester = new AutobahnTester(loggerFactory, spec))
            {
                await tester.DeployTestAndAddToSpec(ServerType.Kestrel, ssl : false, expectationConfig : expect => expect
                                                    .NonStrict("6.4.3", "6.4.4"));

                result = await tester.Run();

                tester.Verify(result);
            }
        }