Beispiel #1
0
        public static void RunTestsBatchLocal(string[] files, Config config)
        {
            List <string> expectedOutput;

            byte[] payload = PrepareBatch(files, config.DebugDumpFile, out expectedOutput);

            int numberOfTests = files.Length;

            Logger.LogLine($"Running a batch of {numberOfTests} tests");
            Stopwatch sw = new Stopwatch();

            sw.Start();
            RunResult result = ExecuteLocal(payload, Utility.GetProcess(config), Utility.GetArenaHost(config), numberOfTests * 50, numberOfTests * 50 + 5, config.DebugDumpFile);

            sw.Stop();
            string time = TimeFormatter.FormatTime(sw.Elapsed);

            if (numberOfTests != result.Output?.Count || result.Output?.Count != result.Debug?.Count)
            {
                Logger.LogLine($"Error: Unexpected response. Tests:{numberOfTests}, Output:{result.Output?.Count}, Debug: {result.Debug?.Count}");
                ShowWarnings(result.Warnings);
                return;
            }

            // In the batch mode backed returns a single set of warnings for all the tests. Display them here
            if (config.DisplayDebugInfoOnSuccess)
            {
                ShowWarnings(result.Warnings);
            }

            int successes = 0;

            for (int i = 0; i < numberOfTests; i++)
            {
                DisplayTestResultParams p = new DisplayTestResultParams
                {
                    TestName       = GetTestName(files[i]),
                    ExpectedOutput = expectedOutput[i],
                    Output         = config.TrimWhitespacesFromResults ? Utility.TrimWhitespaces(result.Output[i]) : result.Output[i],
                    Debug          = result.Debug[i]
                };
                DisplayTestResult(p, config);
                if (p.Success)
                {
                    successes++;
                }
            }
            Logger.LogLine($"Elapsed: {time}");
            Logger.LogLine($"Result: {successes} succeeded, {numberOfTests - successes} failed");
        }
Beispiel #2
0
        public static bool RunSingleTest(string file, Config config, string counter)
        {
            string testName = GetTestName(file);

            if (config.UseConsoleCodes)
            {
                Logger.Log($"{counter} {testName}...");
            }

            TestDescription test = JsonConvert.DeserializeObject <TestDescription>(Encoding.UTF8.GetString(File.ReadAllBytes(file)));

            RunResult result;
            string    time;
            int       retried = 0;

            // This is the retry loop for flaky HTTP connection. Note that local runs are never over HTTP, so they are never retried
            while (true)
            {
                byte[] compressed = CompressAndDump(test.GetInputBytes(), config.DebugDumpFile, config.LocalRun ? "Local" : "Remote");

                Stopwatch sw = new Stopwatch();
                sw.Start();
                result = config.LocalRun
                    ? ExecuteLocal(compressed, Utility.GetProcess(config), Utility.GetArenaHost(config), 60, 65, config.DebugDumpFile)
                    : ExecuteRemote(compressed, config.RunUrl, config.DebugDumpFile);
                sw.Stop();

                time = TimeFormatter.FormatTime(sw.Elapsed);
                if (!result.HttpFailure || retried++ >= config.Retries)
                {
                    break;
                }
                if (config.UseConsoleCodes)
                {
                    Utility.ClearLine(testName);
                }
                if (config.UseConsoleCodes)
                {
                    Console.ForegroundColor = ConsoleColor.Red;
                }
                if (config.UseConsoleCodes)
                {
                    Logger.Log($"{testName} - FAIL ({time}) Retrying({retried})...");
                }
                else
                {
                    Logger.LogLine($"{counter} {testName} - FAIL ({time}) Retrying({retried})...");
                }
                if (config.UseConsoleCodes)
                {
                    Console.ResetColor();
                }
            }

            if (config.UseConsoleCodes)
            {
                Utility.ClearLine(testName);
            }

            DisplayTestResultParams p = new DisplayTestResultParams
            {
                TestName                             = testName,
                ExpectedOutput                       = test.Output,
                Output                               = config.TrimWhitespacesFromResults ? Utility.TrimWhitespaces(result?.Output?[0]) : result?.Output?[0],
                                            Debug    = result?.Debug?[0],
                                            Time     = time,
                                            Counter  = counter,
                                            Warnings = result.Warnings
            };

            DisplayTestResult(p, config);
            return(p.Success);
        }
Beispiel #3
0
        private static void Execute(string[] args)
        {
            Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
            string configPath = "config.json";
            Config config     = File.Exists(configPath) ?
                                JsonConvert.DeserializeObject <Config>(File.ReadAllText(configPath)) :
                                new Config
            {
                RunUrl   = "https://tio.run/cgi-bin/run/api/no-cache/",
                CheckUrl = "https://tryitonline.net/languages.json",
                TestPath = Path.Combine(Utility.GetAncestor(AppContext.BaseDirectory, 3) ?? "", "HelloWorldTests"),
                TrimWhitespacesFromResults = true,
                DisplayDebugInfoOnError    = true,
                UseConsoleCodes            = true,
                BatchMode = true,
                LocalRoot = "/srv"
            };

            if (!CommandLine.ApplyCommandLineArguments(args, config))
            {
                return;
            }

            if (Directory.Exists(config.TestPath))
            {
                if (!string.IsNullOrWhiteSpace(config.CheckUrl))
                {
                    MissingTestsChecker.Check(config.CheckUrl, config.TestPath, config.UseConsoleCodes);
                }
                string[] files = Directory.GetFiles(config.TestPath, "*.json");
                Array.Sort(files);
                int counter = 0;
                int success = 0;
                if (!config.LocalRun || !config.BatchMode)
                {
                    Stopwatch sw = new Stopwatch();
                    sw.Start();
                    foreach (string file in files)
                    {
                        if (TestRunner.RunSingleTest(file, config, $"[{++counter}/{files.Length}]"))
                        {
                            success++;
                        }
                    }
                    sw.Stop();
                    string time = TimeFormatter.FormatTime(sw.Elapsed);
                    Logger.LogLine($"Elapsed: {time}");
                    Logger.LogLine($"Result: {success} succeeded, {files.Length - success} failed");
                }
                else
                {
                    TestRunner.RunTestsBatchLocal(files, config);
                }
            }
            else if (File.Exists(config.TestPath))
            {
                TestRunner.RunSingleTest(config.TestPath, config, "[1/1]");
            }
            else if (File.Exists(config.TestPath + ".json"))
            {
                TestRunner.RunSingleTest(config.TestPath + ".json", config, "1/1");
            }
            else
            {
                Logger.LogLine($"Tests '{config.TestPath}' not found");
            }
        }