public static void DisplayTestResult(DisplayTestResultParams p, Config config) { if (config.UseConsoleCodes) { Console.ForegroundColor = p.Color; } if (!config.BatchModeEffective) { Logger.LogLine(config.UseConsoleCodes ? $"{p.TestName} - {p.Result} ({p.Time})" : $"{p.Counter} {p.TestName} - {p.Result} ({p.Time})"); } if (config.BatchModeEffective && (!config.Quiet || (p.Success ? config.DisplayDebugInfoOnSuccess : config.DisplayDebugInfoOnError))) { Logger.LogLine($"{p.TestName} - {p.Result}"); } if (config.UseConsoleCodes) { Console.ResetColor(); } if (p.Success ? config.DisplayDebugInfoOnSuccess : config.DisplayDebugInfoOnError) { if (!p.Success) { Logger.LogLine($"Expected: {p.ExpectedOutput}"); Logger.LogLine($"Got: {p.Output}"); } if (!config.BatchModeEffective) { ShowWarnings(p.Warnings); } if (!string.IsNullOrWhiteSpace(p.Debug)) { Logger.LogLine($"Debug {p.Debug}"); } } }
public static void RunTestsBatchLocal(string[] files, Config config) { List <string> expectedOutput; byte[] payload = PrepareBatch(files, config.DebugDumpFile, out expectedOutput); int numberOfTests = files.Length; Logger.LogLine($"Running a batch of {numberOfTests} tests"); Stopwatch sw = new Stopwatch(); sw.Start(); RunResult result = ExecuteLocal(payload, Utility.GetProcess(config), Utility.GetArenaHost(config), numberOfTests * 50, numberOfTests * 50 + 5, config.DebugDumpFile); sw.Stop(); string time = TimeFormatter.FormatTime(sw.Elapsed); if (numberOfTests != result.Output?.Count || result.Output?.Count != result.Debug?.Count) { Logger.LogLine($"Error: Unexpected response. Tests:{numberOfTests}, Output:{result.Output?.Count}, Debug: {result.Debug?.Count}"); ShowWarnings(result.Warnings); return; } // In the batch mode backed returns a single set of warnings for all the tests. Display them here if (config.DisplayDebugInfoOnSuccess) { ShowWarnings(result.Warnings); } int successes = 0; for (int i = 0; i < numberOfTests; i++) { DisplayTestResultParams p = new DisplayTestResultParams { TestName = GetTestName(files[i]), ExpectedOutput = expectedOutput[i], Output = config.TrimWhitespacesFromResults ? Utility.TrimWhitespaces(result.Output[i]) : result.Output[i], Debug = result.Debug[i] }; DisplayTestResult(p, config); if (p.Success) { successes++; } } Logger.LogLine($"Elapsed: {time}"); Logger.LogLine($"Result: {successes} succeeded, {numberOfTests - successes} failed"); }
public static bool RunSingleTest(string file, Config config, string counter) { string testName = GetTestName(file); if (config.UseConsoleCodes) { Logger.Log($"{counter} {testName}..."); } TestDescription test = JsonConvert.DeserializeObject <TestDescription>(Encoding.UTF8.GetString(File.ReadAllBytes(file))); RunResult result; string time; int retried = 0; // This is the retry loop for flaky HTTP connection. Note that local runs are never over HTTP, so they are never retried while (true) { byte[] compressed = CompressAndDump(test.GetInputBytes(), config.DebugDumpFile, config.LocalRun ? "Local" : "Remote"); Stopwatch sw = new Stopwatch(); sw.Start(); result = config.LocalRun ? ExecuteLocal(compressed, Utility.GetProcess(config), Utility.GetArenaHost(config), 60, 65, config.DebugDumpFile) : ExecuteRemote(compressed, config.RunUrl, config.DebugDumpFile); sw.Stop(); time = TimeFormatter.FormatTime(sw.Elapsed); if (!result.HttpFailure || retried++ >= config.Retries) { break; } if (config.UseConsoleCodes) { Utility.ClearLine(testName); } if (config.UseConsoleCodes) { Console.ForegroundColor = ConsoleColor.Red; } if (config.UseConsoleCodes) { Logger.Log($"{testName} - FAIL ({time}) Retrying({retried})..."); } else { Logger.LogLine($"{counter} {testName} - FAIL ({time}) Retrying({retried})..."); } if (config.UseConsoleCodes) { Console.ResetColor(); } } if (config.UseConsoleCodes) { Utility.ClearLine(testName); } DisplayTestResultParams p = new DisplayTestResultParams { TestName = testName, ExpectedOutput = test.Output, Output = config.TrimWhitespacesFromResults ? Utility.TrimWhitespaces(result?.Output?[0]) : result?.Output?[0], Debug = result?.Debug?[0], Time = time, Counter = counter, Warnings = result.Warnings }; DisplayTestResult(p, config); return(p.Success); }