public static void DisplayTestResult(DisplayTestResultParams p, Config config)
{
if (config.UseConsoleCodes)
{
Console.ForegroundColor = p.Color;
}
if (!config.BatchModeEffective)
{
Logger.LogLine(config.UseConsoleCodes ? $"{p.TestName} - {p.Result} ({p.Time})" : $"{p.Counter} {p.TestName} - {p.Result} ({p.Time})");
}
if (config.BatchModeEffective && (!config.Quiet || (p.Success ? config.DisplayDebugInfoOnSuccess : config.DisplayDebugInfoOnError)))
{
Logger.LogLine($"{p.TestName} - {p.Result}");
}
if (config.UseConsoleCodes)
{
Console.ResetColor();
}
if (p.Success ? config.DisplayDebugInfoOnSuccess : config.DisplayDebugInfoOnError)
{
if (!p.Success)
{
Logger.LogLine($"Expected: {p.ExpectedOutput}");
Logger.LogLine($"Got: {p.Output}");
}
if (!config.BatchModeEffective)
{
ShowWarnings(p.Warnings);
}
if (!string.IsNullOrWhiteSpace(p.Debug))
{
Logger.LogLine($"Debug {p.Debug}");
}
}
}
public static void RunTestsBatchLocal(string[] files, Config config)
{
List <string> expectedOutput;
byte[] payload = PrepareBatch(files, config.DebugDumpFile, out expectedOutput);
int numberOfTests = files.Length;
Logger.LogLine($"Running a batch of {numberOfTests} tests");
Stopwatch sw = new Stopwatch();
sw.Start();
RunResult result = ExecuteLocal(payload, Utility.GetProcess(config), Utility.GetArenaHost(config), numberOfTests * 50, numberOfTests * 50 + 5, config.DebugDumpFile);
sw.Stop();
string time = TimeFormatter.FormatTime(sw.Elapsed);
if (numberOfTests != result.Output?.Count || result.Output?.Count != result.Debug?.Count)
{
Logger.LogLine($"Error: Unexpected response. Tests:{numberOfTests}, Output:{result.Output?.Count}, Debug: {result.Debug?.Count}");
ShowWarnings(result.Warnings);
return;
}
// In the batch mode backed returns a single set of warnings for all the tests. Display them here
if (config.DisplayDebugInfoOnSuccess)
{
ShowWarnings(result.Warnings);
}
int successes = 0;
for (int i = 0; i < numberOfTests; i++)
{
DisplayTestResultParams p = new DisplayTestResultParams
{
TestName = GetTestName(files[i]),
ExpectedOutput = expectedOutput[i],
Output = config.TrimWhitespacesFromResults ? Utility.TrimWhitespaces(result.Output[i]) : result.Output[i],
Debug = result.Debug[i]
};
DisplayTestResult(p, config);
if (p.Success)
{
successes++;
}
}
Logger.LogLine($"Elapsed: {time}");
Logger.LogLine($"Result: {successes} succeeded, {numberOfTests - successes} failed");
}
public static bool RunSingleTest(string file, Config config, string counter)
{
string testName = GetTestName(file);
if (config.UseConsoleCodes)
{
Logger.Log($"{counter} {testName}...");
}
TestDescription test = JsonConvert.DeserializeObject <TestDescription>(Encoding.UTF8.GetString(File.ReadAllBytes(file)));
RunResult result;
string time;
int retried = 0;
// This is the retry loop for flaky HTTP connection. Note that local runs are never over HTTP, so they are never retried
while (true)
{
byte[] compressed = CompressAndDump(test.GetInputBytes(), config.DebugDumpFile, config.LocalRun ? "Local" : "Remote");
Stopwatch sw = new Stopwatch();
sw.Start();
result = config.LocalRun
? ExecuteLocal(compressed, Utility.GetProcess(config), Utility.GetArenaHost(config), 60, 65, config.DebugDumpFile)
: ExecuteRemote(compressed, config.RunUrl, config.DebugDumpFile);
sw.Stop();
time = TimeFormatter.FormatTime(sw.Elapsed);
if (!result.HttpFailure || retried++ >= config.Retries)
{
break;
}
if (config.UseConsoleCodes)
{
Utility.ClearLine(testName);
}
if (config.UseConsoleCodes)
{
Console.ForegroundColor = ConsoleColor.Red;
}
if (config.UseConsoleCodes)
{
Logger.Log($"{testName} - FAIL ({time}) Retrying({retried})...");
}
else
{
Logger.LogLine($"{counter} {testName} - FAIL ({time}) Retrying({retried})...");
}
if (config.UseConsoleCodes)
{
Console.ResetColor();
}
}
if (config.UseConsoleCodes)
{
Utility.ClearLine(testName);
}
DisplayTestResultParams p = new DisplayTestResultParams
{
TestName = testName,
ExpectedOutput = test.Output,
Output = config.TrimWhitespacesFromResults ? Utility.TrimWhitespaces(result?.Output?[0]) : result?.Output?[0],
Debug = result?.Debug?[0],
Time = time,
Counter = counter,
Warnings = result.Warnings
};
DisplayTestResult(p, config);
return(p.Success);
}
