public static void RunTestsBatchLocal(string[] files, Config config)
{
List <string> expectedOutput;
byte[] payload = PrepareBatch(files, config.DebugDumpFile, out expectedOutput);
int numberOfTests = files.Length;
Logger.LogLine($"Running a batch of {numberOfTests} tests");
Stopwatch sw = new Stopwatch();
sw.Start();
RunResult result = ExecuteLocal(payload, Utility.GetProcess(config), Utility.GetArenaHost(config), numberOfTests * 50, numberOfTests * 50 + 5, config.DebugDumpFile);
sw.Stop();
string time = TimeFormatter.FormatTime(sw.Elapsed);
if (numberOfTests != result.Output?.Count || result.Output?.Count != result.Debug?.Count)
{
Logger.LogLine($"Error: Unexpected response. Tests:{numberOfTests}, Output:{result.Output?.Count}, Debug: {result.Debug?.Count}");
ShowWarnings(result.Warnings);
return;
}
// In the batch mode backed returns a single set of warnings for all the tests. Display them here
if (config.DisplayDebugInfoOnSuccess)
{
ShowWarnings(result.Warnings);
}
int successes = 0;
for (int i = 0; i < numberOfTests; i++)
{
DisplayTestResultParams p = new DisplayTestResultParams
{
TestName = GetTestName(files[i]),
ExpectedOutput = expectedOutput[i],
Output = config.TrimWhitespacesFromResults ? Utility.TrimWhitespaces(result.Output[i]) : result.Output[i],
Debug = result.Debug[i]
};
DisplayTestResult(p, config);
if (p.Success)
{
successes++;
}
}
Logger.LogLine($"Elapsed: {time}");
Logger.LogLine($"Result: {successes} succeeded, {numberOfTests - successes} failed");
}
public static bool RunSingleTest(string file, Config config, string counter)
{
string testName = GetTestName(file);
if (config.UseConsoleCodes)
{
Logger.Log($"{counter} {testName}...");
}
TestDescription test = JsonConvert.DeserializeObject <TestDescription>(Encoding.UTF8.GetString(File.ReadAllBytes(file)));
RunResult result;
string time;
int retried = 0;
// This is the retry loop for flaky HTTP connection. Note that local runs are never over HTTP, so they are never retried
while (true)
{
byte[] compressed = CompressAndDump(test.GetInputBytes(), config.DebugDumpFile, config.LocalRun ? "Local" : "Remote");
Stopwatch sw = new Stopwatch();
sw.Start();
result = config.LocalRun
? ExecuteLocal(compressed, Utility.GetProcess(config), Utility.GetArenaHost(config), 60, 65, config.DebugDumpFile)
: ExecuteRemote(compressed, config.RunUrl, config.DebugDumpFile);
sw.Stop();
time = TimeFormatter.FormatTime(sw.Elapsed);
if (!result.HttpFailure || retried++ >= config.Retries)
{
break;
}
if (config.UseConsoleCodes)
{
Utility.ClearLine(testName);
}
if (config.UseConsoleCodes)
{
Console.ForegroundColor = ConsoleColor.Red;
}
if (config.UseConsoleCodes)
{
Logger.Log($"{testName} - FAIL ({time}) Retrying({retried})...");
}
else
{
Logger.LogLine($"{counter} {testName} - FAIL ({time}) Retrying({retried})...");
}
if (config.UseConsoleCodes)
{
Console.ResetColor();
}
}
if (config.UseConsoleCodes)
{
Utility.ClearLine(testName);
}
DisplayTestResultParams p = new DisplayTestResultParams
{
TestName = testName,
ExpectedOutput = test.Output,
Output = config.TrimWhitespacesFromResults ? Utility.TrimWhitespaces(result?.Output?[0]) : result?.Output?[0],
Debug = result?.Debug?[0],
Time = time,
Counter = counter,
Warnings = result.Warnings
};
DisplayTestResult(p, config);
return(p.Success);
}
private static void Execute(string[] args)
{
Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
string configPath = "config.json";
Config config = File.Exists(configPath) ?
JsonConvert.DeserializeObject <Config>(File.ReadAllText(configPath)) :
new Config
{
RunUrl = "https://tio.run/cgi-bin/run/api/no-cache/",
CheckUrl = "https://tryitonline.net/languages.json",
TestPath = Path.Combine(Utility.GetAncestor(AppContext.BaseDirectory, 3) ?? "", "HelloWorldTests"),
TrimWhitespacesFromResults = true,
DisplayDebugInfoOnError = true,
UseConsoleCodes = true,
BatchMode = true,
LocalRoot = "/srv"
};
if (!CommandLine.ApplyCommandLineArguments(args, config))
{
return;
}
if (Directory.Exists(config.TestPath))
{
if (!string.IsNullOrWhiteSpace(config.CheckUrl))
{
MissingTestsChecker.Check(config.CheckUrl, config.TestPath, config.UseConsoleCodes);
}
string[] files = Directory.GetFiles(config.TestPath, "*.json");
Array.Sort(files);
int counter = 0;
int success = 0;
if (!config.LocalRun || !config.BatchMode)
{
Stopwatch sw = new Stopwatch();
sw.Start();
foreach (string file in files)
{
if (TestRunner.RunSingleTest(file, config, $"[{++counter}/{files.Length}]"))
{
success++;
}
}
sw.Stop();
string time = TimeFormatter.FormatTime(sw.Elapsed);
Logger.LogLine($"Elapsed: {time}");
Logger.LogLine($"Result: {success} succeeded, {files.Length - success} failed");
}
else
{
TestRunner.RunTestsBatchLocal(files, config);
}
}
else if (File.Exists(config.TestPath))
{
TestRunner.RunSingleTest(config.TestPath, config, "[1/1]");
}
else if (File.Exists(config.TestPath + ".json"))
{
TestRunner.RunSingleTest(config.TestPath + ".json", config, "1/1");
}
else
{
Logger.LogLine($"Tests '{config.TestPath}' not found");
}
}
