public SolutionTest(
StringOrFile inputFile,
StringOrFile expectedOutputFile,
IOutputVerifier outputVerifier,
string processArguments,
TimeSpan timeLimit,
double maxScore,
string testGroup
)
{
if (inputFile == null)
{
throw new ArgumentNullException(nameof(inputFile));
}
if (expectedOutputFile == null)
{
throw new ArgumentNullException(nameof(expectedOutputFile));
}
if (outputVerifier == null)
{
throw new ArgumentNullException(nameof(outputVerifier));
}
if (testGroup == null)
{
throw new ArgumentNullException(nameof(testGroup));
}
ProcessArguments = processArguments;
Input = inputFile;
ExpectedOutput = expectedOutputFile;
OutputVerifier = outputVerifier;
TimeLimit = timeLimit;
MaxScore = maxScore;
TestGroup = testGroup;
}
public void TestSum()
{
string checker = Path.GetFullPath("checker_sum.exe".GetLocalFilePath());
IExecutableOutputVerifier verifier = new ExecutableOutputVerifierMutable {
ConsoleApplication = new FileSystemConsoleApplication(checker, CheckerCore.CrashReporting.CrashReportFinder.Instance),
Bindings = new IVerifierResultBinder[] {
new StdOutContainsBinder("1", new OutputVerificationResult(
OutputVerificationResultType.CorrectAnswer,
1
)),
new StdOutContainsBinder("0", new OutputVerificationResult(
OutputVerificationResultType.WrongAnswer,
0
))
},
Arguments = new VerifierArgumentType[] {
VerifierArgumentType.Solution, //number A
VerifierArgumentType.Stdout //number B
},
Stdin = VerifierArgumentType.ExitCode //number C
};
OutputVerificationInfo info = new OutputVerificationInfo(
5,//C
null,
null,
StringOrFile.FromText("2"), //B
StringOrFile.FromText("3") //A
);
OutputVerificationResult result = verifier.Verify(info);
Assert.AreEqual(result.Type, OutputVerificationResultType.CorrectAnswer);
;
}
public void TestContinuity()
{
ProcessRunResult result = ConsoleApplicationRunner.Instance.Run(
new FileSystemConsoleApplication(DummyPath, CheckerCore.CrashReporting.CrashReportFinder.Instance),
TimeSpan.FromHours(1),
StringOrFile.FromText("continuity 100000 end")
);
Assert.AreEqual(result.ExitType, ProcessExitType.Graceful);
StringOrFile output = result.Output;
string[] lines = output.StringValue
.Split(new char[] { '\n' }, StringSplitOptions.RemoveEmptyEntries)
.Select(x => x.Trim())
.Where(x => !string.IsNullOrWhiteSpace(x))
.ToArray();
int prev = int.Parse(lines.First().Split(' ')[0]);
for (int i = 1; i < lines.Length; i++)
{
int current = int.Parse(lines[i].Split(' ')[0]);
Assert.IsTrue(current == prev + 1);
prev = current;
}
;
}
public ProcessRunArguments(IConsoleApplication application, StringOrFile stdIn, TimeSpan maxRuntime, string processArguments, bool allowCrashReports)
{
Application = application;
StandardInput = stdIn;
ProcessArguments = processArguments ?? string.Empty;
AllowCrashReports = allowCrashReports;
MaxRuntime = maxRuntime;
}
public void TestCmpfiles(bool okay)
{
string checker = Path.GetFullPath("checker_cmpfiles.exe".GetLocalFilePath());
Random r = new Random(1337);
string file1 = Path.GetTempFileName();
string file2 = Path.GetTempFileName();
File.WriteAllText(file1, "The quick brown fox jumps over the lazy dog");
if (okay)
{
File.WriteAllText(file2, "The quick brown fox jumps over the lazy dog");
}
else
{
File.WriteAllText(file2, "The quick brown fox jumps over the lazy dog.");
}
IOutputVerifier verifier = new ExecutableOutputVerifierMutable {
ConsoleApplication = new FileSystemConsoleApplication(checker, CheckerCore.CrashReporting.CrashReportFinder.Instance),
Arguments = new VerifierArgumentType[] {
VerifierArgumentType.FileStdout,
VerifierArgumentType.FileSolution
},
Bindings = new IVerifierResultBinder[] {
new StdOutContainsBinder("OK", new OutputVerificationResult(
OutputVerificationResultType.CorrectAnswer,
1
)),
new StdOutContainsBinder("NO", new OutputVerificationResult(
OutputVerificationResultType.WrongAnswer,
0
))
},
Stdin = VerifierArgumentType.None
};
OutputVerificationInfo info = new OutputVerificationInfo(
0,
null,
null,
StringOrFile.FromFile(file1),
StringOrFile.FromFile(file2)
);
OutputVerificationResult result = verifier.Verify(info);
if (okay)
{
Assert.AreEqual(result.ScoreMultiplier, 1);
Assert.AreEqual(result.Type, OutputVerificationResultType.CorrectAnswer);
}
else
{
Assert.AreEqual(result.ScoreMultiplier, 0);
Assert.AreEqual(result.Type, OutputVerificationResultType.WrongAnswer);
}
}
public ProcessRunResult(ProcessExitType exitType, ICrashReport crashReport, StringOrFile stdOut, StringOrFile stdErr, double memoryUsed, int exitCode, TimeSpan executionTime)
{
ExitType = exitType;
CrashReport = crashReport;
Output = stdOut;
StandardError = stdErr;
MemoryUsed = memoryUsed;
ExitCode = exitCode;
ExecutionTime = executionTime;
}
public void TestCrash()
{
ProcessRunResult result = ConsoleApplicationRunner.Instance.Run(
new FileSystemConsoleApplication(DummyPath, CheckerCore.CrashReporting.CrashReportFinder.Instance),
TimeSpan.FromSeconds(1),
StringOrFile.FromText("sum 1 2 crash")
);
Assert.AreEqual(result.ExitType, ProcessExitType.Crashed);
Assert.IsNotNull(result.CrashReport);
}
public void TestExit()
{
ProcessRunResult result = ConsoleApplicationRunner.Instance.Run(
new FileSystemConsoleApplication(DummyPath, CheckerCore.CrashReporting.CrashReportFinder.Instance),
TimeSpan.FromHours(1),
StringOrFile.FromText("exit 3")
);
Assert.AreEqual(result.ExitCode, 3);
Assert.AreEqual(result.ExitType, ProcessExitType.Graceful);
}
public void TestSlow()
{
ProcessRunResult result = ConsoleApplicationRunner.Instance.Run(
new FileSystemConsoleApplication(DummyPath, CheckerCore.CrashReporting.CrashReportFinder.Instance),
TimeSpan.FromSeconds(1),
StringOrFile.FromText("wait 200 sum 1 5 wait 10000 exit 0")
);
Assert.AreEqual(result.ExitType, ProcessExitType.Timeout);
Assert.IsTrue(result.Output.ToString().Contains("6"));
}
public OutputVerificationInfo(
int exitCode,
StringOrFile standardError,
StringOrFile standardInput,
StringOrFile standardOutput,
StringOrFile expectedOutput)
{
ExitCode = exitCode;
StandardError = standardError;
StandardInput = standardInput;
StandardOutput = standardOutput;
ExpectedOutput = expectedOutput;
}
public bool TryCreateTestViewModels(out List <SolutionTest> tests)
{
//TODO: Fix this shit because it's ugly Java-ish practice
if (!CanCreateTests)
{
tests = null;
return(false);
}
string[] inputs = TestInputFiles.PathsArray.ToArray();
string[] expectedOutputs = TestSolutionFiles.PathsArray.ToArray();
if (SortFilenamesAlphabetically)
{
Array.Sort <string>(inputs, (x, y) => x.CompareTo(y));
Array.Sort <string>(expectedOutputs, (x, y) => x.CompareTo(y));
}
;
double totalExplicitScore =
Tests
.Where(x => !double.IsNaN(x.MaxScore))
.Sum(x => x.MaxScore);
int autoScoreCount =
Tests
.Where(x => double.IsNaN(x.MaxScore))
.Count();
double autoScore = (100.00 - totalExplicitScore) / autoScoreCount;
tests = new List <SolutionTest>();
for (int i = 0; i < inputs.Length; i++)
{
tests.Add(new SolutionTest(
inputFile: StringOrFile.FromFile(inputs[i]),
expectedOutputFile: StringOrFile.FromFile(expectedOutputs[i]),
outputVerifier: Checker.CreateModel(),
processArguments: String.Empty,
timeLimit: TimeSpan.FromSeconds(TimeLimitSeconds),
maxScore:
!double.IsNaN(Tests[i].MaxScore) ?
Tests[i].MaxScore :
autoScore,
testGroup: Tests[i].TestGroup ?? string.Empty
));
}
return(true);
}
public void TestOutput()
{
ProcessRunResult result = ConsoleApplicationRunner.Instance.Run(
new FileSystemConsoleApplication(DummyPath, CheckerCore.CrashReporting.CrashReportFinder.Instance),
TimeSpan.FromSeconds(0.1),
StringOrFile.FromText("sum 1 2 sum 3 5 end")
);
Assert.AreEqual(result.ExitType, ProcessExitType.Graceful);
StringOrFile output = result.Output;
Assert.IsTrue(output.StringValue.IndexOf('3') != -1 && output.StringValue.IndexOf('8') > output.StringValue.IndexOf('3'));
;
}
public ProcessRunResult Run(
IConsoleApplication application,
TimeSpan maxRuntime,
StringOrFile stdIn,
string processArguments = null,
bool allowCrashReports = true
)
{
ProcessRunArguments args = new ProcessRunArguments(
application,
stdIn,
maxRuntime,
processArguments,
allowCrashReports
);
return(Run(args));
}
private ProcessRunResult Run(ProcessRunArguments args)
{
IConsoleProcess process = args.Application.StartProcess(args.ProcessArguments);
bool timelyExit = TryWaitForTimelyExitAsync(process, args).GetAwaiter().GetResult();
bool crashed = process.TryFindCrashReport(out ICrashReport report);
string stdOut = process.GetStandardOutput();
string stdErr = process.GetStandardError();
return(new ProcessRunResult(
DeduceExitType(crashed, timelyExit),
report,
StringOrFile.FromText(stdOut),
StringOrFile.FromText(stdErr),
100,
process.ExitCode,
process.ExecutionTime
));
}
private StringOrFile GetVerifierArgument(OutputVerificationInfo info, VerifierArgumentType thing)
{
switch (thing)
{
case VerifierArgumentType.ExitCode:
return(StringOrFile.FromText(info.ExitCode.ToString()));
case VerifierArgumentType.FileSolution:
return(StringOrFile.FromText(info.ExpectedOutput.FilePath));
case VerifierArgumentType.FileStderr:
return(StringOrFile.FromText(info.StandardError.FilePath));
case VerifierArgumentType.FileStdin:
return(StringOrFile.FromText(info.StandardInput.FilePath));
case VerifierArgumentType.FileStdout:
return(StringOrFile.FromText(info.StandardOutput.FilePath));
case VerifierArgumentType.Solution:
return(info.ExpectedOutput);
case VerifierArgumentType.Stderr:
return(info.StandardError);
case VerifierArgumentType.Stdin:
return(info.StandardInput);
case VerifierArgumentType.Stdout:
return(info.StandardOutput);
default:
case VerifierArgumentType.None:
return(StringOrFile.FromText(string.Empty));
}
}
public void TestIsOdd()
{
string checker = Path.GetFullPath("checker_isodd.exe".GetLocalFilePath());
Random r = new Random(1337);
for (int i = 0; i < 12; i++)
{
int number = r.Next();
IOutputVerifier verifier = new ExecutableOutputVerifierMutable {
ConsoleApplication = new FileSystemConsoleApplication(checker, CheckerCore.CrashReporting.CrashReportFinder.Instance),
Stdin = VerifierArgumentType.Stdout,//number
Bindings = new IVerifierResultBinder[] {
new ExitCodeBinder(1, new OutputVerificationResult(
OutputVerificationResultType.CorrectAnswer,
1
)),
new ExitCodeBinder(0, new OutputVerificationResult(
OutputVerificationResultType.CorrectAnswer,
0
))
}
};
OutputVerificationInfo info = new OutputVerificationInfo(
0,
null,
null,
StringOrFile.FromText(number.ToString()),
null
);
OutputVerificationResult result = verifier.Verify(info);
Assert.AreEqual(result.Type, OutputVerificationResultType.CorrectAnswer);
Assert.AreEqual(result.ScoreMultiplier, number % 2);
}
}
