private SolutionEvaluationTestResult CreateTestResult(SolutionTest test, ProcessRunResult runResult, OutputVerificationResult verificationResult)
{
return(new SolutionEvaluationTestResult()
{
ExpectedOutput = test.ExpectedOutput,
RunResult = runResult,
Score = verificationResult.ScoreMultiplier * test.MaxScore,
Type = TranslateVerificationResultType(verificationResult.Type),
TestGroup = test.TestGroup
});
}
private SolutionEvaluationTestResult RunTest(int testIndex)
{
SolutionTest test = mTests[testIndex];
ProcessRunResult runResult = ConsoleApplicationRunner.Instance.Run(
application: mApplication,
stdIn: test.Input,
maxRuntime: test.TimeLimit,
processArguments: test.ProcessArguments,
allowCrashReports: true
);
if (TryTranslateUngracefulExit(runResult.ExitType, out SolutionEvaluationTestResultType testResultType))
{
return(new SolutionEvaluationTestResult {
ExpectedOutput = test.ExpectedOutput,
RunResult = runResult,
Score = 0,
Type = testResultType,
TestGroup = test.TestGroup
});
}
if (runResult.ExitType != ProcessExitType.Graceful)
{
throw new NotImplementedException();
}
OutputVerificationResult verificationResult = test.OutputVerifier.Verify(new OutputVerificationInfo(
exitCode: runResult.ExitCode,
standardInput: test.Input,
standardError: runResult.StandardError,
standardOutput: runResult.Output,
expectedOutput: test.ExpectedOutput
));
return(CreateTestResult(test, runResult, verificationResult));
}