protected TestResult ExecuteAndCheckTest(TestContext test, ProcessExecutionResult processExecutionResult, IChecker checker, string receivedOutput)
{
var testResult = new TestResult
{
Id = test.Id,
TimeUsed = (int)processExecutionResult.TimeWorked.TotalMilliseconds,
MemoryUsed = (int)processExecutionResult.MemoryUsed,
};
if (processExecutionResult.Type == ProcessExecutionResultType.RunTimeError)
{
testResult.ResultType = TestRunResultType.RunTimeError;
testResult.ExecutionComment = processExecutionResult.ErrorOutput.MaxLength(2048); // Trimming long error texts
}
else if (processExecutionResult.Type == ProcessExecutionResultType.TimeLimit)
{
testResult.ResultType = TestRunResultType.TimeLimit;
}
else if (processExecutionResult.Type == ProcessExecutionResultType.MemoryLimit)
{
testResult.ResultType = TestRunResultType.MemoryLimit;
}
else if (processExecutionResult.Type == ProcessExecutionResultType.Success)
{
var checkerResult = checker.Check(test.Input, receivedOutput, test.Output, test.IsTrialTest);
if (checkerResult.IsCorrect)
{
testResult.ResultType = TestRunResultType.CorrectAnswer;
}
else
{
testResult.ResultType = TestRunResultType.WrongAnswer;
}
// TODO: Do something with checkerResult.ResultType
testResult.CheckerComment = checkerResult.CheckerDetails;
}
else
{
throw new ArgumentOutOfRangeException("processExecutionResult", "Invalid ProcessExecutionResultType value.");
}
return testResult;
}
private void ProcessTests(ProcessExecutionResult processExecutionResult, ExecutionContext executionContext, ExecutionResult result)
{
var jsonResult = JsonExecutionResult.Parse(processExecutionResult.ReceivedOutput, true, true);
var index = 0;
result.TestResults = new List<TestResult>();
foreach (var test in executionContext.Tests)
{
var testResult = new TestResult
{
Id = test.Id,
TimeUsed = (int)processExecutionResult.TimeWorked.TotalMilliseconds,
MemoryUsed = (int)processExecutionResult.MemoryUsed,
};
if (jsonResult.PassingIndexes.Contains(index))
{
testResult.ResultType = TestRunResultType.CorrectAnswer;
}
else
{
testResult.ResultType = TestRunResultType.WrongAnswer;
testResult.CheckerDetails = new CheckerDetails { Comment = "Test failed." };
}
result.TestResults.Add(testResult);
index++;
}
}
protected virtual List <TestResult> ProcessTests(
ExecutionContext executionContext,
IExecutor executor,
IChecker checker,
string codeSavePath,
int numberOfUserTests)
{
var testResults = new List <TestResult>();
var arguments = new List <string>();
arguments.Add(this.MochaModulePath);
arguments.Add(codeSavePath);
arguments.AddRange(executionContext.AdditionalCompilerArguments.Split(' '));
var testCount = 0;
var processExecutionResult = this.ExecuteNodeJsProcess(executionContext, executor, string.Empty, arguments);
var mochaResult = JsonExecutionResult.Parse(processExecutionResult.ReceivedOutput);
var correctSolutionTestPasses = mochaResult.TestsErrors.Take(numberOfUserTests).Count(x => x == null);
var testOffset = numberOfUserTests;
foreach (var test in executionContext.Tests)
{
TestResult testResult = null;
if (testCount == 0)
{
var minTestCount = int.Parse(
Regex.Match(
test.Input,
"<minTestCount>(\\d+)</minTestCount>").Groups[1].Value);
var message = "Test Passed!";
if (numberOfUserTests == 0)
{
message = "The submitted code was either incorrect or contained no tests!";
}
else if (numberOfUserTests < minTestCount)
{
message = $"Insufficient amount of tests, you have to have atleast {minTestCount} tests!";
}
testResult = this.ExecuteAndCheckTest(
test,
processExecutionResult,
checker,
message);
}
else if (testCount == 1)
{
var message = "Test Passed!";
if (numberOfUserTests == 0)
{
message = "The submitted code was either incorrect or contained no tests!";
}
else if (correctSolutionTestPasses != numberOfUserTests)
{
message = "Error: Some tests failed while running the correct solution!";
}
testResult = this.ExecuteAndCheckTest(
test,
processExecutionResult,
checker,
message);
}
else
{
var numberOfPasses = mochaResult.TestsErrors.Skip(testOffset).Take(numberOfUserTests).Count(x => x == null);
var message = "Test Passed!";
if (numberOfPasses >= correctSolutionTestPasses)
{
message = "No unit test covering this functionality!";
mochaResult.TestsErrors
.Take(numberOfUserTests)
.Where(x => x != null)
.ForEach(x => message += x);
}
testResult = this.ExecuteAndCheckTest(
test,
processExecutionResult,
checker,
message);
testOffset += numberOfUserTests;
}
testCount++;
testResults.Add(testResult);
}
return(testResults);
}
