private void ProcessTests(
ProcessExecutionResult processExecutionResult,
ExecutionContext executionContext,
ExecutionResult result)
{
var jsonResult = JsonExecutionResult.Parse(processExecutionResult.ReceivedOutput, true, true);
var index = 0;
result.TestResults = new List <TestResult>();
foreach (var test in executionContext.Tests)
{
var testResult = new TestResult
{
Id = test.Id,
TimeUsed = (int)processExecutionResult.TimeWorked.TotalMilliseconds,
MemoryUsed = (int)processExecutionResult.MemoryUsed,
};
if (jsonResult.PassingTestsIndexes.Contains(index))
{
testResult.ResultType = TestRunResultType.CorrectAnswer;
}
else
{
testResult.ResultType = TestRunResultType.WrongAnswer;
testResult.CheckerDetails = new CheckerDetails {
Comment = "Test failed."
};
}
result.TestResults.Add(testResult);
index++;
}
}
private ICollection <TestResult> ExtractTestResultsFromReceivedOutput(string receivedOutput, int parentTestId)
{
JsonExecutionResult mochaResult = JsonExecutionResult.Parse(this.PreproccessReceivedExecutionOutput(receivedOutput));
if (mochaResult.TotalTests == 0)
{
return(new List <TestResult>
{
new TestResult
{
Id = parentTestId,
IsTrialTest = false,
ResultType = TestRunResultType.WrongAnswer,
CheckerDetails = new CheckerDetails
{
UserOutputFragment = receivedOutput,
},
},
});
}
return(mochaResult.TestErrors
.Select(test => this.ParseTestResult(test, parentTestId))
.ToList());
}
protected virtual int PrerunSubmission(
ExecutionContext executionContext,
IExecutor executor,
string codeSavePath)
{
var arguments = new List <string>();
arguments.Add(this.MochaModulePath);
arguments.Add(codeSavePath);
arguments.AddRange(executionContext.AdditionalCompilerArguments.Split(' '));
var processExecutionResult = this.ExecuteNodeJsProcess(executionContext, executor, string.Empty, arguments);
var mochaResult = JsonExecutionResult.Parse(processExecutionResult.ReceivedOutput);
return(mochaResult.TotalTests);
}
protected override List <TestResult> ProcessTests(
IExecutionContext <TestsInputModel> executionContext,
IExecutor executor,
IChecker checker,
string codeSavePath)
{
var testResults = new List <TestResult>();
var arguments = new List <string>();
arguments.Add(this.MochaModulePath);
arguments.Add(codeSavePath);
arguments.AddRange(executionContext.AdditionalCompilerArguments.Split(' '));
var processExecutionResult = executor.Execute(
this.NodeJsExecutablePath,
string.Empty,
executionContext.TimeLimit,
executionContext.MemoryLimit,
arguments);
var mochaResult = JsonExecutionResult.Parse(processExecutionResult.ReceivedOutput);
var currentTest = 0;
foreach (var test in executionContext.Input.Tests)
{
var message = "yes";
if (!string.IsNullOrEmpty(mochaResult.Error))
{
message = mochaResult.Error;
}
else if (mochaResult.TestErrors[currentTest] != null)
{
message = $"Unexpected error: {mochaResult.TestErrors[currentTest]}";
}
var testResult = this.ExecuteAndCheckTest(
test,
processExecutionResult,
checker,
message);
currentTest++;
testResults.Add(testResult);
}
return(testResults);
}
protected override List <TestResult> ProcessTests(ExecutionContext executionContext, IExecutor executor, IChecker checker, string codeSavePath)
{
var testResults = new List <TestResult>();
var arguments = new List <string>();
arguments.Add(this.mochaModulePath);
arguments.Add(codeSavePath);
arguments.AddRange(executionContext.AdditionalCompilerArguments.Split(' '));
foreach (var test in executionContext.Tests)
{
var processExecutionResult = executor.Execute(this.NodeJsExecutablePath, test.Input, executionContext.TimeLimit, executionContext.MemoryLimit, arguments);
var mochaResult = JsonExecutionResult.Parse(processExecutionResult.ReceivedOutput);
var testResult = this.ExecuteAndCheckTest(test, processExecutionResult, checker, mochaResult.Passed ? "yes" : string.Format("Unexpected error: {0}", mochaResult.Error));
testResults.Add(testResult);
}
return(testResults);
}
protected override List <TestResult> ProcessTests(
ExecutionContext executionContext,
IExecutor executor,
IChecker checker,
string codeSavePath)
{
var testResults = new List <TestResult>();
var arguments = new List <string>();
arguments.Add(this.MochaModulePath);
arguments.Add(codeSavePath);
arguments.AddRange(executionContext.AdditionalCompilerArguments.Split(' '));
foreach (var test in executionContext.Tests)
{
var processExecutionResult = this.ExecuteNodeJsProcess(executionContext, executor, test.Input, arguments);
var mochaResult = JsonExecutionResult.Parse(processExecutionResult.ReceivedOutput);
var message = "yes";
if (mochaResult.Error != string.Empty)
{
message = mochaResult.Error;
}
else if (mochaResult.TotalPassingTests != 1)
{
message = $"Unexpected error: {mochaResult.TestErrors[0]}";
}
var testResult = this.ExecuteAndCheckTest(
test,
processExecutionResult,
checker,
message);
testResults.Add(testResult);
}
return(testResults);
}
protected override List <TestResult> ProcessTests(
IExecutionContext <TestsInputModel> executionContext,
IExecutor executor,
IChecker checker,
string codeSavePath)
{
var testResults = new List <TestResult>();
var arguments = new List <string>();
arguments.Add(this.MochaModulePath);
arguments.Add(codeSavePath);
arguments.AddRange(executionContext.AdditionalCompilerArguments.Split(' '));
var testCount = 0;
var processExecutionResult = executor.Execute(
this.NodeJsExecutablePath,
string.Empty,
executionContext.TimeLimit,
executionContext.MemoryLimit,
arguments);
var mochaResult = JsonExecutionResult.Parse(processExecutionResult.ReceivedOutput);
var numberOfUserTests = mochaResult.UsersTestCount;
var correctSolutionTestPasses = mochaResult.InitialPassingTests;
// an offset for tracking the current subset of tests (by convention we always have 2 Zero tests)
var testOffset = numberOfUserTests * 2;
foreach (var test in executionContext.Input.Tests)
{
var message = "Test Passed!";
TestResult testResult = null;
if (testCount == 0)
{
var minTestCount = int.Parse(
Regex.Match(
test.Input,
"<minTestCount>(\\d+)</minTestCount>").Groups[1].Value);
if (numberOfUserTests < minTestCount)
{
message = $"Insufficient amount of tests, you have to have atleast {minTestCount} tests!";
}
testResult = this.ExecuteAndCheckTest(
test,
processExecutionResult,
checker,
message);
}
else if (testCount == 1)
{
if (numberOfUserTests == 0)
{
message = "The submitted code was either incorrect or contained no tests!";
}
else if (correctSolutionTestPasses != numberOfUserTests)
{
message = "Error: Some tests failed while running the correct solution!";
}
testResult = this.ExecuteAndCheckTest(
test,
processExecutionResult,
checker,
message);
}
else
{
var numberOfPasses = mochaResult.TestErrors.Skip(testOffset).Take(numberOfUserTests).Count(x => x == null);
if (numberOfPasses >= correctSolutionTestPasses)
{
message = "No unit test covering this functionality!";
}
testResult = this.ExecuteAndCheckTest(
test,
processExecutionResult,
checker,
message);
testOffset += numberOfUserTests;
}
testCount++;
testResults.Add(testResult);
}
return(testResults);
}
protected virtual List <TestResult> ProcessTests(
ExecutionContext executionContext,
IExecutor executor,
IChecker checker,
string codeSavePath,
int numberOfUserTests)
{
var testResults = new List <TestResult>();
var arguments = new List <string>();
arguments.Add(this.MochaModulePath);
arguments.Add(codeSavePath);
arguments.AddRange(executionContext.AdditionalCompilerArguments.Split(' '));
var testCount = 0;
var processExecutionResult = this.ExecuteNodeJsProcess(executionContext, executor, string.Empty, arguments);
var mochaResult = JsonExecutionResult.Parse(processExecutionResult.ReceivedOutput);
var correctSolutionTestPasses = mochaResult.TestsErrors.Take(numberOfUserTests).Count(x => x == null);
var testOffset = numberOfUserTests;
foreach (var test in executionContext.Tests)
{
TestResult testResult = null;
if (testCount == 0)
{
var minTestCount = int.Parse(
Regex.Match(
test.Input,
"<minTestCount>(\\d+)</minTestCount>").Groups[1].Value);
var message = "Test Passed!";
if (numberOfUserTests == 0)
{
message = "The submitted code was either incorrect or contained no tests!";
}
else if (numberOfUserTests < minTestCount)
{
message = $"Insufficient amount of tests, you have to have atleast {minTestCount} tests!";
}
testResult = this.ExecuteAndCheckTest(
test,
processExecutionResult,
checker,
message);
}
else if (testCount == 1)
{
var message = "Test Passed!";
if (numberOfUserTests == 0)
{
message = "The submitted code was either incorrect or contained no tests!";
}
else if (correctSolutionTestPasses != numberOfUserTests)
{
message = "Error: Some tests failed while running the correct solution!";
}
testResult = this.ExecuteAndCheckTest(
test,
processExecutionResult,
checker,
message);
}
else
{
var numberOfPasses = mochaResult.TestsErrors.Skip(testOffset).Take(numberOfUserTests).Count(x => x == null);
var message = "Test Passed!";
if (numberOfPasses >= correctSolutionTestPasses)
{
message = "No unit test covering this functionality!";
mochaResult.TestsErrors
.Take(numberOfUserTests)
.Where(x => x != null)
.ForEach(x => message += x);
}
testResult = this.ExecuteAndCheckTest(
test,
processExecutionResult,
checker,
message);
testOffset += numberOfUserTests;
}
testCount++;
testResults.Add(testResult);
}
return(testResults);
}
