protected TestResult ExecuteAndCheckTest(TestContext test, ProcessExecutionResult processExecutionResult, IChecker checker, string receivedOutput) { var testResult = new TestResult { Id = test.Id, TimeUsed = (int)processExecutionResult.TimeWorked.TotalMilliseconds, MemoryUsed = (int)processExecutionResult.MemoryUsed, }; if (processExecutionResult.Type == ProcessExecutionResultType.RunTimeError) { testResult.ResultType = TestRunResultType.RunTimeError; testResult.ExecutionComment = processExecutionResult.ErrorOutput.MaxLength(2048); // Trimming long error texts } else if (processExecutionResult.Type == ProcessExecutionResultType.TimeLimit) { testResult.ResultType = TestRunResultType.TimeLimit; } else if (processExecutionResult.Type == ProcessExecutionResultType.MemoryLimit) { testResult.ResultType = TestRunResultType.MemoryLimit; } else if (processExecutionResult.Type == ProcessExecutionResultType.Success) { var checkerResult = checker.Check(test.Input, receivedOutput, test.Output, test.IsTrialTest); if (checkerResult.IsCorrect) { testResult.ResultType = TestRunResultType.CorrectAnswer; } else { testResult.ResultType = TestRunResultType.WrongAnswer; } // TODO: Do something with checkerResult.ResultType testResult.CheckerComment = checkerResult.CheckerDetails; } else { throw new ArgumentOutOfRangeException("processExecutionResult", "Invalid ProcessExecutionResultType value."); } return testResult; }
private void ProcessTests(ProcessExecutionResult processExecutionResult, ExecutionContext executionContext, ExecutionResult result) { var jsonResult = JsonExecutionResult.Parse(processExecutionResult.ReceivedOutput, true, true); var index = 0; result.TestResults = new List<TestResult>(); foreach (var test in executionContext.Tests) { var testResult = new TestResult { Id = test.Id, TimeUsed = (int)processExecutionResult.TimeWorked.TotalMilliseconds, MemoryUsed = (int)processExecutionResult.MemoryUsed, }; if (jsonResult.PassingIndexes.Contains(index)) { testResult.ResultType = TestRunResultType.CorrectAnswer; } else { testResult.ResultType = TestRunResultType.WrongAnswer; testResult.CheckerDetails = new CheckerDetails { Comment = "Test failed." }; } result.TestResults.Add(testResult); index++; } }
protected virtual List <TestResult> ProcessTests( ExecutionContext executionContext, IExecutor executor, IChecker checker, string codeSavePath, int numberOfUserTests) { var testResults = new List <TestResult>(); var arguments = new List <string>(); arguments.Add(this.MochaModulePath); arguments.Add(codeSavePath); arguments.AddRange(executionContext.AdditionalCompilerArguments.Split(' ')); var testCount = 0; var processExecutionResult = this.ExecuteNodeJsProcess(executionContext, executor, string.Empty, arguments); var mochaResult = JsonExecutionResult.Parse(processExecutionResult.ReceivedOutput); var correctSolutionTestPasses = mochaResult.TestsErrors.Take(numberOfUserTests).Count(x => x == null); var testOffset = numberOfUserTests; foreach (var test in executionContext.Tests) { TestResult testResult = null; if (testCount == 0) { var minTestCount = int.Parse( Regex.Match( test.Input, "<minTestCount>(\\d+)</minTestCount>").Groups[1].Value); var message = "Test Passed!"; if (numberOfUserTests == 0) { message = "The submitted code was either incorrect or contained no tests!"; } else if (numberOfUserTests < minTestCount) { message = $"Insufficient amount of tests, you have to have atleast {minTestCount} tests!"; } testResult = this.ExecuteAndCheckTest( test, processExecutionResult, checker, message); } else if (testCount == 1) { var message = "Test Passed!"; if (numberOfUserTests == 0) { message = "The submitted code was either incorrect or contained no tests!"; } else if (correctSolutionTestPasses != numberOfUserTests) { message = "Error: Some tests failed while running the correct solution!"; } testResult = this.ExecuteAndCheckTest( test, processExecutionResult, checker, message); } else { var numberOfPasses = mochaResult.TestsErrors.Skip(testOffset).Take(numberOfUserTests).Count(x => x == null); var message = "Test Passed!"; if (numberOfPasses >= correctSolutionTestPasses) { message = "No unit test covering this functionality!"; mochaResult.TestsErrors .Take(numberOfUserTests) .Where(x => x != null) .ForEach(x => message += x); } testResult = this.ExecuteAndCheckTest( test, processExecutionResult, checker, message); testOffset += numberOfUserTests; } testCount++; testResults.Add(testResult); } return(testResults); }