private void ProcessTests( ProcessExecutionResult processExecutionResult, ExecutionContext executionContext, ExecutionResult result) { var jsonResult = JsonExecutionResult.Parse(processExecutionResult.ReceivedOutput, true, true); var index = 0; result.TestResults = new List <TestResult>(); foreach (var test in executionContext.Tests) { var testResult = new TestResult { Id = test.Id, TimeUsed = (int)processExecutionResult.TimeWorked.TotalMilliseconds, MemoryUsed = (int)processExecutionResult.MemoryUsed, }; if (jsonResult.PassingTestsIndexes.Contains(index)) { testResult.ResultType = TestRunResultType.CorrectAnswer; } else { testResult.ResultType = TestRunResultType.WrongAnswer; testResult.CheckerDetails = new CheckerDetails { Comment = "Test failed." }; } result.TestResults.Add(testResult); index++; } }
private ICollection <TestResult> ExtractTestResultsFromReceivedOutput(string receivedOutput, int parentTestId) { JsonExecutionResult mochaResult = JsonExecutionResult.Parse(this.PreproccessReceivedExecutionOutput(receivedOutput)); if (mochaResult.TotalTests == 0) { return(new List <TestResult> { new TestResult { Id = parentTestId, IsTrialTest = false, ResultType = TestRunResultType.WrongAnswer, CheckerDetails = new CheckerDetails { UserOutputFragment = receivedOutput, }, }, }); } return(mochaResult.TestErrors .Select(test => this.ParseTestResult(test, parentTestId)) .ToList()); }
protected virtual int PrerunSubmission( ExecutionContext executionContext, IExecutor executor, string codeSavePath) { var arguments = new List <string>(); arguments.Add(this.MochaModulePath); arguments.Add(codeSavePath); arguments.AddRange(executionContext.AdditionalCompilerArguments.Split(' ')); var processExecutionResult = this.ExecuteNodeJsProcess(executionContext, executor, string.Empty, arguments); var mochaResult = JsonExecutionResult.Parse(processExecutionResult.ReceivedOutput); return(mochaResult.TotalTests); }
protected override List <TestResult> ProcessTests( IExecutionContext <TestsInputModel> executionContext, IExecutor executor, IChecker checker, string codeSavePath) { var testResults = new List <TestResult>(); var arguments = new List <string>(); arguments.Add(this.MochaModulePath); arguments.Add(codeSavePath); arguments.AddRange(executionContext.AdditionalCompilerArguments.Split(' ')); var processExecutionResult = executor.Execute( this.NodeJsExecutablePath, string.Empty, executionContext.TimeLimit, executionContext.MemoryLimit, arguments); var mochaResult = JsonExecutionResult.Parse(processExecutionResult.ReceivedOutput); var currentTest = 0; foreach (var test in executionContext.Input.Tests) { var message = "yes"; if (!string.IsNullOrEmpty(mochaResult.Error)) { message = mochaResult.Error; } else if (mochaResult.TestErrors[currentTest] != null) { message = $"Unexpected error: {mochaResult.TestErrors[currentTest]}"; } var testResult = this.ExecuteAndCheckTest( test, processExecutionResult, checker, message); currentTest++; testResults.Add(testResult); } return(testResults); }
protected override List <TestResult> ProcessTests(ExecutionContext executionContext, IExecutor executor, IChecker checker, string codeSavePath) { var testResults = new List <TestResult>(); var arguments = new List <string>(); arguments.Add(this.mochaModulePath); arguments.Add(codeSavePath); arguments.AddRange(executionContext.AdditionalCompilerArguments.Split(' ')); foreach (var test in executionContext.Tests) { var processExecutionResult = executor.Execute(this.NodeJsExecutablePath, test.Input, executionContext.TimeLimit, executionContext.MemoryLimit, arguments); var mochaResult = JsonExecutionResult.Parse(processExecutionResult.ReceivedOutput); var testResult = this.ExecuteAndCheckTest(test, processExecutionResult, checker, mochaResult.Passed ? "yes" : string.Format("Unexpected error: {0}", mochaResult.Error)); testResults.Add(testResult); } return(testResults); }
protected override List <TestResult> ProcessTests( ExecutionContext executionContext, IExecutor executor, IChecker checker, string codeSavePath) { var testResults = new List <TestResult>(); var arguments = new List <string>(); arguments.Add(this.MochaModulePath); arguments.Add(codeSavePath); arguments.AddRange(executionContext.AdditionalCompilerArguments.Split(' ')); foreach (var test in executionContext.Tests) { var processExecutionResult = this.ExecuteNodeJsProcess(executionContext, executor, test.Input, arguments); var mochaResult = JsonExecutionResult.Parse(processExecutionResult.ReceivedOutput); var message = "yes"; if (mochaResult.Error != string.Empty) { message = mochaResult.Error; } else if (mochaResult.TotalPassingTests != 1) { message = $"Unexpected error: {mochaResult.TestErrors[0]}"; } var testResult = this.ExecuteAndCheckTest( test, processExecutionResult, checker, message); testResults.Add(testResult); } return(testResults); }
protected override List <TestResult> ProcessTests( IExecutionContext <TestsInputModel> executionContext, IExecutor executor, IChecker checker, string codeSavePath) { var testResults = new List <TestResult>(); var arguments = new List <string>(); arguments.Add(this.MochaModulePath); arguments.Add(codeSavePath); arguments.AddRange(executionContext.AdditionalCompilerArguments.Split(' ')); var testCount = 0; var processExecutionResult = executor.Execute( this.NodeJsExecutablePath, string.Empty, executionContext.TimeLimit, executionContext.MemoryLimit, arguments); var mochaResult = JsonExecutionResult.Parse(processExecutionResult.ReceivedOutput); var numberOfUserTests = mochaResult.UsersTestCount; var correctSolutionTestPasses = mochaResult.InitialPassingTests; // an offset for tracking the current subset of tests (by convention we always have 2 Zero tests) var testOffset = numberOfUserTests * 2; foreach (var test in executionContext.Input.Tests) { var message = "Test Passed!"; TestResult testResult = null; if (testCount == 0) { var minTestCount = int.Parse( Regex.Match( test.Input, "<minTestCount>(\\d+)</minTestCount>").Groups[1].Value); if (numberOfUserTests < minTestCount) { message = $"Insufficient amount of tests, you have to have atleast {minTestCount} tests!"; } testResult = this.ExecuteAndCheckTest( test, processExecutionResult, checker, message); } else if (testCount == 1) { if (numberOfUserTests == 0) { message = "The submitted code was either incorrect or contained no tests!"; } else if (correctSolutionTestPasses != numberOfUserTests) { message = "Error: Some tests failed while running the correct solution!"; } testResult = this.ExecuteAndCheckTest( test, processExecutionResult, checker, message); } else { var numberOfPasses = mochaResult.TestErrors.Skip(testOffset).Take(numberOfUserTests).Count(x => x == null); if (numberOfPasses >= correctSolutionTestPasses) { message = "No unit test covering this functionality!"; } testResult = this.ExecuteAndCheckTest( test, processExecutionResult, checker, message); testOffset += numberOfUserTests; } testCount++; testResults.Add(testResult); } return(testResults); }
protected virtual List <TestResult> ProcessTests( ExecutionContext executionContext, IExecutor executor, IChecker checker, string codeSavePath, int numberOfUserTests) { var testResults = new List <TestResult>(); var arguments = new List <string>(); arguments.Add(this.MochaModulePath); arguments.Add(codeSavePath); arguments.AddRange(executionContext.AdditionalCompilerArguments.Split(' ')); var testCount = 0; var processExecutionResult = this.ExecuteNodeJsProcess(executionContext, executor, string.Empty, arguments); var mochaResult = JsonExecutionResult.Parse(processExecutionResult.ReceivedOutput); var correctSolutionTestPasses = mochaResult.TestsErrors.Take(numberOfUserTests).Count(x => x == null); var testOffset = numberOfUserTests; foreach (var test in executionContext.Tests) { TestResult testResult = null; if (testCount == 0) { var minTestCount = int.Parse( Regex.Match( test.Input, "<minTestCount>(\\d+)</minTestCount>").Groups[1].Value); var message = "Test Passed!"; if (numberOfUserTests == 0) { message = "The submitted code was either incorrect or contained no tests!"; } else if (numberOfUserTests < minTestCount) { message = $"Insufficient amount of tests, you have to have atleast {minTestCount} tests!"; } testResult = this.ExecuteAndCheckTest( test, processExecutionResult, checker, message); } else if (testCount == 1) { var message = "Test Passed!"; if (numberOfUserTests == 0) { message = "The submitted code was either incorrect or contained no tests!"; } else if (correctSolutionTestPasses != numberOfUserTests) { message = "Error: Some tests failed while running the correct solution!"; } testResult = this.ExecuteAndCheckTest( test, processExecutionResult, checker, message); } else { var numberOfPasses = mochaResult.TestsErrors.Skip(testOffset).Take(numberOfUserTests).Count(x => x == null); var message = "Test Passed!"; if (numberOfPasses >= correctSolutionTestPasses) { message = "No unit test covering this functionality!"; mochaResult.TestsErrors .Take(numberOfUserTests) .Where(x => x != null) .ForEach(x => message += x); } testResult = this.ExecuteAndCheckTest( test, processExecutionResult, checker, message); testOffset += numberOfUserTests; } testCount++; testResults.Add(testResult); } return(testResults); }