public async Task GradeSubmissionAsync_ExecutionFailed_Error() { var question = GetCodeQuestion(); var submission = new CodeQuestionSubmission() { Contents = "A A B B C C" }; var simulatedResult = new MethodJobResult() { Status = CodeJobStatus.Error, DiagnosticOutput = "Job Failed" }; var grader = GetCodeQuestionGrader(question, submission, simulatedResult); var result = await grader.GradeSubmissionAsync(submission); var codeQuestionResult = (CodeQuestionResult)result.Result; var diagnosticErrors = codeQuestionResult.Errors .Cast<DiagnosticError>() .ToList(); Assert.Equal(0.0, result.Score); Assert.Equal(1, diagnosticErrors.Count); Assert.Equal("Job Failed", diagnosticErrors.Single().DiagnosticOutput); }
/// <summary> /// Returns a code runner service that responds with the /// given result, when called with the given job. /// </summary> public ICodeRunnerService GetCodeRunnerService( MethodJobResult expectedMethodJobResult, Expression<Func<MethodJob, bool>> expectedMethodJob = null) { if (expectedMethodJob == null) { expectedMethodJob = job => true; } var codeRunnerService = new Mock<ICodeRunnerService>(); codeRunnerService .Setup(crs => crs.ExecuteMethodJobAsync(It.Is(expectedMethodJob))) .ReturnsAsync(expectedMethodJobResult); return codeRunnerService.Object; }
public async Task GradeSubmissionAsync_TestFailsException_AccurateResults() { var question = GetCodeQuestion(); var submission = new CodeQuestionSubmission() { Contents = "A A B B C C" }; var simulatedResult = new MethodJobResult() { Status = CodeJobStatus.Completed, ClassCompilationResult = new CompilationResult() { Success = true }, TestsCompilationResult = new CompilationResult() { Success = true }, TestResults = Collections.CreateList ( new CodeTestResult() { Name = "test1", Completed = false, Exception = "exception" } ) }; var grader = GetCodeQuestionGrader(question, submission, simulatedResult); var result = await grader.GradeSubmissionAsync(submission); var codeQuestionResult = (CodeQuestionResult)result.Result; var testResults = codeQuestionResult.TestResults; Assert.Equal(1, testResults.Count); Assert.Equal("test1", testResults[0].Description); Assert.False(testResults[0].Succeeded); Assert.Equal("exception", testResults[0].ExceptionText); Assert.Equal("expectedReturnValue", testResults[0].ExpectedReturnValue); Assert.Null(testResults[0].ActualReturnValue); Assert.Equal("expectedOutput", testResults[0].ExpectedOutput); Assert.Null(testResults[0].ActualOutput); }
public async Task GradeSubmissionAsync_SomeTestsPass_NoCredit() { var question = GetCodeQuestion(numTests: 2, allowPartialCredit: false); var submission = new CodeQuestionSubmission() { Contents = "A A B B C C" }; var simulatedResult = new MethodJobResult() { Status = CodeJobStatus.Completed, ClassCompilationResult = new CompilationResult() { Success = true }, TestsCompilationResult = new CompilationResult() { Success = true }, TestResults = Collections.CreateList ( new CodeTestResult() { Name = "test1", Completed = true, ReturnValue = "expectedReturnValue", Output = "expectedOutput" }, new CodeTestResult() { Name = "test2", Completed = false, Exception = "exception" } ) }; var grader = GetCodeQuestionGrader(question, submission, simulatedResult); var result = await grader.GradeSubmissionAsync(submission); var codeQuestionResult = (CodeQuestionResult)result.Result; Assert.Equal(0.0, result.Score); Assert.Equal(0, codeQuestionResult.Errors.Count); }
public async Task GradeSubmissionAsync_TestCompilationFailed_Error() { var question = GetCodeQuestion(); var submission = new CodeQuestionSubmission() { Contents = "A A B B C C" }; var simulatedResult = new MethodJobResult() { Status = CodeJobStatus.Completed, ClassCompilationResult = new CompilationResult() { Success = true }, TestsCompilationResult = new CompilationResult() { Success = false, Errors = Collections.CreateList ( new CompileError() { Message = "ShortError1", FullError = "FullError1" }, new CompileError() { Message = "ShortError2", FullError = "FullError2" } ) } }; var grader = GetCodeQuestionGrader(question, submission, simulatedResult); var result = await grader.GradeSubmissionAsync(submission); var codeQuestionResult = (CodeQuestionResult)result.Result; var testsCompilationErrors = codeQuestionResult.Errors .Cast<TestCompilationError>() .ToList(); Assert.Equal(0.0, result.Score); Assert.Equal(2, testsCompilationErrors.Count); Assert.Null(testsCompilationErrors[0].LineNumber); Assert.Null(testsCompilationErrors[0].LineErrorText); Assert.True(testsCompilationErrors[0].FullErrorText.Contains("FullError1")); Assert.Null(testsCompilationErrors[1].LineNumber); Assert.Null(testsCompilationErrors[1].LineErrorText); Assert.True(testsCompilationErrors[1].FullErrorText.Contains("FullError2")); }
public async Task GradeSubmissionAsync_DefinitionMismatch_Error() { var question = GetCodeQuestion(); var submission = new CodeQuestionSubmission() { Contents = "A A B B C C" }; var simulatedResult = new MethodJobResult() { Status = CodeJobStatus.Completed, ClassCompilationResult = new CompilationResult() { Success = true } }; var expectedDefinitionErrors = Collections.CreateList<DefinitionError> ( new MethodNameError("expectedMethod", "actualMethod"), new MethodReturnTypeError("expectedMethod", "expectedReturnType", "actualReturnType") ); var grader = GetCodeQuestionGrader(question, submission, simulatedResult, expectedDefinitionErrors); var result = await grader.GradeSubmissionAsync(submission); var codeQuestionResult = (CodeQuestionResult)result.Result; var actualDefinitionErrors = codeQuestionResult.Errors .Cast<DefinitionError>() .ToList(); Assert.Equal(0.0, result.Score); Assert.Equal(expectedDefinitionErrors, actualDefinitionErrors); }
public async Task GradeSubmissionAsync_ExecutionTimedOut_Error() { var question = GetCodeQuestion(); var submission = new CodeQuestionSubmission() { Contents = "A A B B C C" }; var simulatedResult = new MethodJobResult() { Status = CodeJobStatus.Timeout }; var grader = GetCodeQuestionGrader(question, submission, simulatedResult); var result = await grader.GradeSubmissionAsync(submission); var codeQuestionResult = (CodeQuestionResult)result.Result; var timeoutErrors = codeQuestionResult.Errors .Cast<TimeoutError>() .ToList(); Assert.Equal(0.0, result.Score); Assert.Equal(1, timeoutErrors.Count); }