public async Task GradeSubmissionAsync_ExecutionFailed_Error()
{
var question = GetCodeQuestion();
var submission = new CodeQuestionSubmission() { Contents = "A A B B C C" };
var simulatedResult = new MethodJobResult()
{
Status = CodeJobStatus.Error,
DiagnosticOutput = "Job Failed"
};
var grader = GetCodeQuestionGrader(question, submission, simulatedResult);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
var diagnosticErrors = codeQuestionResult.Errors
.Cast<DiagnosticError>()
.ToList();
Assert.Equal(0.0, result.Score);
Assert.Equal(1, diagnosticErrors.Count);
Assert.Equal("Job Failed", diagnosticErrors.Single().DiagnosticOutput);
}
/// <summary>
/// Returns a code runner service that responds with the
/// given result, when called with the given job.
/// </summary>
public ICodeRunnerService GetCodeRunnerService(
MethodJobResult expectedMethodJobResult,
Expression<Func<MethodJob, bool>> expectedMethodJob = null)
{
if (expectedMethodJob == null)
{
expectedMethodJob = job => true;
}
var codeRunnerService = new Mock<ICodeRunnerService>();
codeRunnerService
.Setup(crs => crs.ExecuteMethodJobAsync(It.Is(expectedMethodJob)))
.ReturnsAsync(expectedMethodJobResult);
return codeRunnerService.Object;
}
public async Task GradeSubmissionAsync_TestFailsException_AccurateResults()
{
var question = GetCodeQuestion();
var submission = new CodeQuestionSubmission() { Contents = "A A B B C C" };
var simulatedResult = new MethodJobResult()
{
Status = CodeJobStatus.Completed,
ClassCompilationResult = new CompilationResult() { Success = true },
TestsCompilationResult = new CompilationResult() { Success = true },
TestResults = Collections.CreateList
(
new CodeTestResult()
{
Name = "test1",
Completed = false,
Exception = "exception"
}
)
};
var grader = GetCodeQuestionGrader(question, submission, simulatedResult);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
var testResults = codeQuestionResult.TestResults;
Assert.Equal(1, testResults.Count);
Assert.Equal("test1", testResults[0].Description);
Assert.False(testResults[0].Succeeded);
Assert.Equal("exception", testResults[0].ExceptionText);
Assert.Equal("expectedReturnValue", testResults[0].ExpectedReturnValue);
Assert.Null(testResults[0].ActualReturnValue);
Assert.Equal("expectedOutput", testResults[0].ExpectedOutput);
Assert.Null(testResults[0].ActualOutput);
}
public async Task GradeSubmissionAsync_SomeTestsPass_NoCredit()
{
var question = GetCodeQuestion(numTests: 2, allowPartialCredit: false);
var submission = new CodeQuestionSubmission() { Contents = "A A B B C C" };
var simulatedResult = new MethodJobResult()
{
Status = CodeJobStatus.Completed,
ClassCompilationResult = new CompilationResult() { Success = true },
TestsCompilationResult = new CompilationResult() { Success = true },
TestResults = Collections.CreateList
(
new CodeTestResult()
{
Name = "test1",
Completed = true,
ReturnValue = "expectedReturnValue",
Output = "expectedOutput"
},
new CodeTestResult()
{
Name = "test2",
Completed = false,
Exception = "exception"
}
)
};
var grader = GetCodeQuestionGrader(question, submission, simulatedResult);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
Assert.Equal(0.0, result.Score);
Assert.Equal(0, codeQuestionResult.Errors.Count);
}
public async Task GradeSubmissionAsync_TestCompilationFailed_Error()
{
var question = GetCodeQuestion();
var submission = new CodeQuestionSubmission() { Contents = "A A B B C C" };
var simulatedResult = new MethodJobResult()
{
Status = CodeJobStatus.Completed,
ClassCompilationResult = new CompilationResult() { Success = true },
TestsCompilationResult = new CompilationResult()
{
Success = false,
Errors = Collections.CreateList
(
new CompileError()
{
Message = "ShortError1",
FullError = "FullError1"
},
new CompileError()
{
Message = "ShortError2",
FullError = "FullError2"
}
)
}
};
var grader = GetCodeQuestionGrader(question, submission, simulatedResult);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
var testsCompilationErrors = codeQuestionResult.Errors
.Cast<TestCompilationError>()
.ToList();
Assert.Equal(0.0, result.Score);
Assert.Equal(2, testsCompilationErrors.Count);
Assert.Null(testsCompilationErrors[0].LineNumber);
Assert.Null(testsCompilationErrors[0].LineErrorText);
Assert.True(testsCompilationErrors[0].FullErrorText.Contains("FullError1"));
Assert.Null(testsCompilationErrors[1].LineNumber);
Assert.Null(testsCompilationErrors[1].LineErrorText);
Assert.True(testsCompilationErrors[1].FullErrorText.Contains("FullError2"));
}
public async Task GradeSubmissionAsync_DefinitionMismatch_Error()
{
var question = GetCodeQuestion();
var submission = new CodeQuestionSubmission() { Contents = "A A B B C C" };
var simulatedResult = new MethodJobResult()
{
Status = CodeJobStatus.Completed,
ClassCompilationResult = new CompilationResult() { Success = true }
};
var expectedDefinitionErrors = Collections.CreateList<DefinitionError>
(
new MethodNameError("expectedMethod", "actualMethod"),
new MethodReturnTypeError("expectedMethod", "expectedReturnType", "actualReturnType")
);
var grader = GetCodeQuestionGrader(question, submission, simulatedResult, expectedDefinitionErrors);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
var actualDefinitionErrors = codeQuestionResult.Errors
.Cast<DefinitionError>()
.ToList();
Assert.Equal(0.0, result.Score);
Assert.Equal(expectedDefinitionErrors, actualDefinitionErrors);
}
public async Task GradeSubmissionAsync_ExecutionTimedOut_Error()
{
var question = GetCodeQuestion();
var submission = new CodeQuestionSubmission() { Contents = "A A B B C C" };
var simulatedResult = new MethodJobResult()
{
Status = CodeJobStatus.Timeout
};
var grader = GetCodeQuestionGrader(question, submission, simulatedResult);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
var timeoutErrors = codeQuestionResult.Errors
.Cast<TimeoutError>()
.ToList();
Assert.Equal(0.0, result.Score);
Assert.Equal(1, timeoutErrors.Count);
}
