public async Task GradeSubmissionAsync_ExecutionFailed_Error()
{
var question = GetCodeQuestion();
var submission = new CodeQuestionSubmission()
{
Contents = "A A B B C C"
};
var simulatedResult = new MethodJobResult()
{
Status = CodeJobStatus.Error,
DiagnosticOutput = "Job Failed"
};
var grader = GetCodeQuestionGrader(question, submission, simulatedResult);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
var diagnosticErrors = codeQuestionResult.Errors
.Cast <DiagnosticError>()
.ToList();
Assert.Equal(0.0, result.Score);
Assert.Single(diagnosticErrors);
Assert.Equal("Job Failed", diagnosticErrors.Single().DiagnosticOutput);
}
public async Task GradeSubmissionAsync_DefinitionMismatch_Error()
{
var question = GetCodeQuestion();
var submission = new CodeQuestionSubmission()
{
Contents = "A A B B C C"
};
var simulatedResult = new MethodJobResult()
{
Status = CodeJobStatus.Completed,
ClassCompilationResult = new CompilationResult()
{
Success = true
}
};
var expectedDefinitionErrors = Collections.CreateList <DefinitionError>
(
new MethodNameError("expectedMethod", "actualMethod"),
new MethodReturnTypeError("expectedMethod", "expectedReturnType", "actualReturnType")
);
var grader = GetCodeQuestionGrader(question, submission, simulatedResult, expectedDefinitionErrors);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
var actualDefinitionErrors = codeQuestionResult.Errors
.Cast <DefinitionError>()
.ToList();
Assert.Equal(0.0, result.Score);
Assert.Equal(expectedDefinitionErrors, actualDefinitionErrors);
}
public async Task GradeSubmissionAsync_TestCompilationFailed_Error()
{
var question = GetCodeQuestion();
var submission = new CodeQuestionSubmission()
{
Contents = "A A B B C C"
};
var simulatedResult = new MethodJobResult()
{
Status = CodeJobStatus.Completed,
ClassCompilationResult = new CompilationResult()
{
Success = true
},
TestsCompilationResult = new CompilationResult()
{
Success = false,
Errors = Collections.CreateList
(
new CompileError()
{
Message = "ShortError1",
FullError = "FullError1"
},
new CompileError()
{
Message = "ShortError2",
FullError = "FullError2"
}
)
}
};
var grader = GetCodeQuestionGrader(question, submission, simulatedResult);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
var testsCompilationErrors = codeQuestionResult.Errors
.Cast <TestCompilationError>()
.ToList();
Assert.Equal(0.0, result.Score);
Assert.Equal(2, testsCompilationErrors.Count);
Assert.Null(testsCompilationErrors[0].LineNumber);
Assert.Null(testsCompilationErrors[0].LineErrorText);
Assert.Contains("FullError1", testsCompilationErrors[0].FullErrorText);
Assert.Null(testsCompilationErrors[1].LineNumber);
Assert.Null(testsCompilationErrors[1].LineErrorText);
Assert.Contains("FullError2", testsCompilationErrors[1].FullErrorText);
}
/// <summary>
/// Returns a code runner service that responds with the
/// given result, when called with the given job.
/// </summary>
public ICodeRunnerService GetCodeRunnerService(
MethodJobResult expectedMethodJobResult,
Expression <Func <MethodJob, bool> > expectedMethodJob = null)
{
if (expectedMethodJob == null)
{
expectedMethodJob = job => true;
}
var codeRunnerService = new Mock <ICodeRunnerService>();
codeRunnerService
.Setup(crs => crs.ExecuteMethodJobAsync(It.Is(expectedMethodJob)))
.ReturnsAsync(expectedMethodJobResult);
return(codeRunnerService.Object);
}
public async Task GradeSubmissionAsync_TestFailsIncorrectReturnValue_AccurateResults()
{
var question = GetCodeQuestion();
var submission = new CodeQuestionSubmission()
{
Contents = "A A B B C C"
};
var simulatedResult = new MethodJobResult()
{
Status = CodeJobStatus.Completed,
ClassCompilationResult = new CompilationResult()
{
Success = true
},
TestsCompilationResult = new CompilationResult()
{
Success = true
},
TestResults = Collections.CreateList
(
new CodeTestResult()
{
Name = "test1",
Completed = true,
ReturnValue = "incorrectReturnValue",
Output = "expectedOutput"
}
)
};
var grader = GetCodeQuestionGrader(question, submission, simulatedResult);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
var testResults = codeQuestionResult.TestResults;
Assert.Single(testResults);
Assert.Equal("test1", testResults[0].Description);
Assert.False(testResults[0].Succeeded);
Assert.Null(testResults[0].ExceptionText);
Assert.Equal("expectedReturnValue", testResults[0].ExpectedReturnValue);
Assert.Equal("incorrectReturnValue", testResults[0].ActualReturnValue);
Assert.Equal("expectedOutput", testResults[0].ExpectedOutput);
Assert.Equal("expectedOutput", testResults[0].ActualOutput);
}
public async Task GradeSubmissionAsync_TestsAllPass_FullCredit()
{
var question = GetCodeQuestion(numTests: 2);
var submission = new CodeQuestionSubmission()
{
Contents = "A A B B C C"
};
var simulatedResult = new MethodJobResult()
{
Status = CodeJobStatus.Completed,
ClassCompilationResult = new CompilationResult()
{
Success = true
},
TestsCompilationResult = new CompilationResult()
{
Success = true
},
TestResults = Collections.CreateList
(
new CodeTestResult()
{
Name = "test1",
Completed = true,
ReturnValue = "expectedReturnValue",
Output = "expectedOutput"
},
new CodeTestResult()
{
Name = "test2",
Completed = true,
ReturnValue = "expectedReturnValue",
Output = "expectedOutput"
}
)
};
var grader = GetCodeQuestionGrader(question, submission, simulatedResult);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
Assert.Equal(1.0, result.Score);
Assert.Empty(codeQuestionResult.Errors);
}
public async Task GradeSubmissionAsync_ExecutionTimedOut_Error()
{
var question = GetCodeQuestion();
var submission = new CodeQuestionSubmission()
{
Contents = "A A B B C C"
};
var simulatedResult = new MethodJobResult()
{
Status = CodeJobStatus.Timeout
};
var grader = GetCodeQuestionGrader(question, submission, simulatedResult);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
var timeoutErrors = codeQuestionResult.Errors
.Cast <TimeoutError>()
.ToList();
Assert.Equal(0.0, result.Score);
Assert.Single(timeoutErrors);
}
