public async Task GradeSubmissionAsync_CodeConstraintsViolated_Error()
{
var question = GetCodeQuestion();
var submission = new CodeQuestionSubmission()
{
Contents = "A B C C C"
};
var grader = GetCodeQuestionGrader(question, submission, simulatedResult: null);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
var codeConstraintErrors = codeQuestionResult.Errors
.Cast<CodeConstraintError>()
.ToList();
Assert.Equal(0.0, result.Score);
Assert.Equal(3, codeConstraintErrors.Count);
Assert.Equal("A", codeConstraintErrors[0].Regex);
Assert.Equal(1, codeConstraintErrors[0].ActualFrequency);
Assert.Equal("A-AtLeastTwice", codeConstraintErrors[0].FullErrorText);
Assert.Equal("B", codeConstraintErrors[1].Regex);
Assert.Equal(1, codeConstraintErrors[1].ActualFrequency);
Assert.Equal("B-ExactlyTwice", codeConstraintErrors[1].FullErrorText);
Assert.Equal("C", codeConstraintErrors[2].Regex);
Assert.Equal(3, codeConstraintErrors[2].ActualFrequency);
Assert.Equal("C-AtMostTwice", codeConstraintErrors[2].FullErrorText);
}
public async Task GradeSubmissionAsync_NoSubmission_Error()
{
var question = GetCodeQuestion();
var submission = new CodeQuestionSubmission() { Contents = "" };
var grader = GetCodeQuestionGrader(question, submission, simulatedResult: null);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
var noSubmissionErrors = codeQuestionResult.Errors
.Cast<NoSubmissionError>()
.ToList();
Assert.Equal(0.0, result.Score);
Assert.Equal(1, noSubmissionErrors.Count);
}
public async Task GradeSubmissionAsync_ClassJobHasCorrectClassName()
{
var question = GetClassQuestion();
var classJobResult = GetClassJobResult(success: true);
var submission = new CodeQuestionSubmission() { Contents = "Submission" };
var codeRunnerService = GetCodeRunnerService
(
classJobResult,
job => job.ClassName == "ExpectedClass"
);
var grader = new ClassQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
Assert.Equal(1.0, result.Score);
}
public async Task GradeSubmissionAsync_MethodJobHasCorrectCode()
{
var question = GetMethodQuestion();
var methodJobResult = GetSuccessfulMethodJobResult();
var submission = new CodeQuestionSubmission() { Contents = "Submission %" };
var codeRunnerService = GetCodeRunnerService
(
methodJobResult,
job => job.MethodCode == "Submission %%"
);
var grader = new MethodQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
Assert.Equal(1.0, result.Score);
}
public async Task GradeSubmissionAsync_ClassJobHasImportedClasses()
{
var question = GetClassQuestion();
var classJobResult = GetClassJobResult(success: true);
var submission = new CodeQuestionSubmission() { Contents = "Submission" };
var codeRunnerService = GetCodeRunnerService
(
classJobResult,
job => job.ClassesToImport.Count == 1
&& job.ClassesToImport[0] == "package.classToImport"
);
var grader = new ClassQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
Assert.Equal(1.0, result.Score);
}
public async Task GradeSubmissionAsync_MethodJobHasTests()
{
var question = GetMethodQuestion();
var methodJobResult = GetSuccessfulMethodJobResult();
var submission = new CodeQuestionSubmission() { Contents = "Submission %" };
var codeRunnerService = GetCodeRunnerService
(
methodJobResult,
job => job.Tests.Count == 1
&& job.Tests[0].TestName == "test1"
&& job.Tests[0].ParamValues == "1, 2"
);
var grader = new MethodQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
Assert.Equal(1.0, result.Score);
}
public async Task GradeSubmissionAsync_ClassJobHasCorrectFileContents()
{
var question = GetClassQuestion();
var classJobResult = GetClassJobResult(success: true);
var submission = new CodeQuestionSubmission() { Contents = "Submission %" };
var codeRunnerService = GetCodeRunnerService
(
classJobResult,
job => job.FileContents ==
"class InternalClass\n" +
"{\n" +
"}\n" +
"\n"
+ "Submission %%"
);
var grader = new ClassQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
Assert.Equal(1.0, result.Score);
}
public async Task GradeSubmissionAsync_ExecutionFailed_Error()
{
var question = GetCodeQuestion();
var submission = new CodeQuestionSubmission() { Contents = "A A B B C C" };
var simulatedResult = new MethodJobResult()
{
Status = CodeJobStatus.Error,
DiagnosticOutput = "Job Failed"
};
var grader = GetCodeQuestionGrader(question, submission, simulatedResult);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
var diagnosticErrors = codeQuestionResult.Errors
.Cast<DiagnosticError>()
.ToList();
Assert.Equal(0.0, result.Score);
Assert.Equal(1, diagnosticErrors.Count);
Assert.Equal("Job Failed", diagnosticErrors.Single().DiagnosticOutput);
}
public async Task GradeSubmissionAsync_SomeTestsPass_NoCredit()
{
var question = GetCodeQuestion(numTests: 2, allowPartialCredit: false);
var submission = new CodeQuestionSubmission() { Contents = "A A B B C C" };
var simulatedResult = new MethodJobResult()
{
Status = CodeJobStatus.Completed,
ClassCompilationResult = new CompilationResult() { Success = true },
TestsCompilationResult = new CompilationResult() { Success = true },
TestResults = Collections.CreateList
(
new CodeTestResult()
{
Name = "test1",
Completed = true,
ReturnValue = "expectedReturnValue",
Output = "expectedOutput"
},
new CodeTestResult()
{
Name = "test2",
Completed = false,
Exception = "exception"
}
)
};
var grader = GetCodeQuestionGrader(question, submission, simulatedResult);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
Assert.Equal(0.0, result.Score);
Assert.Equal(0, codeQuestionResult.Errors.Count);
}
public async Task GradeSubmissionAsync_PermittedPublicFields_Success()
{
var question = GetClassQuestion(allowPublicFields: true);
var submission = new CodeQuestionSubmission() { Contents = "Submission" };
var classJobResult = GetClassJobResult(success: true);
classJobResult.ClassDefinition.Fields = new List<FieldDefinition>()
{
new FieldDefinition()
{
Name = "field1",
IsPublic = true,
Type = "String"
},
new FieldDefinition()
{
Name = "field2",
IsPublic = false,
Type = "String"
}
};
var codeRunnerService = GetCodeRunnerService(classJobResult);
var grader = new ClassQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
Assert.Equal(1.0, result.Score);
Assert.Equal(0, codeQuestionResult.Errors.Count);
}
public async Task GradeSubmissionAsync_ForbiddenPublicFields_Error()
{
var question = GetClassQuestion(allowPublicFields: false);
var submission = new CodeQuestionSubmission() { Contents = "Submission" };
var classJobResult = GetClassJobResult(success: false);
classJobResult.ClassDefinition.Fields = new List<FieldDefinition>()
{
new FieldDefinition()
{
Name = "field1",
IsPublic = true,
Type = "String"
},
new FieldDefinition()
{
Name = "field2",
IsPublic = false,
Type = "String"
}
};
var codeRunnerService = GetCodeRunnerService(classJobResult);
var grader = new ClassQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
var fieldVisibilityError = codeQuestionResult.Errors
.Cast<FieldVisibilityError>()
.Single();
Assert.Equal(0.0, result.Score);
Assert.Equal("ExpectedClass", fieldVisibilityError.ClassName);
}
public async Task GradeSubmissionAsync_WrongOverloadSignature_Error()
{
var question = GetClassQuestion(overloadedMethods: true);
var submission = new CodeQuestionSubmission() { Contents = "Submission" };
var classJobResult = GetClassJobResult(success: false, overloadedMethods: true);
classJobResult.ClassDefinition.Methods[0].ParameterTypes[0] = "String";
var codeRunnerService = GetCodeRunnerService(classJobResult);
var grader = new ClassQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
var methodOverloadDefinitionError = codeQuestionResult.Errors
.Cast<MethodOverloadDefinitionError>()
.Single();
Assert.Equal(0.0, result.Score);
Assert.Equal("ExpectedClass", methodOverloadDefinitionError.ClassName);
Assert.Equal("requiredOverloadedMethod", methodOverloadDefinitionError.ExpectedMethodName);
Assert.True(methodOverloadDefinitionError.ExpectedPublic);
Assert.False(methodOverloadDefinitionError.ExpectedStatic);
Assert.Equal("int", methodOverloadDefinitionError.ExpectedParamTypes);
Assert.Equal("boolean", methodOverloadDefinitionError.ExpectedReturnType);
}
public async Task GradeSubmissionAsync_TestCompilationFailed_Error()
{
var question = GetCodeQuestion();
var submission = new CodeQuestionSubmission() { Contents = "A A B B C C" };
var simulatedResult = new MethodJobResult()
{
Status = CodeJobStatus.Completed,
ClassCompilationResult = new CompilationResult() { Success = true },
TestsCompilationResult = new CompilationResult()
{
Success = false,
Errors = Collections.CreateList
(
new CompileError()
{
Message = "ShortError1",
FullError = "FullError1"
},
new CompileError()
{
Message = "ShortError2",
FullError = "FullError2"
}
)
}
};
var grader = GetCodeQuestionGrader(question, submission, simulatedResult);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
var testsCompilationErrors = codeQuestionResult.Errors
.Cast<TestCompilationError>()
.ToList();
Assert.Equal(0.0, result.Score);
Assert.Equal(2, testsCompilationErrors.Count);
Assert.Null(testsCompilationErrors[0].LineNumber);
Assert.Null(testsCompilationErrors[0].LineErrorText);
Assert.True(testsCompilationErrors[0].FullErrorText.Contains("FullError1"));
Assert.Null(testsCompilationErrors[1].LineNumber);
Assert.Null(testsCompilationErrors[1].LineErrorText);
Assert.True(testsCompilationErrors[1].FullErrorText.Contains("FullError2"));
}
public async Task GradeSubmissionAsync_TestFailsException_AccurateResults()
{
var question = GetCodeQuestion();
var submission = new CodeQuestionSubmission() { Contents = "A A B B C C" };
var simulatedResult = new MethodJobResult()
{
Status = CodeJobStatus.Completed,
ClassCompilationResult = new CompilationResult() { Success = true },
TestsCompilationResult = new CompilationResult() { Success = true },
TestResults = Collections.CreateList
(
new CodeTestResult()
{
Name = "test1",
Completed = false,
Exception = "exception"
}
)
};
var grader = GetCodeQuestionGrader(question, submission, simulatedResult);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
var testResults = codeQuestionResult.TestResults;
Assert.Equal(1, testResults.Count);
Assert.Equal("test1", testResults[0].Description);
Assert.False(testResults[0].Succeeded);
Assert.Equal("exception", testResults[0].ExceptionText);
Assert.Equal("expectedReturnValue", testResults[0].ExpectedReturnValue);
Assert.Null(testResults[0].ActualReturnValue);
Assert.Equal("expectedOutput", testResults[0].ExpectedOutput);
Assert.Null(testResults[0].ActualOutput);
}
public async Task GradeSubmissionAsync_WrongParameterTypes_Error()
{
var question = GetClassQuestion();
var submission = new CodeQuestionSubmission() { Contents = "Submission" };
var classJobResult = GetClassJobResult(success: false);
classJobResult.ClassDefinition.Methods[0].ParameterTypes[1] = "double";
var codeRunnerService = GetCodeRunnerService(classJobResult);
var grader = new ClassQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
var methodParameterTypesError = codeQuestionResult.Errors
.Cast<MethodParameterTypesError>()
.Single();
Assert.Equal(0.0, result.Score);
Assert.Equal("requiredMethod", methodParameterTypesError.MethodName);
Assert.Equal(2, methodParameterTypesError.ExpectedParamTypes.Count);
Assert.Equal("int", methodParameterTypesError.ExpectedParamTypes[0]);
Assert.Equal("int", methodParameterTypesError.ExpectedParamTypes[1]);
Assert.Equal("int", methodParameterTypesError.ActualParamTypes[0]);
Assert.Equal("double", methodParameterTypesError.ActualParamTypes[1]);
}
public async Task GradeSubmissionAsync_WrongParameterTypes_Error()
{
var question = GetMethodQuestion();
var submission = new CodeQuestionSubmission() { Contents = "Submission" };
var methodJobResult = GetFailedMethodJobResult
(
new MethodDefinition()
{
IsPublic = true,
IsStatic = true,
Name = "expectedMethod",
ParameterTypes = Collections.CreateList("int", "double"),
ReturnType = "int"
}
);
var codeRunnerService = GetCodeRunnerService(methodJobResult);
var grader = new MethodQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
var methodParameterTypesError = codeQuestionResult.Errors
.Cast<MethodParameterTypesError>()
.Single();
Assert.Equal(0.0, result.Score);
Assert.Equal("expectedMethod", methodParameterTypesError.MethodName);
Assert.Equal(2, methodParameterTypesError.ExpectedParamTypes.Count);
Assert.Equal("int", methodParameterTypesError.ExpectedParamTypes[0]);
Assert.Equal("int", methodParameterTypesError.ExpectedParamTypes[1]);
Assert.Equal("int", methodParameterTypesError.ActualParamTypes[0]);
Assert.Equal("double", methodParameterTypesError.ActualParamTypes[1]);
}
public async Task GradeSubmissionAsync_CorrectSubmission_ValidTestDescription()
{
var question = GetMethodQuestion();
var submission = new CodeQuestionSubmission() { Contents = "Submission" };
var methodJobResult = GetSuccessfulMethodJobResult();
var codeRunnerService = GetCodeRunnerService(methodJobResult);
var grader = new MethodQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
var testResult = codeQuestionResult.TestResults.Single();
Assert.Equal(1.0, result.Score);
Assert.Equal(0, codeQuestionResult.Errors.Count);
Assert.Equal("expectedMethod(1, 2)", testResult.Description);
}
public async Task GradeSubmissionAsync_NormalQuestion_StoresAndReturnsResult()
{
var database = new TestDatabaseBuilder()
.AddClassroom("Class1")
.AddSection("Class1", "Section1")
.AddStudent("User1", "Last", "First", "Class1", "Section1")
.AddQuestionCategory("Class1", "Category1")
.AddQuestion("Class1", "Category1", new MethodQuestion() { Name = "Question1" })
.Build();
var questionId = database.Context.Questions.First().Id;
var userId = database.Context.Users.First().Id;
database.Reload();
var loaderFactory = GetMockQuestionLoaderFactory();
var graderFactory = GetMockQuestionGraderFactory();
var timeProvider = GetMockTimeProvider(new DateTime(2016, 1, 1));
var questionSubmission = new CodeQuestionSubmission()
{
Contents = "SubmissionContents"
};
var serializer = new Mock<IJsonSerializer>();
serializer
.Setup(s => s.Serialize<QuestionSubmission>(questionSubmission))
.Returns("SerializedSubmissionContents");
var questionService = CreateQuestionService
(
database.Context,
questionLoaderFactory: loaderFactory.Object,
questionGraderFactory: graderFactory.Object,
jsonSerializer: serializer.Object,
timeProvider: timeProvider.Object
);
var result = await questionService.GradeSubmissionAsync
(
"Class1",
userId,
questionId,
questionSubmission
);
Assert.Equal(1.0, result.Score);
database.Reload();
var submission = database.Context
.UserQuestionSubmissions
.Where(uqs => uqs.UserQuestionData.UserId == userId)
.Include(uqs => uqs.UserQuestionData)
.Single(uqs => uqs.UserQuestionData.QuestionId == questionId);
Assert.Equal(1.0, submission.Score);
Assert.Equal
(
"SerializedSubmissionContents",
submission.UserQuestionData.LastQuestionSubmission
);
Assert.Equal
(
new DateTime(2016, 1, 1),
submission.DateSubmitted
);
}
public async Task GradeSubmissionAsync_ClassJobHasCorrectLineOffset()
{
var question = GetProgramQuestion();
var classJobResult = GetClassJobResult(success: true);
var submission = new CodeQuestionSubmission() { Contents = "Submission" };
var codeRunnerService = GetCodeRunnerService
(
classJobResult,
job => job.LineNumberOffset == 0
);
var grader = new ProgramQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
Assert.Equal(1.0, result.Score);
}
public async Task GradeSubmissionAsync_WrongMainMethodParameterTypes_Error()
{
var question = GetProgramQuestion();
var submission = new CodeQuestionSubmission() { Contents = "Submission" };
var classJobResult = GetClassJobResult(success: false);
classJobResult.ClassDefinition.Methods[0].ParameterTypes[0] = "double";
var codeRunnerService = GetCodeRunnerService(classJobResult);
var grader = new ProgramQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
var mainMethodMissingError = codeQuestionResult.Errors
.Cast<MainMethodMissingError>()
.Single();
Assert.Equal(0.0, result.Score);
Assert.Equal("ExpectedProgram", mainMethodMissingError.ClassName);
}
public async Task GradeSubmissionAsync_UnexpectedStaticMethod_Error()
{
var question = GetClassQuestion();
var submission = new CodeQuestionSubmission() { Contents = "Submission" };
var classJobResult = GetClassJobResult(success: false);
classJobResult.ClassDefinition.Methods[0].IsStatic = true;
var codeRunnerService = GetCodeRunnerService(classJobResult);
var grader = new ClassQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
var methodStaticError = codeQuestionResult.Errors
.Cast<MethodStaticError>()
.Single();
Assert.Equal(0.0, result.Score);
Assert.Equal("requiredMethod", methodStaticError.MethodName);
Assert.False(methodStaticError.ExpectedStatic);
}
public async Task GradeSubmissionAsync_DefinitionMismatch_Error()
{
var question = GetCodeQuestion();
var submission = new CodeQuestionSubmission() { Contents = "A A B B C C" };
var simulatedResult = new MethodJobResult()
{
Status = CodeJobStatus.Completed,
ClassCompilationResult = new CompilationResult() { Success = true }
};
var expectedDefinitionErrors = Collections.CreateList<DefinitionError>
(
new MethodNameError("expectedMethod", "actualMethod"),
new MethodReturnTypeError("expectedMethod", "expectedReturnType", "actualReturnType")
);
var grader = GetCodeQuestionGrader(question, submission, simulatedResult, expectedDefinitionErrors);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
var actualDefinitionErrors = codeQuestionResult.Errors
.Cast<DefinitionError>()
.ToList();
Assert.Equal(0.0, result.Score);
Assert.Equal(expectedDefinitionErrors, actualDefinitionErrors);
}
public async Task GradeSubmissionAsync_WrongReturnType_Error()
{
var question = GetClassQuestion();
var submission = new CodeQuestionSubmission() { Contents = "Submission" };
var classJobResult = GetClassJobResult(success: false);
classJobResult.ClassDefinition.Methods[0].ReturnType = "boolean";
var codeRunnerService = GetCodeRunnerService(classJobResult);
var grader = new ClassQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
var methodReturnTypeError = codeQuestionResult.Errors
.Cast<MethodReturnTypeError>()
.Single();
Assert.Equal(0.0, result.Score);
Assert.Equal("requiredMethod", methodReturnTypeError.MethodName);
Assert.Equal("String", methodReturnTypeError.ExpectedReturnType);
Assert.Equal("boolean", methodReturnTypeError.ActualReturnType);
}
public async Task GradeSubmissionAsync_MissingMethod_Error()
{
var question = GetMethodQuestion();
var submission = new CodeQuestionSubmission() { Contents = "Submission" };
var methodJobResult = GetFailedMethodJobResult(definition: null);
var codeRunnerService = GetCodeRunnerService(methodJobResult);
var grader = new MethodQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
var methodMissingError = codeQuestionResult.Errors
.Cast<MethodMissingError>()
.Single();
Assert.Equal(0.0, result.Score);
Assert.Equal("expectedMethod", methodMissingError.ExpectedMethodName);
Assert.True(methodMissingError.ExpectedStatic);
}
public async Task GradeSubmissionAsync_WrongOverloadCount_Error()
{
var question = GetClassQuestion(overloadedMethods: true);
var submission = new CodeQuestionSubmission() { Contents = "Submission" };
var classJobResult = GetClassJobResult(success: false, overloadedMethods: true);
classJobResult.ClassDefinition.Methods.RemoveAt(0);
var codeRunnerService = GetCodeRunnerService(classJobResult);
var grader = new ClassQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
var methodOverloadCountError = codeQuestionResult.Errors
.Cast<MethodOverloadCountError>()
.Single();
Assert.Equal(0.0, result.Score);
Assert.Equal("ExpectedClass", methodOverloadCountError.ClassName);
Assert.Equal("requiredOverloadedMethod", methodOverloadCountError.ExpectedMethodName);
Assert.Equal(2, methodOverloadCountError.ExpectedOverloadCount);
Assert.False(methodOverloadCountError.ExpectedStatic);
}
public async Task GradeSubmissionAsync_ClassJobHasTests()
{
var question = GetClassQuestion();
var classJobResult = GetClassJobResult(success: true);
var submission = new CodeQuestionSubmission() { Contents = "Submission %" };
var codeRunnerService = GetCodeRunnerService
(
classJobResult,
job => job.Tests.Count == 1
&& job.Tests[0].TestName == "test1"
&& job.Tests[0].MethodBody == "Method Body"
&& job.Tests[0].ReturnType == "String"
);
var grader = new ClassQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
Assert.Equal(1.0, result.Score);
}
public async Task GradeSubmissionAsync_CorrectSubmission_ValidTestDescription()
{
var question = GetClassQuestion();
var classJobResult = GetClassJobResult(success: true);
var submission = new CodeQuestionSubmission() { Contents = "Submission" };
var codeRunnerService = GetCodeRunnerService(classJobResult);
var grader = new ClassQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
var testResult = codeQuestionResult.TestResults.Single();
Assert.Equal(1.0, result.Score);
Assert.Equal(0, codeQuestionResult.Errors.Count);
Assert.Equal("Description", testResult.Description);
}
public async Task GradeSubmissionAsync_MissingExpectedClass_Error()
{
var question = GetClassQuestion();
var submission = new CodeQuestionSubmission() { Contents = "Submission" };
var classJobResult = GetClassJobResult(success: false);
classJobResult.TestsCompilationResult = null;
classJobResult.ClassDefinition = null;
classJobResult.ClassCompilationResult = new CompilationResult()
{
Success = false,
Errors = Collections.CreateList
(
new CompileError()
{
Message = "class ExpectedClass should be declared in ExpectedClass.java"
}
)
};
var codeRunnerService = GetCodeRunnerService(classJobResult);
var grader = new ClassQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
var missingRequiredClassError = codeQuestionResult.Errors
.Cast<MissingRequiredClassError>()
.Single();
Assert.Equal(0.0, result.Score);
Assert.Equal("ExpectedClass", missingRequiredClassError.RequiredClassName);
}
public async Task GradeSubmissionAsync_ExecutionTimedOut_Error()
{
var question = GetCodeQuestion();
var submission = new CodeQuestionSubmission() { Contents = "A A B B C C" };
var simulatedResult = new MethodJobResult()
{
Status = CodeJobStatus.Timeout
};
var grader = GetCodeQuestionGrader(question, submission, simulatedResult);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
var timeoutErrors = codeQuestionResult.Errors
.Cast<TimeoutError>()
.ToList();
Assert.Equal(0.0, result.Score);
Assert.Equal(1, timeoutErrors.Count);
}
/// <summary>
/// Returns a code question grader. For a given question, the grader
/// returns a given simulated result for a given submission.
/// </summary>
public CodeQuestionGrader<CodeQuestion> GetCodeQuestionGrader(
CodeQuestion question,
CodeQuestionSubmission submission,
CodeJobResult simulatedResult,
IList<DefinitionError> definitionErrors = null)
{
var grader = new Mock<CodeQuestionGrader<CodeQuestion>>
(
question,
null /*codeRunnerService*/
);
grader.CallBase = true;
grader.Protected()
.Setup<Task<CodeJobResult>>
(
"ExecuteJobAsync",
ItExpr.Is<CodeQuestionSubmission>(s => s == submission)
).Returns(Task.FromResult(simulatedResult));
grader.Protected()
.Setup<string>("GetTestDescription", ItExpr.IsAny<CodeQuestionTest>())
.Returns<CodeQuestionTest>(test => test.Name);
if (definitionErrors != null)
{
grader.Protected()
.Setup<IEnumerable<CodeQuestionError>>
(
"GetDefinitionErrors",
ItExpr.Is<CodeQuestionSubmission>(s => s == submission),
ItExpr.Is<CodeJobResult>(r => r == simulatedResult)
).Returns(definitionErrors);
}
return grader.Object;
}