public async Task GradeSubmissionAsync_WrongOverloadSignature_Error()
{
var question = GetClassQuestion(overloadedMethods: true);
var submission = new CodeQuestionSubmission()
{
Contents = "Submission"
};
var classJobResult = GetClassJobResult(success: false, overloadedMethods: true);
classJobResult.ClassDefinition.Methods[0].ParameterTypes[0] = "String";
var codeRunnerService = GetCodeRunnerService(classJobResult);
var grader = new ClassQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
var methodOverloadDefinitionError = codeQuestionResult.Errors
.Cast <MethodDefinitionError>()
.Single();
Assert.Equal(0.0, result.Score);
Assert.Equal("ExpectedClass", methodOverloadDefinitionError.ClassName);
Assert.Equal("requiredOverloadedMethod", methodOverloadDefinitionError.ExpectedMethodName);
Assert.True(methodOverloadDefinitionError.ExpectedPublic);
Assert.False(methodOverloadDefinitionError.ExpectedStatic);
Assert.Equal("int", methodOverloadDefinitionError.ExpectedParamTypes);
Assert.Equal("boolean", methodOverloadDefinitionError.ExpectedReturnType);
}
public async Task GradeSubmissionAsync_CorrectSubmission_CorrectScore(
bool constructor,
bool useGenerics,
bool overloadedMethods)
{
var question = GetClassQuestion
(
constructor: constructor,
useGenerics: useGenerics,
overloadedMethods: overloadedMethods
);
var classJobResult = GetClassJobResult
(
success: true,
constructor: constructor,
useGenerics: useGenerics,
overloadedMethods: overloadedMethods
);
var submission = new CodeQuestionSubmission()
{
Contents = "Submission"
};
var codeRunnerService = GetCodeRunnerService(classJobResult);
var grader = new ClassQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
Assert.Equal(1.0, result.Score);
Assert.Empty(codeQuestionResult.Errors);
}
public async Task GradeSubmissionAsync_WrongParameterTypes_Error()
{
var question = GetClassQuestion();
var submission = new CodeQuestionSubmission()
{
Contents = "Submission"
};
var classJobResult = GetClassJobResult(success: false);
classJobResult.ClassDefinition.Methods[0].ParameterTypes[1] = "double";
var codeRunnerService = GetCodeRunnerService(classJobResult);
var grader = new ClassQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
var methodDefinitionError = codeQuestionResult.Errors
.Cast <MethodDefinitionError>()
.Single();
Assert.Equal(0.0, result.Score);
Assert.Equal("requiredMethod", methodDefinitionError.ExpectedMethodName);
Assert.Equal("int, int", methodDefinitionError.ExpectedParamTypes);
}
public async Task GradeSubmissionAsync_TooManyOverloads_Error()
{
var question = GetClassQuestion(overloadedMethods: true);
var submission = new CodeQuestionSubmission()
{
Contents = "Submission"
};
question.RequiredMethods.RemoveAt(0);
var classJobResult = GetClassJobResult(success: false, overloadedMethods: true);
var codeRunnerService = GetCodeRunnerService(classJobResult);
var grader = new ClassQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
var methodOverloadCountError = codeQuestionResult.Errors
.Cast <MethodCountError>()
.Single();
Assert.Equal(0.0, result.Score);
Assert.Equal("ExpectedClass", methodOverloadCountError.ClassName);
Assert.Equal("requiredOverloadedMethod", methodOverloadCountError.ExpectedMethodName);
Assert.Equal(1, methodOverloadCountError.ExpectedCount);
Assert.False(methodOverloadCountError.ExpectedStatic);
}
public async Task GradeSubmissionAsync_PermittedPublicFields_Success()
{
var question = GetClassQuestion(allowPublicFields: true);
var submission = new CodeQuestionSubmission()
{
Contents = "Submission"
};
var classJobResult = GetClassJobResult(success: true);
classJobResult.ClassDefinition.Fields = new List <FieldDefinition>()
{
new FieldDefinition()
{
Name = "field1",
IsPublic = true,
Type = "String"
},
new FieldDefinition()
{
Name = "field2",
IsPublic = false,
Type = "String"
}
};
var codeRunnerService = GetCodeRunnerService(classJobResult);
var grader = new ClassQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
Assert.Equal(1.0, result.Score);
Assert.Empty(codeQuestionResult.Errors);
}
public async Task GradeSubmissionAsync_UnexpectedStaticMethod_Error()
{
var question = GetClassQuestion();
var submission = new CodeQuestionSubmission()
{
Contents = "Submission"
};
var classJobResult = GetClassJobResult(success: false);
classJobResult.ClassDefinition.Methods[0].IsStatic = true;
var codeRunnerService = GetCodeRunnerService(classJobResult);
var grader = new ClassQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
var methodDefinitionError = codeQuestionResult.Errors
.Cast <MethodDefinitionError>()
.Single();
Assert.Equal(0.0, result.Score);
Assert.Equal("requiredMethod", methodDefinitionError.ExpectedMethodName);
Assert.False(methodDefinitionError.ExpectedStatic);
}
public async Task GradeSubmissionAsync_WrongReturnType_Error()
{
var question = GetClassQuestion();
var submission = new CodeQuestionSubmission()
{
Contents = "Submission"
};
var classJobResult = GetClassJobResult(success: false);
classJobResult.ClassDefinition.Methods[0].ReturnType = "boolean";
var codeRunnerService = GetCodeRunnerService(classJobResult);
var grader = new ClassQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
var methodReturnTypeError = codeQuestionResult.Errors
.Cast <MethodReturnTypeError>()
.Single();
Assert.Equal(0.0, result.Score);
Assert.Equal("requiredMethod", methodReturnTypeError.MethodName);
Assert.Equal("String", methodReturnTypeError.ExpectedReturnType);
Assert.Equal("boolean", methodReturnTypeError.ActualReturnType);
}
public async Task GradeSubmissionAsync_ClassJobHasCorrectClassName()
{
var question = GetClassQuestion();
var classJobResult = GetClassJobResult(success: true);
var submission = new CodeQuestionSubmission() { Contents = "Submission" };
var codeRunnerService = GetCodeRunnerService
(
classJobResult,
job => job.ClassName == "ExpectedClass"
);
var grader = new ClassQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
Assert.Equal(1.0, result.Score);
}
public async Task GradeSubmissionAsync_ClassJobHasImportedClasses()
{
var question = GetClassQuestion();
var classJobResult = GetClassJobResult(success: true);
var submission = new CodeQuestionSubmission() { Contents = "Submission" };
var codeRunnerService = GetCodeRunnerService
(
classJobResult,
job => job.ClassesToImport.Count == 1
&& job.ClassesToImport[0] == "package.classToImport"
);
var grader = new ClassQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
Assert.Equal(1.0, result.Score);
}
public async Task GradeSubmissionAsync_CorrectSubmission_ValidTestDescription()
{
var question = GetClassQuestion();
var classJobResult = GetClassJobResult(success: true);
var submission = new CodeQuestionSubmission()
{
Contents = "Submission"
};
var codeRunnerService = GetCodeRunnerService(classJobResult);
var grader = new ClassQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
var testResult = codeQuestionResult.TestResults.Single();
Assert.Equal("Description", testResult.Description);
}
public async Task GradeSubmissionAsync_ClassJobHasCorrectClassName()
{
var question = GetClassQuestion();
var classJobResult = GetClassJobResult(success: true);
var submission = new CodeQuestionSubmission()
{
Contents = "Submission"
};
var codeRunnerService = GetCodeRunnerService
(
classJobResult,
job => job.ClassName == "ExpectedClass"
);
var grader = new ClassQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
Assert.Equal(1.0, result.Score);
}
public async Task GradeSubmissionAsync_ClassJobHasImportedClasses()
{
var question = GetClassQuestion();
var classJobResult = GetClassJobResult(success: true);
var submission = new CodeQuestionSubmission()
{
Contents = "Submission"
};
var codeRunnerService = GetCodeRunnerService
(
classJobResult,
job => job.ClassesToImport.Count == 1 &&
job.ClassesToImport[0] == "package.classToImport"
);
var grader = new ClassQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
Assert.Equal(1.0, result.Score);
}
public async Task GradeSubmissionAsync_ClassJobHasTests()
{
var question = GetClassQuestion();
var classJobResult = GetClassJobResult(success: true);
var submission = new CodeQuestionSubmission()
{
Contents = "Submission %"
};
var codeRunnerService = GetCodeRunnerService
(
classJobResult,
job => job.Tests.Count == 1 &&
job.Tests[0].TestName == "test1" &&
job.Tests[0].MethodBody == "Method Body" &&
job.Tests[0].ReturnType == "String"
);
var grader = new ClassQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
Assert.Equal(1.0, result.Score);
}
public async Task GradeSubmissionAsync_ForbiddenPublicFields_Error()
{
var question = GetClassQuestion(allowPublicFields: false);
var submission = new CodeQuestionSubmission()
{
Contents = "Submission"
};
var classJobResult = GetClassJobResult(success: false);
classJobResult.ClassDefinition.Fields = new List <FieldDefinition>()
{
new FieldDefinition()
{
Name = "field1",
IsPublic = true,
Type = "String"
},
new FieldDefinition()
{
Name = "field2",
IsPublic = false,
Type = "String"
}
};
var codeRunnerService = GetCodeRunnerService(classJobResult);
var grader = new ClassQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
var fieldVisibilityError = codeQuestionResult.Errors
.Cast <FieldVisibilityError>()
.Single();
Assert.Equal(0.0, result.Score);
Assert.Equal("ExpectedClass", fieldVisibilityError.ClassName);
}
public async Task GradeSubmissionAsync_MissingExpectedClass_Error()
{
var question = GetClassQuestion();
var submission = new CodeQuestionSubmission()
{
Contents = "Submission"
};
var classJobResult = GetClassJobResult(success: false);
classJobResult.TestsCompilationResult = null;
classJobResult.ClassDefinition = null;
classJobResult.ClassCompilationResult = new CompilationResult()
{
Success = false,
Errors = Collections.CreateList
(
new CompileError()
{
Message = "class ExpectedClass should be declared in ExpectedClass.java"
}
)
};
var codeRunnerService = GetCodeRunnerService(classJobResult);
var grader = new ClassQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
var missingRequiredClassError = codeQuestionResult.Errors
.Cast <MissingRequiredClassError>()
.Single();
Assert.Equal(0.0, result.Score);
Assert.Equal("ExpectedClass", missingRequiredClassError.RequiredClassName);
}
public async Task GradeSubmissionAsync_ClassJobHasCorrectFileContents()
{
var question = GetClassQuestion();
var classJobResult = GetClassJobResult(success: true);
var submission = new CodeQuestionSubmission()
{
Contents = "Submission %"
};
var codeRunnerService = GetCodeRunnerService
(
classJobResult,
job => job.FileContents ==
"class InternalClass\n" +
"{\n" +
"}\n" +
"\n"
+ "Submission %%"
);
var grader = new ClassQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
Assert.Equal(1.0, result.Score);
}
public async Task GradeSubmissionAsync_ClassJobHasTests()
{
var question = GetClassQuestion();
var classJobResult = GetClassJobResult(success: true);
var submission = new CodeQuestionSubmission() { Contents = "Submission %" };
var codeRunnerService = GetCodeRunnerService
(
classJobResult,
job => job.Tests.Count == 1
&& job.Tests[0].TestName == "test1"
&& job.Tests[0].MethodBody == "Method Body"
&& job.Tests[0].ReturnType == "String"
);
var grader = new ClassQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
Assert.Equal(1.0, result.Score);
}
public async Task GradeSubmissionAsync_ClassJobHasCorrectFileContents()
{
var question = GetClassQuestion();
var classJobResult = GetClassJobResult(success: true);
var submission = new CodeQuestionSubmission() { Contents = "Submission %" };
var codeRunnerService = GetCodeRunnerService
(
classJobResult,
job => job.FileContents ==
"class InternalClass\n" +
"{\n" +
"}\n" +
"\n"
+ "Submission %%"
);
var grader = new ClassQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
Assert.Equal(1.0, result.Score);
}
public async Task GradeSubmissionAsync_WrongOverloadSignature_Error()
{
var question = GetClassQuestion(overloadedMethods: true);
var submission = new CodeQuestionSubmission() { Contents = "Submission" };
var classJobResult = GetClassJobResult(success: false, overloadedMethods: true);
classJobResult.ClassDefinition.Methods[0].ParameterTypes[0] = "String";
var codeRunnerService = GetCodeRunnerService(classJobResult);
var grader = new ClassQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
var methodOverloadDefinitionError = codeQuestionResult.Errors
.Cast<MethodOverloadDefinitionError>()
.Single();
Assert.Equal(0.0, result.Score);
Assert.Equal("ExpectedClass", methodOverloadDefinitionError.ClassName);
Assert.Equal("requiredOverloadedMethod", methodOverloadDefinitionError.ExpectedMethodName);
Assert.True(methodOverloadDefinitionError.ExpectedPublic);
Assert.False(methodOverloadDefinitionError.ExpectedStatic);
Assert.Equal("int", methodOverloadDefinitionError.ExpectedParamTypes);
Assert.Equal("boolean", methodOverloadDefinitionError.ExpectedReturnType);
}
public async Task GradeSubmissionAsync_CorrectSubmission_ValidTestDescription()
{
var question = GetClassQuestion();
var classJobResult = GetClassJobResult(success: true);
var submission = new CodeQuestionSubmission() { Contents = "Submission" };
var codeRunnerService = GetCodeRunnerService(classJobResult);
var grader = new ClassQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
var testResult = codeQuestionResult.TestResults.Single();
Assert.Equal(1.0, result.Score);
Assert.Equal(0, codeQuestionResult.Errors.Count);
Assert.Equal("Description", testResult.Description);
}
public async Task GradeSubmissionAsync_WrongParameterTypes_Error()
{
var question = GetClassQuestion();
var submission = new CodeQuestionSubmission() { Contents = "Submission" };
var classJobResult = GetClassJobResult(success: false);
classJobResult.ClassDefinition.Methods[0].ParameterTypes[1] = "double";
var codeRunnerService = GetCodeRunnerService(classJobResult);
var grader = new ClassQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
var methodParameterTypesError = codeQuestionResult.Errors
.Cast<MethodParameterTypesError>()
.Single();
Assert.Equal(0.0, result.Score);
Assert.Equal("requiredMethod", methodParameterTypesError.MethodName);
Assert.Equal(2, methodParameterTypesError.ExpectedParamTypes.Count);
Assert.Equal("int", methodParameterTypesError.ExpectedParamTypes[0]);
Assert.Equal("int", methodParameterTypesError.ExpectedParamTypes[1]);
Assert.Equal("int", methodParameterTypesError.ActualParamTypes[0]);
Assert.Equal("double", methodParameterTypesError.ActualParamTypes[1]);
}
public async Task GradeSubmissionAsync_WrongOverloadCount_Error()
{
var question = GetClassQuestion(overloadedMethods: true);
var submission = new CodeQuestionSubmission() { Contents = "Submission" };
var classJobResult = GetClassJobResult(success: false, overloadedMethods: true);
classJobResult.ClassDefinition.Methods.RemoveAt(0);
var codeRunnerService = GetCodeRunnerService(classJobResult);
var grader = new ClassQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
var methodOverloadCountError = codeQuestionResult.Errors
.Cast<MethodOverloadCountError>()
.Single();
Assert.Equal(0.0, result.Score);
Assert.Equal("ExpectedClass", methodOverloadCountError.ClassName);
Assert.Equal("requiredOverloadedMethod", methodOverloadCountError.ExpectedMethodName);
Assert.Equal(2, methodOverloadCountError.ExpectedOverloadCount);
Assert.False(methodOverloadCountError.ExpectedStatic);
}
public async Task GradeSubmissionAsync_WrongReturnType_Error()
{
var question = GetClassQuestion();
var submission = new CodeQuestionSubmission() { Contents = "Submission" };
var classJobResult = GetClassJobResult(success: false);
classJobResult.ClassDefinition.Methods[0].ReturnType = "boolean";
var codeRunnerService = GetCodeRunnerService(classJobResult);
var grader = new ClassQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
var methodReturnTypeError = codeQuestionResult.Errors
.Cast<MethodReturnTypeError>()
.Single();
Assert.Equal(0.0, result.Score);
Assert.Equal("requiredMethod", methodReturnTypeError.MethodName);
Assert.Equal("String", methodReturnTypeError.ExpectedReturnType);
Assert.Equal("boolean", methodReturnTypeError.ActualReturnType);
}
public async Task GradeSubmissionAsync_UnexpectedStaticMethod_Error()
{
var question = GetClassQuestion();
var submission = new CodeQuestionSubmission() { Contents = "Submission" };
var classJobResult = GetClassJobResult(success: false);
classJobResult.ClassDefinition.Methods[0].IsStatic = true;
var codeRunnerService = GetCodeRunnerService(classJobResult);
var grader = new ClassQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
var methodStaticError = codeQuestionResult.Errors
.Cast<MethodStaticError>()
.Single();
Assert.Equal(0.0, result.Score);
Assert.Equal("requiredMethod", methodStaticError.MethodName);
Assert.False(methodStaticError.ExpectedStatic);
}
public async Task GradeSubmissionAsync_MissingExpectedClass_Error()
{
var question = GetClassQuestion();
var submission = new CodeQuestionSubmission() { Contents = "Submission" };
var classJobResult = GetClassJobResult(success: false);
classJobResult.TestsCompilationResult = null;
classJobResult.ClassDefinition = null;
classJobResult.ClassCompilationResult = new CompilationResult()
{
Success = false,
Errors = Collections.CreateList
(
new CompileError()
{
Message = "class ExpectedClass should be declared in ExpectedClass.java"
}
)
};
var codeRunnerService = GetCodeRunnerService(classJobResult);
var grader = new ClassQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
var missingRequiredClassError = codeQuestionResult.Errors
.Cast<MissingRequiredClassError>()
.Single();
Assert.Equal(0.0, result.Score);
Assert.Equal("ExpectedClass", missingRequiredClassError.RequiredClassName);
}
public async Task GradeSubmissionAsync_PermittedPublicFields_Success()
{
var question = GetClassQuestion(allowPublicFields: true);
var submission = new CodeQuestionSubmission() { Contents = "Submission" };
var classJobResult = GetClassJobResult(success: true);
classJobResult.ClassDefinition.Fields = new List<FieldDefinition>()
{
new FieldDefinition()
{
Name = "field1",
IsPublic = true,
Type = "String"
},
new FieldDefinition()
{
Name = "field2",
IsPublic = false,
Type = "String"
}
};
var codeRunnerService = GetCodeRunnerService(classJobResult);
var grader = new ClassQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
Assert.Equal(1.0, result.Score);
Assert.Equal(0, codeQuestionResult.Errors.Count);
}
public async Task GradeSubmissionAsync_ForbiddenPublicFields_Error()
{
var question = GetClassQuestion(allowPublicFields: false);
var submission = new CodeQuestionSubmission() { Contents = "Submission" };
var classJobResult = GetClassJobResult(success: false);
classJobResult.ClassDefinition.Fields = new List<FieldDefinition>()
{
new FieldDefinition()
{
Name = "field1",
IsPublic = true,
Type = "String"
},
new FieldDefinition()
{
Name = "field2",
IsPublic = false,
Type = "String"
}
};
var codeRunnerService = GetCodeRunnerService(classJobResult);
var grader = new ClassQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
var fieldVisibilityError = codeQuestionResult.Errors
.Cast<FieldVisibilityError>()
.Single();
Assert.Equal(0.0, result.Score);
Assert.Equal("ExpectedClass", fieldVisibilityError.ClassName);
}
