/// <summary>
/// Returns the URL to visualize the execution of a test.
/// </summary>
private string GetTestVisualizeUrl(CodeQuestionTest test, CodeQuestionSubmission submission)
{
var urlEncodedCode = WebUtility.UrlEncode(GetVisualizerFileContents(test, submission));
var urlEncodedArgs = GetVisualizerCommandLineArguments(test).Select(WebUtility.UrlEncode).ToList();
var argsQueryStringVar = urlEncodedArgs.Any()
? $"&args={string.Join(",", urlEncodedArgs)}"
: string.Empty;
return($"{c_visualizerBaseUrl}#code={urlEncodedCode}{argsQueryStringVar}");
}
/// <summary>
/// Returns any errors with the submission's definition (classes/methods).
/// </summary>
protected override IEnumerable <CodeQuestionError> GetDefinitionErrors(
CodeQuestionSubmission submission,
CodeJobResult jobResult)
{
var classJobResult = jobResult as ClassJobResult;
if (classJobResult == null)
{
throw new ArgumentException("Invalid job result type", nameof(jobResult));
}
var submittedClass = classJobResult.ClassDefinition;
if (submittedClass == null)
{
yield break;
}
if (!Question.AllowPublicFields && submittedClass.Fields.Any(field => field.IsPublic))
{
yield return(new FieldVisibilityError(Question.ClassName));
}
if (Question.RequiredMethods != null)
{
foreach (var requiredMethodGroup in Question.RequiredMethods.GroupBy(rm => rm.Name))
{
IEnumerable <CodeQuestionError> methodDefinitionErrors;
if (requiredMethodGroup.Count() == 1)
{
var requiredMethod = requiredMethodGroup.First();
methodDefinitionErrors = GetSingleMethodDefinitionErrors
(
requiredMethod,
submittedClass
);
}
else
{
methodDefinitionErrors = GetOverloadedMethodDefinitionErrors
(
requiredMethodGroup,
submittedClass
);
}
foreach (var error in methodDefinitionErrors)
{
yield return(error);
}
}
}
}
/// <summary>
/// Returns the question file template, with the submission substituted for the placeholder.
/// </summary>
private string GetFileTemplateWithSubmission(CodeQuestionSubmission submission)
{
var templateLines = Question.FileTemplate.Split('\n');
var submissionLine = templateLines.Single(line => line.Contains(ClassQuestion.SubmissionPlaceholder));
var linePrefix = submissionLine.Substring(0, submissionLine.IndexOf(ClassQuestion.SubmissionPlaceholder));
var submissionLines = submission.Contents.Split('\n');
var indentedSubmission = string.Join("\n", submissionLines.Select(line => $"{linePrefix}{line}"));
return(Question.FileTemplate.Replace(submissionLine, indentedSubmission));
}
/// <summary>
/// Adds the body of the main method to the visualizer class.
/// </summary>
private void AddVisualizerMainMethodBody(
JavaFileBuilder builder,
MethodQuestionTest methodTest,
CodeQuestionSubmission submission)
{
string variableDeclaration = Question.ReturnType == "void"
? ""
: $"{Question.ReturnType} returnValue = ";
builder.AddLine($"{variableDeclaration}{Question.MethodName}({methodTest.ParameterValues});");
}
/// <summary>
/// Returns any errors with the submission's definition (classes/methods).
/// </summary>
protected override IEnumerable <CodeQuestionError> GetDefinitionErrors(
CodeQuestionSubmission submission,
CodeJobResult jobResult)
{
var methodJobResult = jobResult as MethodJobResult;
if (methodJobResult == null)
{
throw new ArgumentException("Invalid job result type", nameof(jobResult));
}
var submittedMethod = methodJobResult.MethodDefinition;
if (jobResult.ClassCompilationResult.Success && submittedMethod == null)
{
yield return(new MethodMissingError(null /*className*/, Question.MethodName, expectedStatic: true));
yield break;
}
if (submittedMethod.Name != Question.MethodName)
{
yield return(new MethodNameError(Question.MethodName, submittedMethod.Name));
}
if (!submittedMethod.IsPublic)
{
yield return(new MethodVisibilityError(Question.MethodName, expectedPublic: true));
}
if (!submittedMethod.IsStatic)
{
yield return(new MethodStaticError(Question.MethodName, expectedStatic: true));
}
if (submittedMethod.ReturnType != Question.ReturnType)
{
yield return(new MethodReturnTypeError(Question.MethodName, Question.ReturnType, submittedMethod.ReturnType));
}
var expectedParamTypes = Question.ParameterTypes
.Split(',')
.Select(paramType => paramType.Trim())
.ToList();
if (!expectedParamTypes.SequenceEqual(submittedMethod.ParameterTypes))
{
yield return(new MethodParameterTypesError(Question.MethodName, expectedParamTypes, submittedMethod.ParameterTypes));
}
}
public async Task GradeSubmissionAsync_TestCompilationFailed_Error()
{
var question = GetCodeQuestion();
var submission = new CodeQuestionSubmission()
{
Contents = "A A B B C C"
};
var simulatedResult = new MethodJobResult()
{
Status = CodeJobStatus.Completed,
ClassCompilationResult = new CompilationResult()
{
Success = true
},
TestsCompilationResult = new CompilationResult()
{
Success = false,
Errors = Collections.CreateList
(
new CompileError()
{
Message = "ShortError1",
FullError = "FullError1"
},
new CompileError()
{
Message = "ShortError2",
FullError = "FullError2"
}
)
}
};
var grader = GetCodeQuestionGrader(question, submission, simulatedResult);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
var testsCompilationErrors = codeQuestionResult.Errors
.Cast <TestCompilationError>()
.ToList();
Assert.Equal(0.0, result.Score);
Assert.Equal(2, testsCompilationErrors.Count);
Assert.Null(testsCompilationErrors[0].LineNumber);
Assert.Null(testsCompilationErrors[0].LineErrorText);
Assert.Contains("FullError1", testsCompilationErrors[0].FullErrorText);
Assert.Null(testsCompilationErrors[1].LineNumber);
Assert.Null(testsCompilationErrors[1].LineErrorText);
Assert.Contains("FullError2", testsCompilationErrors[1].FullErrorText);
}
/// <summary>
/// Adds a class to a java file to run in the visualizer, for the given test.
/// </summary>
protected override void AddVisualizerClass(
JavaFileBuilder builder,
CodeQuestionTest test,
CodeQuestionSubmission submission)
{
var classTest = test as ProgramQuestionTest;
if (test == null)
{
throw new ArgumentException("Invalid test type", nameof(test));
}
builder.AddLines(submission.Contents);
}
/// <summary>
/// Returns the contents of the java file to run with the java visualizer, for the given test.
/// </summary>
private string GetVisualizerFileContents(CodeQuestionTest test, CodeQuestionSubmission submission)
{
JavaFileBuilder builder = new JavaFileBuilder();
if (Question.ImportedClasses != null)
{
builder.AddLines(Question.ImportedClasses.Select(importedClass => $"import {importedClass.ClassName};"));
}
builder.AddBlankLine();
AddVisualizerClass(builder, test, submission);
return(builder.GetFileContents());
}
/// <summary>
/// Returns the first non-empty set computed by the given compute functions.
/// </summary>
private IEnumerable <CodeQuestionError> GetFirstNonEmptyErrorSet(
CodeQuestionSubmission submission,
CodeJobResult jobResult,
ComputeErrorSet[] computeErrorSets)
{
foreach (var computeErrorSet in computeErrorSets)
{
var errors = computeErrorSet(submission, jobResult).ToList();
if (errors.Any())
{
return(errors.ToList());
}
}
return(Enumerable.Empty <CodeQuestionError>());
}
/// <summary>
/// Adds a class to a java file to run in the visualizer, for the given test.
/// </summary>
protected override void AddVisualizerClass(
JavaFileBuilder builder,
CodeQuestionTest test,
CodeQuestionSubmission submission)
{
var classTest = test as ClassQuestionTest;
if (test == null)
{
throw new ArgumentException("Invalid test type", nameof(test));
}
var templateWithSubmission = GetFileTemplateWithSubmission(submission);
var regex = new Regex("public\\s+class");
var modifiedSubmission = regex.Replace(templateWithSubmission, "class");
builder.AddLines(modifiedSubmission);
builder.AddBlankLine();
builder.AddLine("public class Runner");
builder.BeginScope("{");
if (classTest.ReturnType != "void")
{
builder.AddLine($"public static {classTest.ReturnType} runTest()");
builder.BeginScope("{");
builder.AddLines(classTest.MethodBody);
builder.EndScope("}");
}
builder.AddBlankLine();
builder.AddLine("public static void main(String[] args)");
builder.BeginScope("{");
if (classTest.ReturnType != "void")
{
builder.AddLine($"{classTest.ReturnType} returnVal = runTest();");
}
else
{
builder.AddLines(classTest.MethodBody);
}
builder.EndScope("}");
builder.EndScope("}");
}
/// <summary>
/// Returns a list of errors for the job.
/// </summary>
private IEnumerable <CodeQuestionError> GetPostJobExecutionErrors(
CodeQuestionSubmission submission,
CodeJobResult jobResult)
{
return(GetFirstNonEmptyErrorSet
(
submission,
jobResult,
new ComputeErrorSet[]
{
GetJobExecutionErrors,
GetClassCompilationErrors,
GetDefinitionErrors,
GetTestCompilationErrors
}
));
}
public async Task GradeSubmissionAsync_TestFailsIncorrectReturnValue_AccurateResults()
{
var question = GetCodeQuestion();
var submission = new CodeQuestionSubmission()
{
Contents = "A A B B C C"
};
var simulatedResult = new MethodJobResult()
{
Status = CodeJobStatus.Completed,
ClassCompilationResult = new CompilationResult()
{
Success = true
},
TestsCompilationResult = new CompilationResult()
{
Success = true
},
TestResults = Collections.CreateList
(
new CodeTestResult()
{
Name = "test1",
Completed = true,
ReturnValue = "incorrectReturnValue",
Output = "expectedOutput"
}
)
};
var grader = GetCodeQuestionGrader(question, submission, simulatedResult);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
var testResults = codeQuestionResult.TestResults;
Assert.Single(testResults);
Assert.Equal("test1", testResults[0].Description);
Assert.False(testResults[0].Succeeded);
Assert.Null(testResults[0].ExceptionText);
Assert.Equal("expectedReturnValue", testResults[0].ExpectedReturnValue);
Assert.Equal("incorrectReturnValue", testResults[0].ActualReturnValue);
Assert.Equal("expectedOutput", testResults[0].ExpectedOutput);
Assert.Equal("expectedOutput", testResults[0].ActualOutput);
}
public async Task GradeSubmissionAsync_TestsAllPass_FullCredit()
{
var question = GetCodeQuestion(numTests: 2);
var submission = new CodeQuestionSubmission()
{
Contents = "A A B B C C"
};
var simulatedResult = new MethodJobResult()
{
Status = CodeJobStatus.Completed,
ClassCompilationResult = new CompilationResult()
{
Success = true
},
TestsCompilationResult = new CompilationResult()
{
Success = true
},
TestResults = Collections.CreateList
(
new CodeTestResult()
{
Name = "test1",
Completed = true,
ReturnValue = "expectedReturnValue",
Output = "expectedOutput"
},
new CodeTestResult()
{
Name = "test2",
Completed = true,
ReturnValue = "expectedReturnValue",
Output = "expectedOutput"
}
)
};
var grader = GetCodeQuestionGrader(question, submission, simulatedResult);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
Assert.Equal(1.0, result.Score);
Assert.Empty(codeQuestionResult.Errors);
}
public async Task GradeSubmissionAsync_CorrectSubmission_ValidTestDescription()
{
var question = GetClassQuestion();
var classJobResult = GetClassJobResult(success: true);
var submission = new CodeQuestionSubmission()
{
Contents = "Submission"
};
var codeRunnerService = GetCodeRunnerService(classJobResult);
var grader = new ClassQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
var testResult = codeQuestionResult.TestResults.Single();
Assert.Equal("Description", testResult.Description);
}
public async Task GradeSubmissionAsync_CorrectSubmission_ValidTestDescription()
{
var question = GetMethodQuestion();
var submission = new CodeQuestionSubmission()
{
Contents = "Submission"
};
var methodJobResult = GetSuccessfulMethodJobResult();
var codeRunnerService = GetCodeRunnerService(methodJobResult);
var grader = new MethodQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
var testResult = codeQuestionResult.TestResults.Single();
Assert.Equal("expectedMethod(1, 2)", testResult.Description);
}
public async Task GradeSubmissionAsync_CorrectSubmission_CorrectScore(bool useGenerics)
{
var question = GetMethodQuestion(useGenerics);
var submission = new CodeQuestionSubmission()
{
Contents = "Submission"
};
var methodJobResult = GetSuccessfulMethodJobResult(useGenerics);
var codeRunnerService = GetCodeRunnerService(methodJobResult);
var grader = new MethodQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
var testResult = codeQuestionResult.TestResults.Single();
Assert.Equal(1.0, result.Score);
Assert.Empty(codeQuestionResult.Errors);
}
public async Task GradeSubmissionAsync_NoSubmission_Error()
{
var question = GetCodeQuestion();
var submission = new CodeQuestionSubmission()
{
Contents = ""
};
var grader = GetCodeQuestionGrader(question, submission, simulatedResult: null);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
var noSubmissionErrors = codeQuestionResult.Errors
.Cast <NoSubmissionError>()
.ToList();
Assert.Equal(0.0, result.Score);
Assert.Single(noSubmissionErrors);
}
public async Task GradeSubmissionAsync_ClassJobHasCorrectLineOffset()
{
var question = GetProgramQuestion();
var classJobResult = GetClassJobResult(success: true);
var submission = new CodeQuestionSubmission()
{
Contents = "Submission"
};
var codeRunnerService = GetCodeRunnerService
(
classJobResult,
job => job.LineNumberOffset == 0
);
var grader = new ProgramQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
Assert.Equal(1.0, result.Score);
}
public async Task GradeSubmissionAsync_MethodJobHasCorrectCode()
{
var question = GetMethodQuestion();
var methodJobResult = GetSuccessfulMethodJobResult();
var submission = new CodeQuestionSubmission()
{
Contents = "Submission %"
};
var codeRunnerService = GetCodeRunnerService
(
methodJobResult,
job => job.MethodCode == "Submission %%"
);
var grader = new MethodQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
Assert.Equal(1.0, result.Score);
}
public async Task GradeSubmissionAsync_ClassJobHasCorrectClassName()
{
var question = GetClassQuestion();
var classJobResult = GetClassJobResult(success: true);
var submission = new CodeQuestionSubmission()
{
Contents = "Submission"
};
var codeRunnerService = GetCodeRunnerService
(
classJobResult,
job => job.ClassName == "ExpectedClass"
);
var grader = new ClassQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
Assert.Equal(1.0, result.Score);
}
/// <summary>
/// Returns any errors that prevented the job from executing.
/// </summary>
private IEnumerable <CodeQuestionError> GetConstraintErrors(
CodeQuestionSubmission submission)
{
if (Question.CodeConstraints == null)
{
yield break;
}
foreach (var constraint in Question.CodeConstraints)
{
int count = Regex.Matches(submission.Contents, constraint.Regex).Count;
bool constraintMet;
switch (constraint.Type)
{
case CodeConstraintType.AtLeast:
constraintMet = count >= constraint.Frequency;
break;
case CodeConstraintType.Exactly:
constraintMet = count == constraint.Frequency;
break;
case CodeConstraintType.AtMost:
constraintMet = count <= constraint.Frequency;
break;
default:
throw new InvalidOperationException("Invalid code constraint type.");
}
if (!constraintMet)
{
yield return(new CodeConstraintError
(
constraint.Regex,
count,
constraint.ErrorMessage
));
}
}
}
public async Task GradeSubmissionAsync_ClassJobHasImportedClasses()
{
var question = GetClassQuestion();
var classJobResult = GetClassJobResult(success: true);
var submission = new CodeQuestionSubmission()
{
Contents = "Submission"
};
var codeRunnerService = GetCodeRunnerService
(
classJobResult,
job => job.ClassesToImport.Count == 1 &&
job.ClassesToImport[0] == "package.classToImport"
);
var grader = new ClassQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
Assert.Equal(1.0, result.Score);
}
public async Task GradeSubmissionAsync_MethodJobHasTests()
{
var question = GetMethodQuestion();
var methodJobResult = GetSuccessfulMethodJobResult();
var submission = new CodeQuestionSubmission()
{
Contents = "Submission %"
};
var codeRunnerService = GetCodeRunnerService
(
methodJobResult,
job => job.Tests.Count == 1 &&
job.Tests[0].TestName == "test1" &&
job.Tests[0].ParamValues == "1, 2"
);
var grader = new MethodQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
Assert.Equal(1.0, result.Score);
}
public async Task GradeSubmissionAsync_MissingMethod_Error()
{
var question = GetMethodQuestion();
var submission = new CodeQuestionSubmission()
{
Contents = "Submission"
};
var methodJobResult = GetFailedMethodJobResult(definition: null);
var codeRunnerService = GetCodeRunnerService(methodJobResult);
var grader = new MethodQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
var methodMissingError = codeQuestionResult.Errors
.Cast <MethodMissingError>()
.Single();
Assert.Equal(0.0, result.Score);
Assert.Equal("expectedMethod", methodMissingError.ExpectedMethodName);
Assert.True(methodMissingError.ExpectedStatic);
}
public async Task GradeSubmissionAsync_ClassJobHasTests()
{
var question = GetClassQuestion();
var classJobResult = GetClassJobResult(success: true);
var submission = new CodeQuestionSubmission()
{
Contents = "Submission %"
};
var codeRunnerService = GetCodeRunnerService
(
classJobResult,
job => job.Tests.Count == 1 &&
job.Tests[0].TestName == "test1" &&
job.Tests[0].MethodBody == "Method Body" &&
job.Tests[0].ReturnType == "String"
);
var grader = new ClassQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
Assert.Equal(1.0, result.Score);
}
/// <summary>
/// Adds a class to a java file to run in the visualizer, for the given test.
/// </summary>
protected override void AddVisualizerClass(
JavaFileBuilder builder,
CodeQuestionTest test,
CodeQuestionSubmission submission)
{
var methodTest = test as MethodQuestionTest;
if (test == null)
{
throw new ArgumentException("Invalid test type", nameof(test));
}
builder.AddLine("public class Exercise");
builder.BeginScope("{");
builder.AddLines(submission.Contents);
builder.AddBlankLine();
builder.AddLine("public static void main(String[] args)");
builder.BeginScope("{");
AddVisualizerMainMethodBody(builder, methodTest, submission);
builder.EndScope("}");
builder.EndScope("}");
}
/// <summary>
/// Returns a code question grader. For a given question, the grader
/// returns a given simulated result for a given submission.
/// </summary>
public CodeQuestionGrader <CodeQuestion> GetCodeQuestionGrader(
CodeQuestion question,
CodeQuestionSubmission submission,
CodeJobResult simulatedResult,
IList <DefinitionError> definitionErrors = null)
{
var grader = new Mock <CodeQuestionGrader <CodeQuestion> >
(
question,
null /*codeRunnerService*/
);
grader.CallBase = true;
grader.Protected()
.Setup <Task <CodeJobResult> >
(
"ExecuteJobAsync",
ItExpr.Is <CodeQuestionSubmission>(s => s == submission)
).Returns(Task.FromResult(simulatedResult));
grader.Protected()
.Setup <string>("GetTestDescription", ItExpr.IsAny <CodeQuestionTest>())
.Returns <CodeQuestionTest>(test => test.Name);
if (definitionErrors != null)
{
grader.Protected()
.Setup <IEnumerable <CodeQuestionError> >
(
"GetDefinitionErrors",
ItExpr.Is <CodeQuestionSubmission>(s => s == submission),
ItExpr.Is <CodeJobResult>(r => r == simulatedResult)
).Returns(definitionErrors);
}
return(grader.Object);
}
public async Task GradeSubmissionAsync_ForbiddenPublicFields_Error()
{
var question = GetClassQuestion(allowPublicFields: false);
var submission = new CodeQuestionSubmission()
{
Contents = "Submission"
};
var classJobResult = GetClassJobResult(success: false);
classJobResult.ClassDefinition.Fields = new List <FieldDefinition>()
{
new FieldDefinition()
{
Name = "field1",
IsPublic = true,
Type = "String"
},
new FieldDefinition()
{
Name = "field2",
IsPublic = false,
Type = "String"
}
};
var codeRunnerService = GetCodeRunnerService(classJobResult);
var grader = new ClassQuestionGrader(question, codeRunnerService);
var result = await grader.GradeSubmissionAsync(submission);
var codeQuestionResult = (CodeQuestionResult)result.Result;
var fieldVisibilityError = codeQuestionResult.Errors
.Cast <FieldVisibilityError>()
.Single();
Assert.Equal(0.0, result.Score);
Assert.Equal("ExpectedClass", fieldVisibilityError.ClassName);
}
/// <summary>
/// Returns any errors that prevented the job from executing.
/// </summary>
private IEnumerable <CodeQuestionError> GetJobExecutionErrors(
CodeQuestionSubmission submission,
CodeJobResult jobResult)
{
switch (jobResult.Status)
{
case CodeJobStatus.Timeout:
yield return(new TimeoutError());
break;
case CodeJobStatus.Error:
yield return(new DiagnosticError(jobResult.DiagnosticOutput));
break;
case CodeJobStatus.Completed:
yield break;
default:
throw new InvalidOperationException("Unuspported code job status.");
}
}
/// <summary>
/// Executes a code job to grade the question.
/// </summary>
protected override async Task <CodeJobResult> ExecuteJobAsync(CodeQuestionSubmission submission)
{
return(await CodeRunnerService.ExecuteMethodJobAsync
(
new MethodJob()
{
ClassesToImport = Question.ImportedClasses
?.Select(importedClass => importedClass.ClassName)
?.ToList() ?? new List <string>(),
MethodCode = submission.Contents.Replace("%", "%%"),
Tests = Question.Tests.Select
(
test => new MethodTest()
{
TestName = GetCodeTestName(test),
ParamValues = test.ParameterValues
}
).ToList()
}
));
}
