/// <summary> /// Returns the URL to visualize the execution of a test. /// </summary> private string GetTestVisualizeUrl(CodeQuestionTest test, CodeQuestionSubmission submission) { var urlEncodedCode = WebUtility.UrlEncode(GetVisualizerFileContents(test, submission)); var urlEncodedArgs = GetVisualizerCommandLineArguments(test).Select(WebUtility.UrlEncode).ToList(); var argsQueryStringVar = urlEncodedArgs.Any() ? $"&args={string.Join(",", urlEncodedArgs)}" : string.Empty; return($"{c_visualizerBaseUrl}#code={urlEncodedCode}{argsQueryStringVar}"); }
/// <summary> /// Returns any errors with the submission's definition (classes/methods). /// </summary> protected override IEnumerable <CodeQuestionError> GetDefinitionErrors( CodeQuestionSubmission submission, CodeJobResult jobResult) { var classJobResult = jobResult as ClassJobResult; if (classJobResult == null) { throw new ArgumentException("Invalid job result type", nameof(jobResult)); } var submittedClass = classJobResult.ClassDefinition; if (submittedClass == null) { yield break; } if (!Question.AllowPublicFields && submittedClass.Fields.Any(field => field.IsPublic)) { yield return(new FieldVisibilityError(Question.ClassName)); } if (Question.RequiredMethods != null) { foreach (var requiredMethodGroup in Question.RequiredMethods.GroupBy(rm => rm.Name)) { IEnumerable <CodeQuestionError> methodDefinitionErrors; if (requiredMethodGroup.Count() == 1) { var requiredMethod = requiredMethodGroup.First(); methodDefinitionErrors = GetSingleMethodDefinitionErrors ( requiredMethod, submittedClass ); } else { methodDefinitionErrors = GetOverloadedMethodDefinitionErrors ( requiredMethodGroup, submittedClass ); } foreach (var error in methodDefinitionErrors) { yield return(error); } } } }
/// <summary> /// Returns the question file template, with the submission substituted for the placeholder. /// </summary> private string GetFileTemplateWithSubmission(CodeQuestionSubmission submission) { var templateLines = Question.FileTemplate.Split('\n'); var submissionLine = templateLines.Single(line => line.Contains(ClassQuestion.SubmissionPlaceholder)); var linePrefix = submissionLine.Substring(0, submissionLine.IndexOf(ClassQuestion.SubmissionPlaceholder)); var submissionLines = submission.Contents.Split('\n'); var indentedSubmission = string.Join("\n", submissionLines.Select(line => $"{linePrefix}{line}")); return(Question.FileTemplate.Replace(submissionLine, indentedSubmission)); }
/// <summary> /// Adds the body of the main method to the visualizer class. /// </summary> private void AddVisualizerMainMethodBody( JavaFileBuilder builder, MethodQuestionTest methodTest, CodeQuestionSubmission submission) { string variableDeclaration = Question.ReturnType == "void" ? "" : $"{Question.ReturnType} returnValue = "; builder.AddLine($"{variableDeclaration}{Question.MethodName}({methodTest.ParameterValues});"); }
/// <summary> /// Returns any errors with the submission's definition (classes/methods). /// </summary> protected override IEnumerable <CodeQuestionError> GetDefinitionErrors( CodeQuestionSubmission submission, CodeJobResult jobResult) { var methodJobResult = jobResult as MethodJobResult; if (methodJobResult == null) { throw new ArgumentException("Invalid job result type", nameof(jobResult)); } var submittedMethod = methodJobResult.MethodDefinition; if (jobResult.ClassCompilationResult.Success && submittedMethod == null) { yield return(new MethodMissingError(null /*className*/, Question.MethodName, expectedStatic: true)); yield break; } if (submittedMethod.Name != Question.MethodName) { yield return(new MethodNameError(Question.MethodName, submittedMethod.Name)); } if (!submittedMethod.IsPublic) { yield return(new MethodVisibilityError(Question.MethodName, expectedPublic: true)); } if (!submittedMethod.IsStatic) { yield return(new MethodStaticError(Question.MethodName, expectedStatic: true)); } if (submittedMethod.ReturnType != Question.ReturnType) { yield return(new MethodReturnTypeError(Question.MethodName, Question.ReturnType, submittedMethod.ReturnType)); } var expectedParamTypes = Question.ParameterTypes .Split(',') .Select(paramType => paramType.Trim()) .ToList(); if (!expectedParamTypes.SequenceEqual(submittedMethod.ParameterTypes)) { yield return(new MethodParameterTypesError(Question.MethodName, expectedParamTypes, submittedMethod.ParameterTypes)); } }
public async Task GradeSubmissionAsync_TestCompilationFailed_Error() { var question = GetCodeQuestion(); var submission = new CodeQuestionSubmission() { Contents = "A A B B C C" }; var simulatedResult = new MethodJobResult() { Status = CodeJobStatus.Completed, ClassCompilationResult = new CompilationResult() { Success = true }, TestsCompilationResult = new CompilationResult() { Success = false, Errors = Collections.CreateList ( new CompileError() { Message = "ShortError1", FullError = "FullError1" }, new CompileError() { Message = "ShortError2", FullError = "FullError2" } ) } }; var grader = GetCodeQuestionGrader(question, submission, simulatedResult); var result = await grader.GradeSubmissionAsync(submission); var codeQuestionResult = (CodeQuestionResult)result.Result; var testsCompilationErrors = codeQuestionResult.Errors .Cast <TestCompilationError>() .ToList(); Assert.Equal(0.0, result.Score); Assert.Equal(2, testsCompilationErrors.Count); Assert.Null(testsCompilationErrors[0].LineNumber); Assert.Null(testsCompilationErrors[0].LineErrorText); Assert.Contains("FullError1", testsCompilationErrors[0].FullErrorText); Assert.Null(testsCompilationErrors[1].LineNumber); Assert.Null(testsCompilationErrors[1].LineErrorText); Assert.Contains("FullError2", testsCompilationErrors[1].FullErrorText); }
/// <summary> /// Adds a class to a java file to run in the visualizer, for the given test. /// </summary> protected override void AddVisualizerClass( JavaFileBuilder builder, CodeQuestionTest test, CodeQuestionSubmission submission) { var classTest = test as ProgramQuestionTest; if (test == null) { throw new ArgumentException("Invalid test type", nameof(test)); } builder.AddLines(submission.Contents); }
/// <summary> /// Returns the contents of the java file to run with the java visualizer, for the given test. /// </summary> private string GetVisualizerFileContents(CodeQuestionTest test, CodeQuestionSubmission submission) { JavaFileBuilder builder = new JavaFileBuilder(); if (Question.ImportedClasses != null) { builder.AddLines(Question.ImportedClasses.Select(importedClass => $"import {importedClass.ClassName};")); } builder.AddBlankLine(); AddVisualizerClass(builder, test, submission); return(builder.GetFileContents()); }
/// <summary> /// Returns the first non-empty set computed by the given compute functions. /// </summary> private IEnumerable <CodeQuestionError> GetFirstNonEmptyErrorSet( CodeQuestionSubmission submission, CodeJobResult jobResult, ComputeErrorSet[] computeErrorSets) { foreach (var computeErrorSet in computeErrorSets) { var errors = computeErrorSet(submission, jobResult).ToList(); if (errors.Any()) { return(errors.ToList()); } } return(Enumerable.Empty <CodeQuestionError>()); }
/// <summary> /// Adds a class to a java file to run in the visualizer, for the given test. /// </summary> protected override void AddVisualizerClass( JavaFileBuilder builder, CodeQuestionTest test, CodeQuestionSubmission submission) { var classTest = test as ClassQuestionTest; if (test == null) { throw new ArgumentException("Invalid test type", nameof(test)); } var templateWithSubmission = GetFileTemplateWithSubmission(submission); var regex = new Regex("public\\s+class"); var modifiedSubmission = regex.Replace(templateWithSubmission, "class"); builder.AddLines(modifiedSubmission); builder.AddBlankLine(); builder.AddLine("public class Runner"); builder.BeginScope("{"); if (classTest.ReturnType != "void") { builder.AddLine($"public static {classTest.ReturnType} runTest()"); builder.BeginScope("{"); builder.AddLines(classTest.MethodBody); builder.EndScope("}"); } builder.AddBlankLine(); builder.AddLine("public static void main(String[] args)"); builder.BeginScope("{"); if (classTest.ReturnType != "void") { builder.AddLine($"{classTest.ReturnType} returnVal = runTest();"); } else { builder.AddLines(classTest.MethodBody); } builder.EndScope("}"); builder.EndScope("}"); }
/// <summary> /// Returns a list of errors for the job. /// </summary> private IEnumerable <CodeQuestionError> GetPostJobExecutionErrors( CodeQuestionSubmission submission, CodeJobResult jobResult) { return(GetFirstNonEmptyErrorSet ( submission, jobResult, new ComputeErrorSet[] { GetJobExecutionErrors, GetClassCompilationErrors, GetDefinitionErrors, GetTestCompilationErrors } )); }
public async Task GradeSubmissionAsync_TestFailsIncorrectReturnValue_AccurateResults() { var question = GetCodeQuestion(); var submission = new CodeQuestionSubmission() { Contents = "A A B B C C" }; var simulatedResult = new MethodJobResult() { Status = CodeJobStatus.Completed, ClassCompilationResult = new CompilationResult() { Success = true }, TestsCompilationResult = new CompilationResult() { Success = true }, TestResults = Collections.CreateList ( new CodeTestResult() { Name = "test1", Completed = true, ReturnValue = "incorrectReturnValue", Output = "expectedOutput" } ) }; var grader = GetCodeQuestionGrader(question, submission, simulatedResult); var result = await grader.GradeSubmissionAsync(submission); var codeQuestionResult = (CodeQuestionResult)result.Result; var testResults = codeQuestionResult.TestResults; Assert.Single(testResults); Assert.Equal("test1", testResults[0].Description); Assert.False(testResults[0].Succeeded); Assert.Null(testResults[0].ExceptionText); Assert.Equal("expectedReturnValue", testResults[0].ExpectedReturnValue); Assert.Equal("incorrectReturnValue", testResults[0].ActualReturnValue); Assert.Equal("expectedOutput", testResults[0].ExpectedOutput); Assert.Equal("expectedOutput", testResults[0].ActualOutput); }
public async Task GradeSubmissionAsync_TestsAllPass_FullCredit() { var question = GetCodeQuestion(numTests: 2); var submission = new CodeQuestionSubmission() { Contents = "A A B B C C" }; var simulatedResult = new MethodJobResult() { Status = CodeJobStatus.Completed, ClassCompilationResult = new CompilationResult() { Success = true }, TestsCompilationResult = new CompilationResult() { Success = true }, TestResults = Collections.CreateList ( new CodeTestResult() { Name = "test1", Completed = true, ReturnValue = "expectedReturnValue", Output = "expectedOutput" }, new CodeTestResult() { Name = "test2", Completed = true, ReturnValue = "expectedReturnValue", Output = "expectedOutput" } ) }; var grader = GetCodeQuestionGrader(question, submission, simulatedResult); var result = await grader.GradeSubmissionAsync(submission); var codeQuestionResult = (CodeQuestionResult)result.Result; Assert.Equal(1.0, result.Score); Assert.Empty(codeQuestionResult.Errors); }
public async Task GradeSubmissionAsync_CorrectSubmission_ValidTestDescription() { var question = GetClassQuestion(); var classJobResult = GetClassJobResult(success: true); var submission = new CodeQuestionSubmission() { Contents = "Submission" }; var codeRunnerService = GetCodeRunnerService(classJobResult); var grader = new ClassQuestionGrader(question, codeRunnerService); var result = await grader.GradeSubmissionAsync(submission); var codeQuestionResult = (CodeQuestionResult)result.Result; var testResult = codeQuestionResult.TestResults.Single(); Assert.Equal("Description", testResult.Description); }
public async Task GradeSubmissionAsync_CorrectSubmission_ValidTestDescription() { var question = GetMethodQuestion(); var submission = new CodeQuestionSubmission() { Contents = "Submission" }; var methodJobResult = GetSuccessfulMethodJobResult(); var codeRunnerService = GetCodeRunnerService(methodJobResult); var grader = new MethodQuestionGrader(question, codeRunnerService); var result = await grader.GradeSubmissionAsync(submission); var codeQuestionResult = (CodeQuestionResult)result.Result; var testResult = codeQuestionResult.TestResults.Single(); Assert.Equal("expectedMethod(1, 2)", testResult.Description); }
public async Task GradeSubmissionAsync_CorrectSubmission_CorrectScore(bool useGenerics) { var question = GetMethodQuestion(useGenerics); var submission = new CodeQuestionSubmission() { Contents = "Submission" }; var methodJobResult = GetSuccessfulMethodJobResult(useGenerics); var codeRunnerService = GetCodeRunnerService(methodJobResult); var grader = new MethodQuestionGrader(question, codeRunnerService); var result = await grader.GradeSubmissionAsync(submission); var codeQuestionResult = (CodeQuestionResult)result.Result; var testResult = codeQuestionResult.TestResults.Single(); Assert.Equal(1.0, result.Score); Assert.Empty(codeQuestionResult.Errors); }
public async Task GradeSubmissionAsync_NoSubmission_Error() { var question = GetCodeQuestion(); var submission = new CodeQuestionSubmission() { Contents = "" }; var grader = GetCodeQuestionGrader(question, submission, simulatedResult: null); var result = await grader.GradeSubmissionAsync(submission); var codeQuestionResult = (CodeQuestionResult)result.Result; var noSubmissionErrors = codeQuestionResult.Errors .Cast <NoSubmissionError>() .ToList(); Assert.Equal(0.0, result.Score); Assert.Single(noSubmissionErrors); }
public async Task GradeSubmissionAsync_ClassJobHasCorrectLineOffset() { var question = GetProgramQuestion(); var classJobResult = GetClassJobResult(success: true); var submission = new CodeQuestionSubmission() { Contents = "Submission" }; var codeRunnerService = GetCodeRunnerService ( classJobResult, job => job.LineNumberOffset == 0 ); var grader = new ProgramQuestionGrader(question, codeRunnerService); var result = await grader.GradeSubmissionAsync(submission); Assert.Equal(1.0, result.Score); }
public async Task GradeSubmissionAsync_MethodJobHasCorrectCode() { var question = GetMethodQuestion(); var methodJobResult = GetSuccessfulMethodJobResult(); var submission = new CodeQuestionSubmission() { Contents = "Submission %" }; var codeRunnerService = GetCodeRunnerService ( methodJobResult, job => job.MethodCode == "Submission %%" ); var grader = new MethodQuestionGrader(question, codeRunnerService); var result = await grader.GradeSubmissionAsync(submission); Assert.Equal(1.0, result.Score); }
public async Task GradeSubmissionAsync_ClassJobHasCorrectClassName() { var question = GetClassQuestion(); var classJobResult = GetClassJobResult(success: true); var submission = new CodeQuestionSubmission() { Contents = "Submission" }; var codeRunnerService = GetCodeRunnerService ( classJobResult, job => job.ClassName == "ExpectedClass" ); var grader = new ClassQuestionGrader(question, codeRunnerService); var result = await grader.GradeSubmissionAsync(submission); Assert.Equal(1.0, result.Score); }
/// <summary> /// Returns any errors that prevented the job from executing. /// </summary> private IEnumerable <CodeQuestionError> GetConstraintErrors( CodeQuestionSubmission submission) { if (Question.CodeConstraints == null) { yield break; } foreach (var constraint in Question.CodeConstraints) { int count = Regex.Matches(submission.Contents, constraint.Regex).Count; bool constraintMet; switch (constraint.Type) { case CodeConstraintType.AtLeast: constraintMet = count >= constraint.Frequency; break; case CodeConstraintType.Exactly: constraintMet = count == constraint.Frequency; break; case CodeConstraintType.AtMost: constraintMet = count <= constraint.Frequency; break; default: throw new InvalidOperationException("Invalid code constraint type."); } if (!constraintMet) { yield return(new CodeConstraintError ( constraint.Regex, count, constraint.ErrorMessage )); } } }
public async Task GradeSubmissionAsync_ClassJobHasImportedClasses() { var question = GetClassQuestion(); var classJobResult = GetClassJobResult(success: true); var submission = new CodeQuestionSubmission() { Contents = "Submission" }; var codeRunnerService = GetCodeRunnerService ( classJobResult, job => job.ClassesToImport.Count == 1 && job.ClassesToImport[0] == "package.classToImport" ); var grader = new ClassQuestionGrader(question, codeRunnerService); var result = await grader.GradeSubmissionAsync(submission); Assert.Equal(1.0, result.Score); }
public async Task GradeSubmissionAsync_MethodJobHasTests() { var question = GetMethodQuestion(); var methodJobResult = GetSuccessfulMethodJobResult(); var submission = new CodeQuestionSubmission() { Contents = "Submission %" }; var codeRunnerService = GetCodeRunnerService ( methodJobResult, job => job.Tests.Count == 1 && job.Tests[0].TestName == "test1" && job.Tests[0].ParamValues == "1, 2" ); var grader = new MethodQuestionGrader(question, codeRunnerService); var result = await grader.GradeSubmissionAsync(submission); Assert.Equal(1.0, result.Score); }
public async Task GradeSubmissionAsync_MissingMethod_Error() { var question = GetMethodQuestion(); var submission = new CodeQuestionSubmission() { Contents = "Submission" }; var methodJobResult = GetFailedMethodJobResult(definition: null); var codeRunnerService = GetCodeRunnerService(methodJobResult); var grader = new MethodQuestionGrader(question, codeRunnerService); var result = await grader.GradeSubmissionAsync(submission); var codeQuestionResult = (CodeQuestionResult)result.Result; var methodMissingError = codeQuestionResult.Errors .Cast <MethodMissingError>() .Single(); Assert.Equal(0.0, result.Score); Assert.Equal("expectedMethod", methodMissingError.ExpectedMethodName); Assert.True(methodMissingError.ExpectedStatic); }
public async Task GradeSubmissionAsync_ClassJobHasTests() { var question = GetClassQuestion(); var classJobResult = GetClassJobResult(success: true); var submission = new CodeQuestionSubmission() { Contents = "Submission %" }; var codeRunnerService = GetCodeRunnerService ( classJobResult, job => job.Tests.Count == 1 && job.Tests[0].TestName == "test1" && job.Tests[0].MethodBody == "Method Body" && job.Tests[0].ReturnType == "String" ); var grader = new ClassQuestionGrader(question, codeRunnerService); var result = await grader.GradeSubmissionAsync(submission); Assert.Equal(1.0, result.Score); }
/// <summary> /// Adds a class to a java file to run in the visualizer, for the given test. /// </summary> protected override void AddVisualizerClass( JavaFileBuilder builder, CodeQuestionTest test, CodeQuestionSubmission submission) { var methodTest = test as MethodQuestionTest; if (test == null) { throw new ArgumentException("Invalid test type", nameof(test)); } builder.AddLine("public class Exercise"); builder.BeginScope("{"); builder.AddLines(submission.Contents); builder.AddBlankLine(); builder.AddLine("public static void main(String[] args)"); builder.BeginScope("{"); AddVisualizerMainMethodBody(builder, methodTest, submission); builder.EndScope("}"); builder.EndScope("}"); }
/// <summary> /// Returns a code question grader. For a given question, the grader /// returns a given simulated result for a given submission. /// </summary> public CodeQuestionGrader <CodeQuestion> GetCodeQuestionGrader( CodeQuestion question, CodeQuestionSubmission submission, CodeJobResult simulatedResult, IList <DefinitionError> definitionErrors = null) { var grader = new Mock <CodeQuestionGrader <CodeQuestion> > ( question, null /*codeRunnerService*/ ); grader.CallBase = true; grader.Protected() .Setup <Task <CodeJobResult> > ( "ExecuteJobAsync", ItExpr.Is <CodeQuestionSubmission>(s => s == submission) ).Returns(Task.FromResult(simulatedResult)); grader.Protected() .Setup <string>("GetTestDescription", ItExpr.IsAny <CodeQuestionTest>()) .Returns <CodeQuestionTest>(test => test.Name); if (definitionErrors != null) { grader.Protected() .Setup <IEnumerable <CodeQuestionError> > ( "GetDefinitionErrors", ItExpr.Is <CodeQuestionSubmission>(s => s == submission), ItExpr.Is <CodeJobResult>(r => r == simulatedResult) ).Returns(definitionErrors); } return(grader.Object); }
public async Task GradeSubmissionAsync_ForbiddenPublicFields_Error() { var question = GetClassQuestion(allowPublicFields: false); var submission = new CodeQuestionSubmission() { Contents = "Submission" }; var classJobResult = GetClassJobResult(success: false); classJobResult.ClassDefinition.Fields = new List <FieldDefinition>() { new FieldDefinition() { Name = "field1", IsPublic = true, Type = "String" }, new FieldDefinition() { Name = "field2", IsPublic = false, Type = "String" } }; var codeRunnerService = GetCodeRunnerService(classJobResult); var grader = new ClassQuestionGrader(question, codeRunnerService); var result = await grader.GradeSubmissionAsync(submission); var codeQuestionResult = (CodeQuestionResult)result.Result; var fieldVisibilityError = codeQuestionResult.Errors .Cast <FieldVisibilityError>() .Single(); Assert.Equal(0.0, result.Score); Assert.Equal("ExpectedClass", fieldVisibilityError.ClassName); }
/// <summary> /// Returns any errors that prevented the job from executing. /// </summary> private IEnumerable <CodeQuestionError> GetJobExecutionErrors( CodeQuestionSubmission submission, CodeJobResult jobResult) { switch (jobResult.Status) { case CodeJobStatus.Timeout: yield return(new TimeoutError()); break; case CodeJobStatus.Error: yield return(new DiagnosticError(jobResult.DiagnosticOutput)); break; case CodeJobStatus.Completed: yield break; default: throw new InvalidOperationException("Unuspported code job status."); } }
/// <summary> /// Executes a code job to grade the question. /// </summary> protected override async Task <CodeJobResult> ExecuteJobAsync(CodeQuestionSubmission submission) { return(await CodeRunnerService.ExecuteMethodJobAsync ( new MethodJob() { ClassesToImport = Question.ImportedClasses ?.Select(importedClass => importedClass.ClassName) ?.ToList() ?? new List <string>(), MethodCode = submission.Contents.Replace("%", "%%"), Tests = Question.Tests.Select ( test => new MethodTest() { TestName = GetCodeTestName(test), ParamValues = test.ParameterValues } ).ToList() } )); }