/// <summary> /// Returns a list of test results for the job. /// </summary> private IEnumerable <CodeQuestionTestResult> GetTestResults( CodeJobResult jobResult, CodeQuestionSubmission submission) { return(jobResult.TestResults?.Select ( testResult => new { Test = Question.GetTests().Single ( test => GetCodeTestName(test) == testResult.Name ), TestResult = testResult } ) ?.OrderBy ( result => result.Test.Order ) ?.Select ( result => new CodeQuestionTestResult ( GetTestDescription(result.Test), result.Test.ExpectedOutput?.Replace("\r\n", "\n"), result.TestResult.Output?.Replace("\r\n", "\n"), result.Test.ExpectedReturnValue, result.TestResult.ReturnValue, result.TestResult.Exception, GetTestVisualizeUrl(result.Test, submission), GetTestSucceeded(result.Test, result.TestResult) ) ) ?? Enumerable.Empty <CodeQuestionTestResult>()); }
/// <summary> /// Returns any errors with the submission's definition (classes/methods). /// </summary> protected override IEnumerable <CodeQuestionError> GetDefinitionErrors( CodeQuestionSubmission submission, CodeJobResult jobResult) { var classJobResult = jobResult as ClassJobResult; if (classJobResult == null) { throw new ArgumentException("Invalid job result type", nameof(jobResult)); } var submittedClass = classJobResult.ClassDefinition; if (submittedClass == null) { yield break; } var hasMainMethod = submittedClass.Methods.Any ( method => method.IsPublic && method.IsStatic && method.Name == "main" && method.ParameterTypes.Count == 1 && method.ParameterTypes[0] == "String[]" && method.ReturnType == "void" ); if (!hasMainMethod) { yield return(new MainMethodMissingError(Question.ProgramClassName)); } }
/// <summary> /// Return any compilation errors from the code job's class. /// </summary> private IEnumerable <CodeQuestionError> GetClassCompilationErrors( CodeQuestionSubmission submission, CodeJobResult jobResult) { if (jobResult.ClassCompilationResult == null || (!jobResult.ClassCompilationResult.Success && jobResult.ClassCompilationResult.Errors.Count == 0)) { return(new List <CodeQuestionError>() { new NoClassError() }); } return(jobResult.ClassCompilationResult?.Errors?.Select ( compilationError => GetFriendlyError ( new ClassCompilationError ( compilationError.LineNumber, compilationError.Message, compilationError.FullError ) ) ) ?? Enumerable.Empty <CodeQuestionError>()); }
/// <summary> /// Return any compilation errors from the code job's tests. /// </summary> private IEnumerable <CodeQuestionError> GetTestCompilationErrors( CodeQuestionSubmission submission, CodeJobResult jobResult) { return(jobResult.TestsCompilationResult?.Errors?.Select ( compilationError => new TestCompilationError(compilationError.FullError) ) ?? Enumerable.Empty <CodeQuestionError>()); }
/// <summary> /// Returns any errors with the submission's definition (classes/methods). /// </summary> protected override IEnumerable <CodeQuestionError> GetDefinitionErrors( CodeQuestionSubmission submission, CodeJobResult jobResult) { var classJobResult = jobResult as ClassJobResult; if (classJobResult == null) { throw new ArgumentException("Invalid job result type", nameof(jobResult)); } var submittedClass = classJobResult.ClassDefinition; if (submittedClass == null) { yield break; } if (!Question.AllowPublicFields && submittedClass.Fields.Any(field => field.IsPublic)) { yield return(new FieldVisibilityError(Question.ClassName)); } if (Question.RequiredMethods != null) { foreach (var requiredMethodGroup in Question.RequiredMethods.GroupBy(rm => rm.Name)) { IEnumerable <CodeQuestionError> methodDefinitionErrors; if (requiredMethodGroup.Count() == 1) { var requiredMethod = requiredMethodGroup.First(); methodDefinitionErrors = GetSingleMethodDefinitionErrors ( requiredMethod, submittedClass ); } else { methodDefinitionErrors = GetOverloadedMethodDefinitionErrors ( requiredMethodGroup, submittedClass ); } foreach (var error in methodDefinitionErrors) { yield return(error); } } } }
/// <summary> /// Returns any errors with the submission's definition (classes/methods). /// </summary> protected override IEnumerable <CodeQuestionError> GetDefinitionErrors( CodeQuestionSubmission submission, CodeJobResult jobResult) { var methodJobResult = jobResult as MethodJobResult; if (methodJobResult == null) { throw new ArgumentException("Invalid job result type", nameof(jobResult)); } var submittedMethod = methodJobResult.MethodDefinition; if (jobResult.ClassCompilationResult.Success && submittedMethod == null) { yield return(new MethodMissingError(null /*className*/, Question.MethodName, expectedStatic: true)); yield break; } if (submittedMethod.Name != Question.MethodName) { yield return(new MethodNameError(Question.MethodName, submittedMethod.Name)); } if (!submittedMethod.IsPublic) { yield return(new MethodVisibilityError(Question.MethodName, expectedPublic: true)); } if (!submittedMethod.IsStatic) { yield return(new MethodStaticError(Question.MethodName, expectedStatic: true)); } if (submittedMethod.ReturnType != Question.ReturnType) { yield return(new MethodReturnTypeError(Question.MethodName, Question.ReturnType, submittedMethod.ReturnType)); } var expectedParamTypes = Question.ParameterTypes .Split(',') .Select(paramType => paramType.Trim()) .ToList(); if (!expectedParamTypes.SequenceEqual(submittedMethod.ParameterTypes)) { yield return(new MethodParameterTypesError(Question.MethodName, expectedParamTypes, submittedMethod.ParameterTypes)); } }
/// <summary> /// Returns the first non-empty set computed by the given compute functions. /// </summary> private IEnumerable <CodeQuestionError> GetFirstNonEmptyErrorSet( CodeQuestionSubmission submission, CodeJobResult jobResult, ComputeErrorSet[] computeErrorSets) { foreach (var computeErrorSet in computeErrorSets) { var errors = computeErrorSet(submission, jobResult).ToList(); if (errors.Any()) { return(errors.ToList()); } } return(Enumerable.Empty <CodeQuestionError>()); }
/// <summary> /// Returns a list of errors for the job. /// </summary> private IEnumerable <CodeQuestionError> GetPostJobExecutionErrors( CodeQuestionSubmission submission, CodeJobResult jobResult) { return(GetFirstNonEmptyErrorSet ( submission, jobResult, new ComputeErrorSet[] { GetJobExecutionErrors, GetClassCompilationErrors, GetDefinitionErrors, GetTestCompilationErrors } )); }
/// <summary> /// Returns a code question grader. For a given question, the grader /// returns a given simulated result for a given submission. /// </summary> public CodeQuestionGrader <CodeQuestion> GetCodeQuestionGrader( CodeQuestion question, CodeQuestionSubmission submission, CodeJobResult simulatedResult, IList <DefinitionError> definitionErrors = null) { var grader = new Mock <CodeQuestionGrader <CodeQuestion> > ( question, null /*codeRunnerService*/ ); grader.CallBase = true; grader.Protected() .Setup <Task <CodeJobResult> > ( "ExecuteJobAsync", ItExpr.Is <CodeQuestionSubmission>(s => s == submission) ).Returns(Task.FromResult(simulatedResult)); grader.Protected() .Setup <string>("GetTestDescription", ItExpr.IsAny <CodeQuestionTest>()) .Returns <CodeQuestionTest>(test => test.Name); if (definitionErrors != null) { grader.Protected() .Setup <IEnumerable <CodeQuestionError> > ( "GetDefinitionErrors", ItExpr.Is <CodeQuestionSubmission>(s => s == submission), ItExpr.Is <CodeJobResult>(r => r == simulatedResult) ).Returns(definitionErrors); } return(grader.Object); }
/// <summary> /// Returns any errors that prevented the job from executing. /// </summary> private IEnumerable <CodeQuestionError> GetJobExecutionErrors( CodeQuestionSubmission submission, CodeJobResult jobResult) { switch (jobResult.Status) { case CodeJobStatus.Timeout: yield return(new TimeoutError()); break; case CodeJobStatus.Error: yield return(new DiagnosticError(jobResult.DiagnosticOutput)); break; case CodeJobStatus.Completed: yield break; default: throw new InvalidOperationException("Unuspported code job status."); } }
/// <summary> /// Returns any errors with the submission's definition (classes/methods/code). /// </summary> protected abstract IEnumerable <CodeQuestionError> GetDefinitionErrors( CodeQuestionSubmission submission, CodeJobResult jobResult);