/// <summary>
/// Returns a list of test results for the job.
/// </summary>
private IEnumerable <CodeQuestionTestResult> GetTestResults(
CodeJobResult jobResult,
CodeQuestionSubmission submission)
{
return(jobResult.TestResults?.Select
(
testResult => new
{
Test = Question.GetTests().Single
(
test => GetCodeTestName(test) == testResult.Name
),
TestResult = testResult
}
)
?.OrderBy
(
result => result.Test.Order
)
?.Select
(
result => new CodeQuestionTestResult
(
GetTestDescription(result.Test),
result.Test.ExpectedOutput?.Replace("\r\n", "\n"),
result.TestResult.Output?.Replace("\r\n", "\n"),
result.Test.ExpectedReturnValue,
result.TestResult.ReturnValue,
result.TestResult.Exception,
GetTestVisualizeUrl(result.Test, submission),
GetTestSucceeded(result.Test, result.TestResult)
)
) ?? Enumerable.Empty <CodeQuestionTestResult>());
}
/// <summary>
/// Returns any errors with the submission's definition (classes/methods).
/// </summary>
protected override IEnumerable <CodeQuestionError> GetDefinitionErrors(
CodeQuestionSubmission submission,
CodeJobResult jobResult)
{
var classJobResult = jobResult as ClassJobResult;
if (classJobResult == null)
{
throw new ArgumentException("Invalid job result type", nameof(jobResult));
}
var submittedClass = classJobResult.ClassDefinition;
if (submittedClass == null)
{
yield break;
}
var hasMainMethod = submittedClass.Methods.Any
(
method =>
method.IsPublic &&
method.IsStatic &&
method.Name == "main" &&
method.ParameterTypes.Count == 1 &&
method.ParameterTypes[0] == "String[]" &&
method.ReturnType == "void"
);
if (!hasMainMethod)
{
yield return(new MainMethodMissingError(Question.ProgramClassName));
}
}
/// <summary>
/// Return any compilation errors from the code job's class.
/// </summary>
private IEnumerable <CodeQuestionError> GetClassCompilationErrors(
CodeQuestionSubmission submission,
CodeJobResult jobResult)
{
if (jobResult.ClassCompilationResult == null ||
(!jobResult.ClassCompilationResult.Success &&
jobResult.ClassCompilationResult.Errors.Count == 0))
{
return(new List <CodeQuestionError>()
{
new NoClassError()
});
}
return(jobResult.ClassCompilationResult?.Errors?.Select
(
compilationError => GetFriendlyError
(
new ClassCompilationError
(
compilationError.LineNumber,
compilationError.Message,
compilationError.FullError
)
)
) ?? Enumerable.Empty <CodeQuestionError>());
}
/// <summary>
/// Return any compilation errors from the code job's tests.
/// </summary>
private IEnumerable <CodeQuestionError> GetTestCompilationErrors(
CodeQuestionSubmission submission,
CodeJobResult jobResult)
{
return(jobResult.TestsCompilationResult?.Errors?.Select
(
compilationError => new TestCompilationError(compilationError.FullError)
) ?? Enumerable.Empty <CodeQuestionError>());
}
/// <summary>
/// Returns any errors with the submission's definition (classes/methods).
/// </summary>
protected override IEnumerable <CodeQuestionError> GetDefinitionErrors(
CodeQuestionSubmission submission,
CodeJobResult jobResult)
{
var classJobResult = jobResult as ClassJobResult;
if (classJobResult == null)
{
throw new ArgumentException("Invalid job result type", nameof(jobResult));
}
var submittedClass = classJobResult.ClassDefinition;
if (submittedClass == null)
{
yield break;
}
if (!Question.AllowPublicFields && submittedClass.Fields.Any(field => field.IsPublic))
{
yield return(new FieldVisibilityError(Question.ClassName));
}
if (Question.RequiredMethods != null)
{
foreach (var requiredMethodGroup in Question.RequiredMethods.GroupBy(rm => rm.Name))
{
IEnumerable <CodeQuestionError> methodDefinitionErrors;
if (requiredMethodGroup.Count() == 1)
{
var requiredMethod = requiredMethodGroup.First();
methodDefinitionErrors = GetSingleMethodDefinitionErrors
(
requiredMethod,
submittedClass
);
}
else
{
methodDefinitionErrors = GetOverloadedMethodDefinitionErrors
(
requiredMethodGroup,
submittedClass
);
}
foreach (var error in methodDefinitionErrors)
{
yield return(error);
}
}
}
}
/// <summary>
/// Returns any errors with the submission's definition (classes/methods).
/// </summary>
protected override IEnumerable <CodeQuestionError> GetDefinitionErrors(
CodeQuestionSubmission submission,
CodeJobResult jobResult)
{
var methodJobResult = jobResult as MethodJobResult;
if (methodJobResult == null)
{
throw new ArgumentException("Invalid job result type", nameof(jobResult));
}
var submittedMethod = methodJobResult.MethodDefinition;
if (jobResult.ClassCompilationResult.Success && submittedMethod == null)
{
yield return(new MethodMissingError(null /*className*/, Question.MethodName, expectedStatic: true));
yield break;
}
if (submittedMethod.Name != Question.MethodName)
{
yield return(new MethodNameError(Question.MethodName, submittedMethod.Name));
}
if (!submittedMethod.IsPublic)
{
yield return(new MethodVisibilityError(Question.MethodName, expectedPublic: true));
}
if (!submittedMethod.IsStatic)
{
yield return(new MethodStaticError(Question.MethodName, expectedStatic: true));
}
if (submittedMethod.ReturnType != Question.ReturnType)
{
yield return(new MethodReturnTypeError(Question.MethodName, Question.ReturnType, submittedMethod.ReturnType));
}
var expectedParamTypes = Question.ParameterTypes
.Split(',')
.Select(paramType => paramType.Trim())
.ToList();
if (!expectedParamTypes.SequenceEqual(submittedMethod.ParameterTypes))
{
yield return(new MethodParameterTypesError(Question.MethodName, expectedParamTypes, submittedMethod.ParameterTypes));
}
}
/// <summary>
/// Returns the first non-empty set computed by the given compute functions.
/// </summary>
private IEnumerable <CodeQuestionError> GetFirstNonEmptyErrorSet(
CodeQuestionSubmission submission,
CodeJobResult jobResult,
ComputeErrorSet[] computeErrorSets)
{
foreach (var computeErrorSet in computeErrorSets)
{
var errors = computeErrorSet(submission, jobResult).ToList();
if (errors.Any())
{
return(errors.ToList());
}
}
return(Enumerable.Empty <CodeQuestionError>());
}
/// <summary>
/// Returns a list of errors for the job.
/// </summary>
private IEnumerable <CodeQuestionError> GetPostJobExecutionErrors(
CodeQuestionSubmission submission,
CodeJobResult jobResult)
{
return(GetFirstNonEmptyErrorSet
(
submission,
jobResult,
new ComputeErrorSet[]
{
GetJobExecutionErrors,
GetClassCompilationErrors,
GetDefinitionErrors,
GetTestCompilationErrors
}
));
}
/// <summary>
/// Returns a code question grader. For a given question, the grader
/// returns a given simulated result for a given submission.
/// </summary>
public CodeQuestionGrader <CodeQuestion> GetCodeQuestionGrader(
CodeQuestion question,
CodeQuestionSubmission submission,
CodeJobResult simulatedResult,
IList <DefinitionError> definitionErrors = null)
{
var grader = new Mock <CodeQuestionGrader <CodeQuestion> >
(
question,
null /*codeRunnerService*/
);
grader.CallBase = true;
grader.Protected()
.Setup <Task <CodeJobResult> >
(
"ExecuteJobAsync",
ItExpr.Is <CodeQuestionSubmission>(s => s == submission)
).Returns(Task.FromResult(simulatedResult));
grader.Protected()
.Setup <string>("GetTestDescription", ItExpr.IsAny <CodeQuestionTest>())
.Returns <CodeQuestionTest>(test => test.Name);
if (definitionErrors != null)
{
grader.Protected()
.Setup <IEnumerable <CodeQuestionError> >
(
"GetDefinitionErrors",
ItExpr.Is <CodeQuestionSubmission>(s => s == submission),
ItExpr.Is <CodeJobResult>(r => r == simulatedResult)
).Returns(definitionErrors);
}
return(grader.Object);
}
/// <summary>
/// Returns any errors that prevented the job from executing.
/// </summary>
private IEnumerable <CodeQuestionError> GetJobExecutionErrors(
CodeQuestionSubmission submission,
CodeJobResult jobResult)
{
switch (jobResult.Status)
{
case CodeJobStatus.Timeout:
yield return(new TimeoutError());
break;
case CodeJobStatus.Error:
yield return(new DiagnosticError(jobResult.DiagnosticOutput));
break;
case CodeJobStatus.Completed:
yield break;
default:
throw new InvalidOperationException("Unuspported code job status.");
}
}
/// <summary>
/// Returns any errors with the submission's definition (classes/methods/code).
/// </summary>
protected abstract IEnumerable <CodeQuestionError> GetDefinitionErrors(
CodeQuestionSubmission submission,
CodeJobResult jobResult);
