public void EthalonSolutions_for_Exercises(ExerciseSlide slide)
{
var solution = slide.Exercise.Solution.BuildSolution(slide.Exercise.EthalonSolution);
if (solution.HasErrors)
{
FailOnError(slide, solution);
}
else
{
var submission = new RunnerSubmition()
{
Code = solution.SourceCode,
Id = slide.Id,
Input = "",
NeedRun = true
};
var result = SandboxRunner.Run(submission);
var output = result.GetOutput().NormalizeEoln();
var isRightAnswer = output.NormalizeEoln().Equals(slide.Exercise.ExpectedOutput.NormalizeEoln());
if (!isRightAnswer)
{
Assert.Fail("mistake in: " + slide.Info.UnitName + " - " + slide.Title + "\n" +
"\tActualOutput: " + output + "\n" +
"\tExpectedOutput: " + slide.Exercise.ExpectedOutput.NormalizeEoln() + "\n" +
"\tCompilationError: " + result.CompilationOutput + "\n" +
"\tSourceCode: " + solution.SourceCode + "\n\n");
}
}
}
private static RunSolutionResult GetRunResult(ExerciseBlock exercise, string code)
{
var buildResult = exercise.BuildSolution(code);
if (buildResult.HasErrors)
{
return new RunSolutionResult {
IsCompileError = true, ErrorMessage = buildResult.ErrorMessage, ExecutionServiceName = "uLearn"
}
}
;
var result = SandboxRunner.Run(exercise.CreateSubmission(Utils.NewNormalizedGuid(), code));
var runSolutionResult = new RunSolutionResult
{
IsRightAnswer = result.Verdict == Verdict.Ok && result.GetOutput().NormalizeEoln() == exercise.ExpectedOutput.NormalizeEoln(),
ActualOutput = result.GetOutput().NormalizeEoln(),
ErrorMessage = result.CompilationOutput,
ExecutionServiceName = "this",
IsCompileError = result.Verdict == Verdict.CompilationError,
ExpectedOutput = exercise.ExpectedOutput.NormalizeEoln(),
SubmissionId = 0,
};
if (buildResult.HasStyleErrors)
{
runSolutionResult.IsStyleViolation = true;
runSolutionResult.StyleMessage = string.Join("\n", buildResult.StyleErrors.Select(e => e.GetMessageWithPositions()));
}
return(runSolutionResult);
}
private static void InitialCodeIsNotSolutionForProjExercise(ExerciseSlide slide)
{
var exercise = slide.Exercise as ProjectExerciseBlock;
var directoryName = Path.Combine(exercise.SlideFolderPath.FullName, exercise.ExerciseDir);
var excluded = (exercise.PathsToExcludeForChecker ?? new string[0]).Concat(new[] { "bin/*", "obj/*" }).ToList();
var exerciseDir = new DirectoryInfo(directoryName);
var bytes = exerciseDir.ToZip(excluded, new[]
{
new FileContent
{
Path = exercise.CsprojFileName,
Data = ProjModifier.ModifyCsproj(exerciseDir.GetFile(exercise.CsprojFileName),
proj => ProjModifier.PrepareForChecking(proj, exercise, excluded))
}
});
var result = SandboxRunner.Run(new ProjRunnerSubmission
{
Id = slide.Id.ToString(),
ZipFileData = bytes,
ProjectFileName = exercise.CsprojFileName,
Input = "",
NeedRun = true
});
Console.WriteLine("Result = " + result);
Assert.AreEqual(Verdict.Ok, result.Verdict);
Assert.AreNotEqual("", result.Output);
}
private static void EthalonSolutionForSingleFileExercises(ExerciseSlide slide)
{
var exercise = (SingleFileExerciseBlock)slide.Exercise;
var solution = exercise.BuildSolution(exercise.EthalonSolution);
if (solution.HasErrors)
{
FailOnError(slide, solution, exercise.EthalonSolution);
return;
}
var result = SandboxRunner.Run(exercise.CreateSubmition(
slide.Id.ToString(),
exercise.EthalonSolution), new SandboxRunnerSettings());
var output = result.GetOutput().NormalizeEoln();
var isRightAnswer = output.NormalizeEoln().Equals(slide.Exercise.ExpectedOutput.NormalizeEoln());
if (!isRightAnswer)
{
Assert.Fail("mistake in: " + slide.Info.Unit.Title + " - " + slide.Title + "\n" +
"\tActualOutput: " + output.NormalizeEoln() + "\n" +
"\tExpectedOutput: " + slide.Exercise.ExpectedOutput.NormalizeEoln() + "\n" +
"\tCompilationError: " + result.CompilationOutput + "\n" +
"\tSourceCode: " + solution.SourceCode + "\n\n");
}
}
private void EthalonSolutionForSingleFileExercises(ExerciseSlide slide)
{
var exercise = (SingleFileExerciseBlock)slide.Exercise;
var solution = exercise.BuildSolution(exercise.EthalonSolution);
if (solution.HasErrors)
{
FailOnError(slide, solution, exercise.EthalonSolution);
return;
}
if (solution.HasStyleIssues)
{
Console.WriteLine("Style issue: " + solution.StyleMessage);
}
var result = SandboxRunner.Run(exercise.CreateSubmition(
slide.Id.ToString(),
exercise.EthalonSolution), settings);
var output = result.GetOutput().NormalizeEoln();
var isRightAnswer = output.NormalizeEoln().Equals(slide.Exercise.ExpectedOutput.NormalizeEoln());
if (!isRightAnswer)
{
ReportSlideError(slide,
"ActualOutput: " + output.NormalizeEoln() + "\n" +
"ExpectedOutput: " + slide.Exercise.ExpectedOutput.NormalizeEoln() + "\n" +
"CompilationError: " + result.CompilationOutput + "\n" +
"SourceCode: " + solution.SourceCode + "\n\n");
}
}
private void ReportIfEthalonSolutionHasErrorsOrIssues(ExerciseSlide slide)
{
var exercise = (SingleFileExerciseBlock)slide.Exercise;
var ethalon = exercise.EthalonSolution.RemoveCommonNesting();
var solution = exercise.BuildSolution(ethalon);
if (solution.HasErrors)
{
FailOnError(slide, solution, ethalon);
return;
}
if (solution.HasStyleErrors)
{
var errorMessages = string.Join("\n", solution.StyleErrors.Select(e => e.GetMessageWithPositions()));
ReportSlideWarning(slide, "Style issue(s): " + errorMessages);
}
var result = SandboxRunner.Run(exercise.CreateSubmission(slide.Id.ToString(), ethalon), settings);
var output = result.GetOutput().NormalizeEoln();
var isRightAnswer = output.NormalizeEoln().Equals(slide.Exercise.ExpectedOutput.NormalizeEoln());
if (!isRightAnswer)
{
ReportSlideError(slide,
"Ethalon solution does not provide right answer\n" +
"ActualOutput: " + output.NormalizeEoln() + "\n" +
"ExpectedOutput: " + slide.Exercise.ExpectedOutput.NormalizeEoln() + "\n" +
"CompilationError: " + result.CompilationOutput + "\n" +
"SourceCode: " + solution.SourceCode + "\n\n");
}
}
private void ReportErrorIfSolutionNotBuildingOrNotPassesTests()
{
var solutionCode = ex.CorrectSolutionFile.ContentAsUtf8();
var submission = ex.CreateSubmission(ex.CsprojFileName, solutionCode);
var result = SandboxRunner.Run(submission, new SandboxRunnerSettings());
if (!IsCompiledAndExecuted(result))
{
ReportSlideError(slide, $"Correct solution file {ex.CorrectSolutionFileName} verdict is not OK. RunResult = {result}");
}
if (!ex.IsCorrectRunResult(result))
{
ReportSlideError(slide, $"Correct solution file {ex.CorrectSolutionFileName} is not solution. RunResult = {result}. " +
$"ExpectedOutput = {ex.ExpectedOutput.NormalizeEoln()} " +
$"RealOutput = {result.GetOutput().NormalizeEoln()}");
}
var buildResult = ex.BuildSolution(solutionCode);
if (buildResult.HasStyleErrors)
{
var errorMessage = string.Join("\n", buildResult.StyleErrors.Select(e => e.GetMessageWithPositions()));
ReportSlideWarning(slide, $"Correct solution file {ex.CorrectSolutionFileName} has style issues. {errorMessage}");
}
}
private void InitialCodeIsNotSolutionForProjExercise(ExerciseSlide slide)
{
var exercise = (ProjectExerciseBlock)slide.Exercise;
var directoryName = Path.Combine(exercise.SlideFolderPath.FullName, exercise.ExerciseDir);
var excluded = (exercise.PathsToExcludeForChecker ?? new string[0]).Concat(new[] { "bin/*", "obj/*" }).ToList();
var exerciseDir = new DirectoryInfo(directoryName);
var bytes = exerciseDir.ToZip(excluded, new[]
{
new FileContent
{
Path = exercise.CsprojFileName,
Data = ProjModifier.ModifyCsproj(exerciseDir.GetFile(exercise.CsprojFileName),
proj => ProjModifier.PrepareForChecking(proj, exercise, excluded))
}
});
var result = SandboxRunner.Run(new ProjRunnerSubmission
{
Id = slide.Id.ToString(),
ZipFileData = bytes,
ProjectFileName = exercise.CsprojFileName,
Input = "",
NeedRun = true
});
var isOk = result.Verdict.IsOneOf(Verdict.Ok, Verdict.MemoryLimit, Verdict.TimeLimit);
if (!isOk)
{
ReportSlideError(slide, "Exercise initial code verdict is not OK. RunResult = " + result);
}
else if (result.Verdict == Verdict.Ok && result.Output == "")
{
ReportSlideError(slide, "Exercise initial code (available to students) is solution!");
}
}
private void ReportErrorIfInitialCodeIsSolutionOrVerdictNotOk()
{
var initialCode = ex.UserCodeFile.ContentAsUtf8();
var submission = ex.CreateSubmission(ex.CsprojFileName, initialCode);
var result = SandboxRunner.Run(submission);
if (ex.StudentZipIsCompilable)
{
ReportErrorIfInitialCodeVerdictIsNotOk(result);
}
ReportErrorIfInitialCodeIsSolution(result);
}
private void ReportWarningIfWrongAnswersAreSolutionsOrNotOk()
{
var filesWithWrongAnswer = FileSystem.GetFiles(ex.ExerciseFolder.FullName, SearchOption.SearchAllSubDirectories)
.Select(name => new FileInfo(name))
.Where(f => ex.IsWrongAnswer(f.Name));
foreach (var waFile in filesWithWrongAnswer)
{
var result = SandboxRunner.Run(ex.CreateSubmission(waFile.Name, waFile.ContentAsUtf8()));
ReportWarningIfWrongAnswerVerdictIsNotOk(waFile.Name, result);
ReportWarningIfWrongAnswerIsSolution(waFile.Name, result);
}
}
public void When_CreateStudentZip_Make_Project_Able_To_Compile_If_Project_Depends_On_Many_Tasks()
{
var submission = new ProjRunnerSubmission
{
Id = "my_id",
Input = "",
NeedRun = true,
ProjectFileName = "test.csproj",
ZipFileData = studentExerciseZipFilePath.Content()
};
var result = SandboxRunner.Run(submission);
result.CompilationOutput.Should().Be("");
result.Error.Should().Be("");
result.Verdict.Should().Be(Verdict.Ok);
}
private static RunSolutionResult GetRunResult(ExerciseBlock exercise, string code)
{
var buildResult = exercise.Solution.BuildSolution(code);
if (buildResult.HasErrors)
{
return new RunSolutionResult {
IsCompileError = true, CompilationError = buildResult.ErrorMessage, ExecutionServiceName = "uLearn"
}
}
;
if (buildResult.HasStyleIssues)
{
return new RunSolutionResult {
IsStyleViolation = true, CompilationError = buildResult.StyleMessage, ExecutionServiceName = "uLearn"
}
}
;
var solution = buildResult.SourceCode;
var submission = new RunnerSubmition
{
Code = solution,
Id = Utils.NewNormalizedGuid(),
Input = "",
NeedRun = true
};
var result = SandboxRunner.Run(submission);
return(new RunSolutionResult
{
IsRightAnswer = result.Verdict == Verdict.Ok && result.GetOutput().NormalizeEoln() == exercise.ExpectedOutput.NormalizeEoln(),
ActualOutput = result.GetOutput().NormalizeEoln(),
CompilationError = result.CompilationOutput,
ExecutionServiceName = "this",
IsCompileError = result.Verdict == Verdict.CompilationError,
ExpectedOutput = exercise.ExpectedOutput.NormalizeEoln()
});
}
private void ReportErrorIfSolutionNotBuildingOrNotPassesTests()
{
var solutionCode = ex.CorrectSolutionFile.ContentAsUtf8();
var submission = ex.CreateSubmission(ex.CsprojFileName, solutionCode);
var result = SandboxRunner.Run(submission, new SandboxRunnerSettings());
if (VerdictIsNotOk(result))
{
ReportSlideError(slide, $"Correct solution file {ex.CorrectSolutionFileName} verdict is not OK. RunResult = {result}");
}
if (!IsSolution(result))
{
ReportSlideError(slide, $"Correct solution file {ex.CorrectSolutionFileName} is not solution. RunResult = {result}");
}
var buildResult = ex.BuildSolution(solutionCode);
if (buildResult.HasStyleErrors)
{
var errorMessage = string.Join("\n", buildResult.StyleErrors.Select(e => e.GetMessageWithPositions()));
ReportSlideWarning(slide, $"Correct solution file {ex.CorrectSolutionFileName} has style issues. {errorMessage}");
}
}