public void EthalonSolutions_for_Exercises(ExerciseSlide slide) { var solution = slide.Exercise.Solution.BuildSolution(slide.Exercise.EthalonSolution); if (solution.HasErrors) { FailOnError(slide, solution); } else { var submission = new RunnerSubmition() { Code = solution.SourceCode, Id = slide.Id, Input = "", NeedRun = true }; var result = SandboxRunner.Run(submission); var output = result.GetOutput().NormalizeEoln(); var isRightAnswer = output.NormalizeEoln().Equals(slide.Exercise.ExpectedOutput.NormalizeEoln()); if (!isRightAnswer) { Assert.Fail("mistake in: " + slide.Info.UnitName + " - " + slide.Title + "\n" + "\tActualOutput: " + output + "\n" + "\tExpectedOutput: " + slide.Exercise.ExpectedOutput.NormalizeEoln() + "\n" + "\tCompilationError: " + result.CompilationOutput + "\n" + "\tSourceCode: " + solution.SourceCode + "\n\n"); } } }
private static RunSolutionResult GetRunResult(ExerciseBlock exercise, string code) { var buildResult = exercise.BuildSolution(code); if (buildResult.HasErrors) { return new RunSolutionResult { IsCompileError = true, ErrorMessage = buildResult.ErrorMessage, ExecutionServiceName = "uLearn" } } ; var result = SandboxRunner.Run(exercise.CreateSubmission(Utils.NewNormalizedGuid(), code)); var runSolutionResult = new RunSolutionResult { IsRightAnswer = result.Verdict == Verdict.Ok && result.GetOutput().NormalizeEoln() == exercise.ExpectedOutput.NormalizeEoln(), ActualOutput = result.GetOutput().NormalizeEoln(), ErrorMessage = result.CompilationOutput, ExecutionServiceName = "this", IsCompileError = result.Verdict == Verdict.CompilationError, ExpectedOutput = exercise.ExpectedOutput.NormalizeEoln(), SubmissionId = 0, }; if (buildResult.HasStyleErrors) { runSolutionResult.IsStyleViolation = true; runSolutionResult.StyleMessage = string.Join("\n", buildResult.StyleErrors.Select(e => e.GetMessageWithPositions())); } return(runSolutionResult); }
private static void InitialCodeIsNotSolutionForProjExercise(ExerciseSlide slide) { var exercise = slide.Exercise as ProjectExerciseBlock; var directoryName = Path.Combine(exercise.SlideFolderPath.FullName, exercise.ExerciseDir); var excluded = (exercise.PathsToExcludeForChecker ?? new string[0]).Concat(new[] { "bin/*", "obj/*" }).ToList(); var exerciseDir = new DirectoryInfo(directoryName); var bytes = exerciseDir.ToZip(excluded, new[] { new FileContent { Path = exercise.CsprojFileName, Data = ProjModifier.ModifyCsproj(exerciseDir.GetFile(exercise.CsprojFileName), proj => ProjModifier.PrepareForChecking(proj, exercise, excluded)) } }); var result = SandboxRunner.Run(new ProjRunnerSubmission { Id = slide.Id.ToString(), ZipFileData = bytes, ProjectFileName = exercise.CsprojFileName, Input = "", NeedRun = true }); Console.WriteLine("Result = " + result); Assert.AreEqual(Verdict.Ok, result.Verdict); Assert.AreNotEqual("", result.Output); }
private static void EthalonSolutionForSingleFileExercises(ExerciseSlide slide) { var exercise = (SingleFileExerciseBlock)slide.Exercise; var solution = exercise.BuildSolution(exercise.EthalonSolution); if (solution.HasErrors) { FailOnError(slide, solution, exercise.EthalonSolution); return; } var result = SandboxRunner.Run(exercise.CreateSubmition( slide.Id.ToString(), exercise.EthalonSolution), new SandboxRunnerSettings()); var output = result.GetOutput().NormalizeEoln(); var isRightAnswer = output.NormalizeEoln().Equals(slide.Exercise.ExpectedOutput.NormalizeEoln()); if (!isRightAnswer) { Assert.Fail("mistake in: " + slide.Info.Unit.Title + " - " + slide.Title + "\n" + "\tActualOutput: " + output.NormalizeEoln() + "\n" + "\tExpectedOutput: " + slide.Exercise.ExpectedOutput.NormalizeEoln() + "\n" + "\tCompilationError: " + result.CompilationOutput + "\n" + "\tSourceCode: " + solution.SourceCode + "\n\n"); } }
private void EthalonSolutionForSingleFileExercises(ExerciseSlide slide) { var exercise = (SingleFileExerciseBlock)slide.Exercise; var solution = exercise.BuildSolution(exercise.EthalonSolution); if (solution.HasErrors) { FailOnError(slide, solution, exercise.EthalonSolution); return; } if (solution.HasStyleIssues) { Console.WriteLine("Style issue: " + solution.StyleMessage); } var result = SandboxRunner.Run(exercise.CreateSubmition( slide.Id.ToString(), exercise.EthalonSolution), settings); var output = result.GetOutput().NormalizeEoln(); var isRightAnswer = output.NormalizeEoln().Equals(slide.Exercise.ExpectedOutput.NormalizeEoln()); if (!isRightAnswer) { ReportSlideError(slide, "ActualOutput: " + output.NormalizeEoln() + "\n" + "ExpectedOutput: " + slide.Exercise.ExpectedOutput.NormalizeEoln() + "\n" + "CompilationError: " + result.CompilationOutput + "\n" + "SourceCode: " + solution.SourceCode + "\n\n"); } }
private void ReportIfEthalonSolutionHasErrorsOrIssues(ExerciseSlide slide) { var exercise = (SingleFileExerciseBlock)slide.Exercise; var ethalon = exercise.EthalonSolution.RemoveCommonNesting(); var solution = exercise.BuildSolution(ethalon); if (solution.HasErrors) { FailOnError(slide, solution, ethalon); return; } if (solution.HasStyleErrors) { var errorMessages = string.Join("\n", solution.StyleErrors.Select(e => e.GetMessageWithPositions())); ReportSlideWarning(slide, "Style issue(s): " + errorMessages); } var result = SandboxRunner.Run(exercise.CreateSubmission(slide.Id.ToString(), ethalon), settings); var output = result.GetOutput().NormalizeEoln(); var isRightAnswer = output.NormalizeEoln().Equals(slide.Exercise.ExpectedOutput.NormalizeEoln()); if (!isRightAnswer) { ReportSlideError(slide, "Ethalon solution does not provide right answer\n" + "ActualOutput: " + output.NormalizeEoln() + "\n" + "ExpectedOutput: " + slide.Exercise.ExpectedOutput.NormalizeEoln() + "\n" + "CompilationError: " + result.CompilationOutput + "\n" + "SourceCode: " + solution.SourceCode + "\n\n"); } }
private void ReportErrorIfSolutionNotBuildingOrNotPassesTests() { var solutionCode = ex.CorrectSolutionFile.ContentAsUtf8(); var submission = ex.CreateSubmission(ex.CsprojFileName, solutionCode); var result = SandboxRunner.Run(submission, new SandboxRunnerSettings()); if (!IsCompiledAndExecuted(result)) { ReportSlideError(slide, $"Correct solution file {ex.CorrectSolutionFileName} verdict is not OK. RunResult = {result}"); } if (!ex.IsCorrectRunResult(result)) { ReportSlideError(slide, $"Correct solution file {ex.CorrectSolutionFileName} is not solution. RunResult = {result}. " + $"ExpectedOutput = {ex.ExpectedOutput.NormalizeEoln()} " + $"RealOutput = {result.GetOutput().NormalizeEoln()}"); } var buildResult = ex.BuildSolution(solutionCode); if (buildResult.HasStyleErrors) { var errorMessage = string.Join("\n", buildResult.StyleErrors.Select(e => e.GetMessageWithPositions())); ReportSlideWarning(slide, $"Correct solution file {ex.CorrectSolutionFileName} has style issues. {errorMessage}"); } }
private void InitialCodeIsNotSolutionForProjExercise(ExerciseSlide slide) { var exercise = (ProjectExerciseBlock)slide.Exercise; var directoryName = Path.Combine(exercise.SlideFolderPath.FullName, exercise.ExerciseDir); var excluded = (exercise.PathsToExcludeForChecker ?? new string[0]).Concat(new[] { "bin/*", "obj/*" }).ToList(); var exerciseDir = new DirectoryInfo(directoryName); var bytes = exerciseDir.ToZip(excluded, new[] { new FileContent { Path = exercise.CsprojFileName, Data = ProjModifier.ModifyCsproj(exerciseDir.GetFile(exercise.CsprojFileName), proj => ProjModifier.PrepareForChecking(proj, exercise, excluded)) } }); var result = SandboxRunner.Run(new ProjRunnerSubmission { Id = slide.Id.ToString(), ZipFileData = bytes, ProjectFileName = exercise.CsprojFileName, Input = "", NeedRun = true }); var isOk = result.Verdict.IsOneOf(Verdict.Ok, Verdict.MemoryLimit, Verdict.TimeLimit); if (!isOk) { ReportSlideError(slide, "Exercise initial code verdict is not OK. RunResult = " + result); } else if (result.Verdict == Verdict.Ok && result.Output == "") { ReportSlideError(slide, "Exercise initial code (available to students) is solution!"); } }
private void ReportErrorIfInitialCodeIsSolutionOrVerdictNotOk() { var initialCode = ex.UserCodeFile.ContentAsUtf8(); var submission = ex.CreateSubmission(ex.CsprojFileName, initialCode); var result = SandboxRunner.Run(submission); if (ex.StudentZipIsCompilable) { ReportErrorIfInitialCodeVerdictIsNotOk(result); } ReportErrorIfInitialCodeIsSolution(result); }
private void ReportWarningIfWrongAnswersAreSolutionsOrNotOk() { var filesWithWrongAnswer = FileSystem.GetFiles(ex.ExerciseFolder.FullName, SearchOption.SearchAllSubDirectories) .Select(name => new FileInfo(name)) .Where(f => ex.IsWrongAnswer(f.Name)); foreach (var waFile in filesWithWrongAnswer) { var result = SandboxRunner.Run(ex.CreateSubmission(waFile.Name, waFile.ContentAsUtf8())); ReportWarningIfWrongAnswerVerdictIsNotOk(waFile.Name, result); ReportWarningIfWrongAnswerIsSolution(waFile.Name, result); } }
public void When_CreateStudentZip_Make_Project_Able_To_Compile_If_Project_Depends_On_Many_Tasks() { var submission = new ProjRunnerSubmission { Id = "my_id", Input = "", NeedRun = true, ProjectFileName = "test.csproj", ZipFileData = studentExerciseZipFilePath.Content() }; var result = SandboxRunner.Run(submission); result.CompilationOutput.Should().Be(""); result.Error.Should().Be(""); result.Verdict.Should().Be(Verdict.Ok); }
private static RunSolutionResult GetRunResult(ExerciseBlock exercise, string code) { var buildResult = exercise.Solution.BuildSolution(code); if (buildResult.HasErrors) { return new RunSolutionResult { IsCompileError = true, CompilationError = buildResult.ErrorMessage, ExecutionServiceName = "uLearn" } } ; if (buildResult.HasStyleIssues) { return new RunSolutionResult { IsStyleViolation = true, CompilationError = buildResult.StyleMessage, ExecutionServiceName = "uLearn" } } ; var solution = buildResult.SourceCode; var submission = new RunnerSubmition { Code = solution, Id = Utils.NewNormalizedGuid(), Input = "", NeedRun = true }; var result = SandboxRunner.Run(submission); return(new RunSolutionResult { IsRightAnswer = result.Verdict == Verdict.Ok && result.GetOutput().NormalizeEoln() == exercise.ExpectedOutput.NormalizeEoln(), ActualOutput = result.GetOutput().NormalizeEoln(), CompilationError = result.CompilationOutput, ExecutionServiceName = "this", IsCompileError = result.Verdict == Verdict.CompilationError, ExpectedOutput = exercise.ExpectedOutput.NormalizeEoln() }); }
private void ReportErrorIfSolutionNotBuildingOrNotPassesTests() { var solutionCode = ex.CorrectSolutionFile.ContentAsUtf8(); var submission = ex.CreateSubmission(ex.CsprojFileName, solutionCode); var result = SandboxRunner.Run(submission, new SandboxRunnerSettings()); if (VerdictIsNotOk(result)) { ReportSlideError(slide, $"Correct solution file {ex.CorrectSolutionFileName} verdict is not OK. RunResult = {result}"); } if (!IsSolution(result)) { ReportSlideError(slide, $"Correct solution file {ex.CorrectSolutionFileName} is not solution. RunResult = {result}"); } var buildResult = ex.BuildSolution(solutionCode); if (buildResult.HasStyleErrors) { var errorMessage = string.Join("\n", buildResult.StyleErrors.Select(e => e.GetMessageWithPositions())); ReportSlideWarning(slide, $"Correct solution file {ex.CorrectSolutionFileName} has style issues. {errorMessage}"); } }