private void HandleTypeUnittest(CourseProblemCase @case, ExecutionResult result) { var caseResult = Item.Results.First(i => i.Case == @case.Id); var errorText = Context.TmpDir.ErrorFile(@case.Id).ReadAllText(); if (string.IsNullOrEmpty(errorText)) { if (result.IsTimeOuted) { caseResult.Status = ProcessStatus.AnswerCorrectTimeout.Value; caseResult.Message = ProcessStatus.AnswerCorrectTimeout.Description; } else { caseResult.Status = ProcessStatus.AnswerCorrect.Value; caseResult.Message = ProcessStatus.AnswerCorrect.Description; } } else { caseResult.Status = ProcessStatus.AnswerWrong.Value; caseResult.Message = ProcessStatus.AnswerWrong.Description; caseResult.Messages = errorText.SplitLines(); } }
private async Task SolveCaseAsync(CourseProblemCase @case) { var result = await SolveCaseBaseAsync(@case); if (result.IsBroken) { return; } switch (Context.CourseProblem.Type) { case ProblemType.Unittest: HandleTypeUnittest(@case, result); return; case ProblemType.LineByLine: HandleTypeLineByLine(@case, result); return; case ProblemType.Program: HandleTypeProgram(@case, result); return; default: throw new ArgumentOutOfRangeException(nameof(ProblemType)); } }
private void HandleTypeLineByLine(CourseProblemCase @case, ExecutionResult result) { var caseResult = Item.Results.First(i => i.Case == @case.Id); var referenceFile = Context.ProblemDir.OutputFile(@case.Id); if (!File.Exists(referenceFile)) { caseResult.SetStatus(ProcessStatus.Skipped); caseResult.Message = "Reference file is missing"; caseResult.Messages = new [] { $"File {referenceFile} does not exists, contact the administrator or a teacher." }; return; } var diffResult = _compareService.CompareFiles(Context, @case); if (diffResult.isOk) { caseResult.SetStatus(result.IsTimeOuted ? ProcessStatus.AnswerCorrectTimeout : ProcessStatus.AnswerCorrect); } else { caseResult.SetStatus(result.IsTimeOuted ? ProcessStatus.AnswerWrongTimeout : ProcessStatus.AnswerWrong); } }
public DiffResultComposite CompareFilesComposite(CourseContext Context, CourseProblemCase @case) { var generatedFile = Context.TmpDir.OutputFile(@case.Id); var referenceFile = Context.ProblemDir.OutputFile(@case.Id); return(CompareFilesComposite(generatedFile, referenceFile)); }
private async Task <ExecutionResult> SolveCaseBaseAsync(CourseProblemCase @case) { var caseId = @case.Id; var subcase = Item.Results.First(i => i.Case == caseId); // skip global timeouts if (TimeBank.IsBusted()) { subcase.SetStatus(ProcessStatus.Skipped); subcase.TimeLimit = 0; subcase.Messages = new[] { "No time left" }; return(new ExecutionResult { Code = ExecutionStatus.GlobalTimeout, Status = ExecutionStatus.GlobalTimeout.ToString(), Duration = 0, ReturnCode = -1, }); } // copy test assets CopyInDocker($"assets/{caseId}/*"); subcase.Status = ProcessStatus.Running.Value; subcase.TimeLimit = @case.Timeout.ScaleTo(Context.Language); var language = Context.Language; var isUnitTest = Context.CourseProblem.Type == ProblemType.Unittest; var pipeline = isUnitTest && language.Unittest.Any() ? language.Unittest : language.Run; SetPermissions(); var executionResult = ExecuteCommand(new ExecutionCommand { Command = $"{string.Join(" ", pipeline)}".ReplaceCommon(Context.MainFileName), Workdir = Context.DockerTmpWorkdir, Timeout = @case.Timeout.ScaleTo(language), Deadline = TimeBank.TimeLeft, IPath = isUnitTest ? null : $"input/{@case.Id}", OPath = $"output/{@case.Id}", EPath = $"error/{@case.Id}" }); CopyOutputFromDocker(@case); CopyErrorFromDocker(@case); CopyFromDocker(".report.json"); CopyFromDocker(".report.simple.json"); foreach (var f in Context.CourseProblem.Export) { CopyFromDocker(f); } // determine messages and statuses _evaluationService.EvaluateTypeSolve(executionResult, this, @case); return(executionResult); }
public void CopyVerificationFromDocker(CourseProblemCase @case) { var dFile = Context.DockerDir.VerificationFile(@case.Id); var tFile = Context.TmpDir.VerificationFile(@case.Id); ProcessUtils.Popen($"docker cp \"{ProcessService.ContainerName}:{dFile}.o\" \"{tFile}.o\""); ProcessUtils.Popen($"docker cp \"{ProcessService.ContainerName}:{dFile}.e\" \"{tFile}.e\""); }
private void HandleTypeProgram(CourseProblemCase @case, ExecutionResult result) { var caseResult = Item.Results.First(i => i.Case == @case.Id); // reference solution var reference = Context.CourseProblem.Reference; var refLanguage = _languageService[reference.Lang]; var args = new List <string> { "-v", "-i", Context.DockerDir.InputFile(@case.Id), "-o", Context.DockerDir.OutputFile(@case.Id), }; var pipeline = new List <string>() .Concat(refLanguage.Run) .Concat(args) .ToList(); var verifyResult = ExecuteCommand(new ExecutionCommand { Command = $"{string.Join(" ", pipeline)}".ReplaceCommon(reference.Name), Workdir = Context.DockerDir.VerificationDir, OPath = $"{@case.Id}.o", EPath = $"{@case.Id}.e", }); if (verifyResult.IsBroken) { // copy files from tmp CopyVerificationFromDocker(@case); var output = File.ReadAllText($"{Context.TmpDir.VerificationDir}/{@case.Id}.o").Trim(); var error = File.ReadAllText($"{Context.TmpDir.VerificationDir}/{@case.Id}.e").Trim(); var messages = new List <string> { "Verification failed:" } .Concat(output.SplitLines()) .Concat(error.SplitLines()) .Where(i => !string.IsNullOrEmpty(i)) .ToList(); caseResult.Status = ProcessStatus.AnswerWrong.Value; caseResult.Message = ProcessStatus.AnswerWrong.Description; caseResult.Messages = messages.ToArray(); } else { caseResult.Status = ProcessStatus.AnswerCorrect.Value; caseResult.Message = ProcessStatus.AnswerCorrect.Description; } }
private async Task <ProcessResult> ProcessCaseMatlabAsync(CourseProblemCase @case, string filename) { var uid = Guid.NewGuid().ToString(); var content = _matlabResetTemplate.Replace("{guid}", uid); var resetFilename = "resetworkspace.m"; await File.WriteAllTextAsync(Context.TmpDir.RootFile(resetFilename), content); CopyToDocker(resetFilename); SetPermissions(); return(await _matlabServer.RunFileAsync(filename, uid, resetFilename, Context.DockerTmpWorkdir)); }
private async Task GenerateInputAction(CourseProblemCase @case) { var subcase = Item.Results.First(i => i.Case == @case.Id); if ([email protected]()) { subcase.Status = ProcessStatus.Skipped.Value; subcase.Message = "Skipping static input file"; subcase.TimeLimit = 0; return; } // skip global timeouts if (TimeBank.IsBusted()) { subcase.SetStatus(ProcessStatus.Skipped); subcase.Messages = new[] { "No time left" }; subcase.TimeLimit = 0; return; } subcase.Status = ProcessStatus.Running.Value; subcase.TimeLimit = @case.Timeout.ScaleTo(Context.Language); var baseCommand = $"{string.Join(" ", Context.Language.Run)}".ReplaceCommon(Context.MainFileName); var fullCommand = $"{baseCommand} {@case.GetArguments()}"; var generateResult = ExecuteCommand(new ExecutionCommand { Command = fullCommand, Workdir = Context.DockerTmpWorkdir, Timeout = @case.Timeout.ScaleTo(Context.Language), Deadline = TimeBank.TimeLeft, OPath = $"output/{@case.Id}", EPath = $"error/{@case.Id}", }); CopyOutputFromDocker(@case); CopyErrorFromDocker(@case); CopyToResultDir(Context.TmpDir.OutputDir, Context.ProblemDir.InputDir, false); _evaluationService.EvaluateTypeGenerate(generateResult, this, @case); }
public void EvaluateTypeSolve(ExecutionResult executionResult, ProcessItem processItem, CourseProblemCase @case) { var context = processItem.Context; var ccData = processItem.Item; var subcase = ccData.Results.First(i => i.Case == @case.Id); subcase.Duration = executionResult.Duration; subcase.Returncode = executionResult.ReturnCode; subcase.FullCommand = executionResult.ExecutionCommand.FullCmd; subcase.Command = executionResult.ExecutionCommand.Command; processItem.TimeBank.WallTime += executionResult.Duration; var pythonReportJson = context.TmpDir.RootFile(".report.json"); if (File.Exists(pythonReportJson)) { var report = PythonReport.FromJson(pythonReportJson.ReadAllText()); ccData.Results.AddRange( report.Report.Tests.Select(i => new CcDataCaseResult { Case = i.Name, Duration = i.Duration, Status = i.Outcome == "passed" ? (int)ProcessStatusCodes.AnswerCorrect : (int)ProcessStatusCodes.AnswerWrong }) ); File.Delete(pythonReportJson); } var simpleReportJson = context.TmpDir.RootFile(".report.simple.json"); if (File.Exists(simpleReportJson)) { var report = SimpleReport.FromJson(simpleReportJson.ReadAllText()); ccData.Results.AddRange( report.Tests.Select(i => new CcDataCaseResult { Case = i.Name, Duration = i.Duration, Message = i.Message, Status = i.Outcome == "passed" ? (int)ProcessStatusCodes.AnswerCorrect : (int)ProcessStatusCodes.AnswerWrong }) ); File.Delete(simpleReportJson); } FillBasicInfo(executionResult, processItem, @case); }
public void EvaluateTypeGenerate(ExecutionResult executionResult, ProcessItem processItem, CourseProblemCase @case) { var context = processItem.Context; var ccData = processItem.Item; var subcase = ccData.Results.First(i => i.Case == @case.Id); subcase.Duration = executionResult.Duration; subcase.Returncode = executionResult.ReturnCode; subcase.FullCommand = executionResult.ExecutionCommand.FullCmd; subcase.Command = executionResult.ExecutionCommand.Command; processItem.TimeBank.WallTime += executionResult.Duration; var inputFile = context.ProblemDir.InputFile(@case.Id); switch (executionResult.Code) { case ExecutionStatus.Ok: case ExecutionStatus.OkTimeout: if (File.Exists(inputFile)) { subcase.SetStatus(ProcessStatus.Ok); subcase.Message = $"Input file generated: ({new FileInfo(inputFile).Length} bytes)"; return; } subcase.SetStatus(ProcessStatus.ErrorWhileRunning); subcase.Message = "Program ended with 0 but no input file was generated"; return; case ExecutionStatus.GlobalTimeout: subcase.SetStatus(ProcessStatus.GlobalTimeout); subcase.Messages = new List <string>() .Concat(executionResult.Messages?.ToArray() ?? new string[] { }) .Concat(context.GetTmpDirErrorMessage(@case.Id).SplitLines()) .ToArray(); return; case ExecutionStatus.Error: case ExecutionStatus.ErrorTimeout: case ExecutionStatus.FatalError: case ExecutionStatus.NoSuchFile: subcase.SetStatus(ProcessStatus.ErrorWhileRunning); subcase.Message = $"[{executionResult.Status}] {executionResult.Message}"; subcase.Messages = new List <string>() .Concat(executionResult.Messages?.ToArray() ?? new string[] { }) .Concat(context.GetTmpDirErrorMessage(@case.Id).SplitLines()) .ToArray(); return; default: throw new Exception( $"Fatal exception when evaluating result, unknown execution status ${executionResult.Code}"); } }
private void FillBasicInfo(ExecutionResult executionResult, ProcessItem processItem, CourseProblemCase @case) { var language = processItem.Context.Language; var subcase = processItem.Item.Results.First(i => i.Case == @case.Id); switch (executionResult.Code) { case ExecutionStatus.Ok: subcase.SetStatus(ProcessStatus.Ok); return; case ExecutionStatus.OkTimeout: subcase.SetStatus(ProcessStatus.Ok); subcase.Messages = executionResult .CreateTimeoutMessages(@case.Timeout, language) .ToArray(); return; case ExecutionStatus.GlobalTimeout: subcase.SetStatus(ProcessStatus.GlobalTimeout); subcase.Messages = executionResult .CreateDeadlineMessages(processItem.TimeBank) .ToArray(); return; case ExecutionStatus.NoSuchFile: subcase.SetStatus(ProcessStatus.ErrorWhileRunning); subcase.Messages = new List <string> { "Error while starting the process (no such file)", executionResult.Message } .ToArray(); return; case ExecutionStatus.ErrorTimeout: subcase.SetStatus(ProcessStatus.ErrorWhileRunning); subcase.Message = executionResult.Message; subcase.Messages = new List <string>() .Concat(executionResult.Messages) .Concat(new[] { "----", "Also, program did not finish in time" }) .Concat(executionResult.CreateTimeoutMessages(@case.Timeout, language)) .ToArray(); return; case ExecutionStatus.Error: case ExecutionStatus.FatalError: subcase.SetStatus(ProcessStatus.ErrorWhileRunning); subcase.Message = executionResult.Message; subcase.Messages = executionResult.Messages?.ToArray(); return; default: throw new Exception( $"Fatal exception when evaluating result, unknown execution status ${executionResult.Code}"); } }
private async Task GenerateOutputAction(CourseProblemCase @case) { await SolveCaseBaseAsync(@case); }
public ProcessResult CopyErrorFromDocker(CourseProblemCase @case) { var cpCommand = $"docker cp \"{ProcessService.ContainerName}:{Context.DockerTmpWorkdir}/error/{@case.Id}\" \"{Context.TmpDir.ErrorDir}\""; return(ProcessUtils.Popen(cpCommand)); }