private void ReportErrorIfInitialCodeIsSolution(RunningResults result) { if (ex.IsCorrectRunResult(result)) { ReportSlideError(slide, "Exercise initial code (available to students) is solution!"); } }
private void ReportWarningIfWrongAnswerIsSolution(string waFileName, RunningResults waResult) { if (ex.IsCorrectRunResult(waResult)) { ReportSlideWarning(slide, $"Code of file with wrong answer ({waFileName}) is solution!"); } }
private void ReportErrorIfInitialCodeVerdictIsNotOk(RunningResults result) { if (!IsCompiledAndExecuted(result)) { ReportSlideError(slide, "Exercise initial code verdict is not OK. RunResult = " + result); } }
private bool IsRightAnswer(RunningResults result, string output, AbstractExerciseBlock exerciseBlock) { if (result.Verdict != Verdict.Ok) { return(false); } /* For sandbox runner */ if (exerciseBlock == null) { return(false); } if (exerciseBlock.ExerciseType == ExerciseType.CheckExitCode) { return(true); } if (exerciseBlock.ExerciseType == ExerciseType.CheckOutput) { var expectedOutput = exerciseBlock.ExpectedOutput.NormalizeEoln(); return(output.Equals(expectedOutput)); } throw new InvalidOperationException($"Unknown exercise type for checking: {exerciseBlock.ExerciseType}"); }
private void ReportWarningIfWrongAnswerVerdictIsNotOk(string waFileName, RunningResults waResult) { if (!IsCompiledAndExecuted(waResult)) { ReportSlideWarning(slide, $"Code verdict of file with wrong answer ({waFileName}) is not OK. RunResult = " + waResult); } }
public async Task SaveResult(RunningResults result, Func <UserExerciseSubmission, Task> onSave) { log.Info($"Сохраняю информацию о проверке решения {result.Id}"); await workQueueRepo.RemoveByItemId(queueId, result.Id); using (var transaction = db.Database.BeginTransaction()) { var submission = await FindSubmissionById(result.Id); if (submission == null) { log.Warn($"Не нашёл в базе данных решение {result.Id}"); return; } var aec = await UpdateAutomaticExerciseChecking(submission.AutomaticChecking, result); await SaveAutomaticExerciseChecking(aec); await onSave(submission); await transaction.CommitAsync(); db.ChangeTracker.AcceptAllChanges(); if (!UnhandledSubmissionsWaiter.HandledSubmissions.TryAdd(submission.Id, DateTime.Now)) { log.Warn($"Не удалось запомнить, что проверка {submission.Id} проверена, а результат сохранен в базу"); } log.Info($"Есть информация о следующих проверках, которые ещё не записаны в базу клиентом: [{string.Join(", ", UnhandledSubmissionsWaiter.HandledSubmissions.Keys)}]"); } }
private async Task <UserSolution> UpdateSubmission(UserSolution submission, RunningResults result) { var compilationErrorHash = (await textsRepo.AddText(result.CompilationOutput)).Hash; var outputHash = (await textsRepo.AddText(result.GetOutput().NormalizeEoln())).Hash; var webRunner = submission.CourseId == "web" && submission.SlideId == "runner"; var exerciseSlide = webRunner ? null : ((ExerciseSlide)courseManager.GetCourse(submission.CourseId).GetSlideById(submission.SlideId)); var updated = new UserSolution { Id = submission.Id, SolutionCodeHash = submission.SolutionCodeHash, CompilationErrorHash = compilationErrorHash, CourseId = submission.CourseId, SlideId = submission.SlideId, IsCompilationError = result.Verdict == Verdict.CompilationError, IsRightAnswer = result.Verdict == Verdict.Ok && (webRunner || exerciseSlide.Exercise.ExpectedOutput.NormalizeEoln() == result.GetOutput().NormalizeEoln()), OutputHash = outputHash, Timestamp = submission.Timestamp, UserId = submission.UserId, CodeHash = submission.CodeHash, Likes = submission.Likes, ExecutionServiceName = submission.ExecutionServiceName, Status = SubmissionStatus.Done, DisplayName = submission.DisplayName, Elapsed = DateTime.Now - submission.Timestamp }; return(updated); }
public async Task SaveResult(RunningResults result, Func <UserExerciseSubmission, Task> onSave) { using (var transaction = db.Database.BeginTransaction()) { log.Info($"Сохраняю информацию о проверке решения {result.Id}"); var submission = FindSubmissionById(result.Id); if (submission == null) { log.Warn($"Не нашёл в базе данных решение {result.Id}"); return; } var aec = await UpdateAutomaticExerciseChecking(submission.AutomaticChecking, result).ConfigureAwait(false); await SaveAll(Enumerable.Repeat(aec, 1)).ConfigureAwait(false); await onSave(submission).ConfigureAwait(false); transaction.Commit(); db.ObjectContext().AcceptAllChanges(); if (!handledSubmissions.TryAdd(submission.Id, DateTime.Now)) { log.Warn($"Не удалось запомнить, что проверка {submission.Id} проверена, а результат сохранен в базу"); } log.Info($"Есть информация о следующих проверках, которые ещё не записаны в базу клиентом: [{string.Join(", ", handledSubmissions.Keys)}]"); } }
private async Task SendResultToObservers(UserExerciseSubmission submission, RunningResults result) { foreach (var o in resultObservers) { await o.ProcessResult(submission, result); } }
public async Task SaveResults(RunningResults result) { var solution = Find(result.Id); var updatedSolution = await UpdateSubmission(solution, result); Save(updatedSolution); hasHandled = true; }
public async Task PostResult([FromUri] string token, RunningResults result) { if (!ModelState.IsValid) { throw new HttpResponseException(HttpStatusCode.BadRequest); } CheckRunner(token); await userSolutionsRepo.SaveResults(result); }
public async Task ProcessResult(UserExerciseSubmission submission, RunningResults result) { if (result.StyleErrors == null || result.StyleErrors.Count == 0) { return; } if (result.Verdict != Verdict.Ok) { return; } var checking = submission.AutomaticChecking; if (!checking.IsRightAnswer) { return; } var exerciseSlide = (await courseManager.FindCourseAsync(submission.CourseId)) ?.FindSlideById(submission.SlideId, true) as ExerciseSlide; if (exerciseSlide == null) { return; } if (ulearnBotUserId == null) { ulearnBotUserId = await usersRepo.GetUlearnBotUserId(); } var exerciseMetricId = ExerciseController.GetExerciseMetricId(submission.CourseId, exerciseSlide); metricSender.SendCount($"exercise.{exerciseMetricId}.StyleViolation"); foreach (var error in result.StyleErrors) { await slideCheckingsRepo.AddExerciseCodeReview( submission.Id, ulearnBotUserId, error.Span.StartLinePosition.Line, error.Span.StartLinePosition.Character, error.Span.EndLinePosition.Line, error.Span.EndLinePosition.Character, error.Message ); var errorName = error.ErrorType; metricSender.SendCount("exercise.style_error"); metricSender.SendCount($"exercise.style_error.{errorName}"); metricSender.SendCount($"exercise.{exerciseMetricId}.style_error"); metricSender.SendCount($"exercise.{exerciseMetricId}.style_error.{errorName}"); } }
private static RunningResults RunDocker(DockerSandboxRunnerSettings settings, DirectoryInfo dir) { var name = Guid.NewGuid(); var dockerCommand = BuildDockerCommand(settings, dir, name); log.Info($"Start process command: docker {dockerCommand}"); using (var dockerShellProcess = BuildShellProcess(dockerCommand)) { var sw = Stopwatch.StartNew(); dockerShellProcess.Start(); var readErrTask = new AsyncReader(dockerShellProcess.StandardError, settings.OutputLimit).GetDataAsync(); var readOutTask = new AsyncReader(dockerShellProcess.StandardOutput, settings.OutputLimit).GetDataAsync(); var isFinished = Task.WaitAll(new Task[] { readErrTask, readOutTask }, (int)(settings.MaintenanceTimeLimit + settings.TestingTimeLimit).TotalMilliseconds); var ms = sw.ElapsedMilliseconds; RunningResults unsuccessfulResult = null; if (!isFinished) { log.Warn($"Не хватило времени ({ms} ms) на работу Docker в папке {dir.FullName}"); unsuccessfulResult = new RunningResults(Verdict.TimeLimit); } else if (readErrTask.Result.Length > settings.OutputLimit || readOutTask.Result.Length > settings.OutputLimit) { log.Warn("Программа вывела слишком много"); unsuccessfulResult = new RunningResults(Verdict.OutputLimit); } else { log.Info($"Docker закончил работу за {ms} ms и написал: {readOutTask.Result}"); } if (!dockerShellProcess.HasExited) { GracefullyShutdownDocker(dockerShellProcess, name, settings); } if (unsuccessfulResult != null) { return(unsuccessfulResult); } if (dockerShellProcess.ExitCode != 0) { log.Info($"Упал в папке {dir.FullName}"); log.Warn($"Docker написал в stderr:\n{readErrTask.Result}"); return(new RunningResults(Verdict.SandboxError, error: readErrTask.Result)); } return(new RunningResults(Verdict.Ok, output: readOutTask.Result, error: readErrTask.Result)); } }
public async void SendResult(RunningResults result) { var uri = GetUri("PostResult"); var responce = await httpClient.PostAsJsonAsync(uri, result); if (responce.IsSuccessStatusCode) { return; } Console.Error.WriteLine(DateTime.Now.ToString("HH:mm:ss")); Console.Error.WriteLine(responce.ToString()); Console.Error.WriteLine(result); }
private void MainLoop(Client client) { var serviceKeepAliver = new ServiceKeepAliver(serviceName); var configuration = ApplicationConfiguration.Read <UlearnConfiguration>(); var keepAliveInterval = TimeSpan.FromSeconds(configuration.KeepAliveInterval ?? 30); while (!shutdownEvent.WaitOne(0)) { var unhandledSubmissions = new List <RunnerSubmission>(); try { unhandledSubmissions.AddRange(client.TryGetSubmission(supportedSandboxes).Result); } catch (Exception e) { log.Error(e, $"Не могу получить решения из ulearn. Следующая попытка через {sleep.TotalSeconds} секунд"); Thread.Sleep(sleep); continue; } log.Info($"Получил {unhandledSubmissions.Count} решение(й) со следующими ID: [{string.Join(", ", unhandledSubmissions.Select(s => s.Id))}]"); foreach (var submission in unhandledSubmissions) { RunningResults result; try { result = SandboxRunnerClient.Run(submission); log.Info($"Результат проверки: [{result}]"); } catch (Exception ex) { result = new RunningResults(submission.Id, Verdict.SandboxError, error: ex.ToString()); log.Error(ex); } try { client.SendResults(result); } catch (Exception e) { log.Error(e, "Не могу отправить результаты проверки на ulearn"); } } serviceKeepAliver.Ping(keepAliveInterval); Thread.Sleep(sleep); } }
public async Task ProcessResult(UserExerciseSubmission submission, RunningResults result) { var xQueueSubmission = await xQueueRepo.FindXQueueSubmission(submission); if (xQueueSubmission == null) { return; } var watcher = xQueueSubmission.Watcher; var client = new XQueueClient(watcher.BaseUrl, watcher.UserName, watcher.Password); await client.Login(); if (await SendSubmissionResultsToQueue(client, xQueueSubmission)) { await xQueueRepo.MarkXQueueSubmissionThatResultIsSent(xQueueSubmission); } }
public async Task ProcessResult(UserExerciseSubmission submission, RunningResults result) { var ltiRequestJson = await ltiRequestsRepo.Find(submission.CourseId, submission.UserId, submission.SlideId); if (ltiRequestJson != null) { try { var exerciseSlide = (await courseManager.FindCourseAsync(submission.CourseId)).FindSlideById(submission.SlideId, true) as ExerciseSlide; var score = await visitsRepo.GetScore(submission.CourseId, submission.SlideId, submission.UserId); await LtiUtils.SubmitScore(exerciseSlide, submission.UserId, score, ltiRequestJson, ltiConsumersRepo); } catch (Exception e) { log.Error(e, "Мы не смогли отправить баллы на образовательную платформу"); } } }
public static void AddCompilationInfo(this RunningResults results, ImmutableArray <Diagnostic> diagnostics) { if (diagnostics.Length == 0) { return; } var sb = new StringBuilder(); var errors = diagnostics.Where(d => d.DefaultSeverity.IsOneOf(DiagnosticSeverity.Error, DiagnosticSeverity.Warning)).ToList(); foreach (var error in errors) { sb.Append(error); } if (errors.Any(e => e.DefaultSeverity == DiagnosticSeverity.Error)) { results.Verdict = Verdict.CompilationError; } results.CompilationOutput = sb.ToString(); }
public static void AddCompilationInfo(this RunningResults results, CompilerResults assembly) { if (assembly.Errors.Count == 0) { return; } var sb = new StringBuilder(); var errors = assembly.Errors .Cast <CompilerError>() .ToList(); foreach (var error in errors) { sb.Append(String.Format("({2},{3}): {0} {1}: {4}\n", error.IsWarning ? "warning" : "error", error.ErrorNumber, error.Line, error.Column, error.ErrorText)); } if (assembly.Errors.HasErrors) { results.Verdict = Verdict.CompilationError; } results.CompilationOutput = sb.ToString(); }
public async Task ProcessResult(UserExerciseSubmission submission, RunningResults result) { /* Ignore all verdicts except SandboxError */ if (result.Verdict != Verdict.SandboxError) { return; } var output = result.GetOutput(); var logs = result.Logs == null ? null : string.Join("\n", result.Logs); var message = $"<b>Решение #{submission.Id} не запустилось в песочнице {submission.Sandbox} (SandboxError).</b>\n"; if (!string.IsNullOrEmpty(output)) { message += $"Вывод:\n<pre>{output.EscapeHtml()}</pre>\n"; } if (!string.IsNullOrEmpty(logs)) { message += $"Логи:\n<pre>{logs.EscapeHtml()}</pre>\n"; } await bot.PostToChannelAsync(message, ParseMode.Html).ConfigureAwait(false); }
private bool IsRightAnswer(RunningResults result, string output, AbstractExerciseBlock exerciseBlock) { if (result.Verdict != Verdict.Ok) { return(false); } /* For sandbox runner */ if (exerciseBlock == null) { return(false); } if (exerciseBlock.ExerciseType == ExerciseType.CheckExitCode) { return(true); } if (exerciseBlock.ExerciseType == ExerciseType.CheckOutput) { var expectedOutput = exerciseBlock.ExpectedOutput.NormalizeEoln(); return(output.Equals(expectedOutput)); } if (exerciseBlock.ExerciseType == ExerciseType.CheckPoints) { if (!result.Points.HasValue) { return(false); } const float eps = 0.00001f; return(exerciseBlock.SmallPointsIsBetter ? result.Points.Value <exerciseBlock.PassingPoints + eps : result.Points.Value> exerciseBlock.PassingPoints - eps); } throw new InvalidOperationException($"Unknown exercise type for checking: {exerciseBlock.ExerciseType}"); }
public async Task ProcessResult(UserExerciseSubmission submission, RunningResults result) { if (result.StyleErrors == null || result.StyleErrors.Count == 0) { return; } if (result.Verdict != Verdict.Ok) { return; } var checking = submission.AutomaticChecking; if (!checking.IsRightAnswer) { return; } var exerciseSlide = (await courseManager.FindCourseAsync(submission.CourseId)) ?.FindSlideById(submission.SlideId, true) as ExerciseSlide; if (exerciseSlide == null) { return; } if (ulearnBotUserId == null) { ulearnBotUserId = await usersRepo.GetUlearnBotUserId(); } var exerciseMetricId = RunnerSetResultController.GetExerciseMetricId(submission.CourseId, exerciseSlide); await CreateStyleErrorsReviewsForSubmission(submission.Id, result.StyleErrors, exerciseMetricId); }
private async Task <AutomaticExerciseChecking> UpdateAutomaticExerciseChecking(AutomaticExerciseChecking checking, RunningResults result) { var compilationErrorHash = (await textsRepo.AddText(result.CompilationOutput)).Hash; var output = result.GetOutput().NormalizeEoln(); var outputHash = (await textsRepo.AddText(output)).Hash; var isWebRunner = checking.CourseId == "web" && checking.SlideId == Guid.Empty; var exerciseSlide = isWebRunner ? null : (ExerciseSlide)courseManager.GetCourse(checking.CourseId).GetSlideById(checking.SlideId); var isRightAnswer = IsRightAnswer(result, output, exerciseSlide?.Exercise); var score = exerciseSlide != null && isRightAnswer ? exerciseSlide.Exercise.CorrectnessScore : 0; /* For skipped slides score is always 0 */ if (visitsRepo.IsSkipped(checking.CourseId, checking.SlideId, checking.UserId)) { score = 0; } var newChecking = new AutomaticExerciseChecking { Id = checking.Id, CourseId = checking.CourseId, SlideId = checking.SlideId, UserId = checking.UserId, Timestamp = checking.Timestamp, CompilationErrorHash = compilationErrorHash, IsCompilationError = result.Verdict == Verdict.CompilationError, OutputHash = outputHash, ExecutionServiceName = checking.ExecutionServiceName, Status = AutomaticExerciseCheckingStatus.Done, DisplayName = checking.DisplayName, Elapsed = DateTime.Now - checking.Timestamp, IsRightAnswer = isRightAnswer, Score = score, CheckingAgentName = checking.CheckingAgentName, }; return(newChecking); }
private static void HandleInnerException(ref RunningResults results, Exception ex) { results.Verdict = Verdict.RuntimeError; results.Error = ex.ToString(); }
private static void HandleInnerException(ref RunningResults results, MemberAccessException ex) { results.Verdict = Verdict.SecurityException; results.Error = ex.ToString(); }
private static void HandleException(ref RunningResults results, TargetInvocationException ex) { HandleInnerException(ref results, (dynamic)ex.InnerException); }
private static void HandleException(ref RunningResults results, Exception ex) { results.Verdict = Verdict.SandboxError; results.Error = ex.ToString(); }
public static bool IsCompilationError(this RunningResults results) { return(results.Verdict == Verdict.CompilationError); }
public static void HandleException(this RunningResults results, Exception ex) { HandleException(ref results, (dynamic)ex); }
protected static bool IsCompiledAndExecuted(RunningResults result) { return(result.Verdict.IsOneOf(Verdict.Ok, Verdict.OutputLimit, Verdict.MemoryLimit, Verdict.TimeLimit)); }