private void ReportErrorIfInitialCodeIsSolution(RunningResults result)
{
if (ex.IsCorrectRunResult(result))
{
ReportSlideError(slide, "Exercise initial code (available to students) is solution!");
}
}
private void ReportWarningIfWrongAnswerIsSolution(string waFileName, RunningResults waResult)
{
if (ex.IsCorrectRunResult(waResult))
{
ReportSlideWarning(slide, $"Code of file with wrong answer ({waFileName}) is solution!");
}
}
private void ReportErrorIfInitialCodeVerdictIsNotOk(RunningResults result)
{
if (!IsCompiledAndExecuted(result))
{
ReportSlideError(slide, "Exercise initial code verdict is not OK. RunResult = " + result);
}
}
private bool IsRightAnswer(RunningResults result, string output, AbstractExerciseBlock exerciseBlock)
{
if (result.Verdict != Verdict.Ok)
{
return(false);
}
/* For sandbox runner */
if (exerciseBlock == null)
{
return(false);
}
if (exerciseBlock.ExerciseType == ExerciseType.CheckExitCode)
{
return(true);
}
if (exerciseBlock.ExerciseType == ExerciseType.CheckOutput)
{
var expectedOutput = exerciseBlock.ExpectedOutput.NormalizeEoln();
return(output.Equals(expectedOutput));
}
throw new InvalidOperationException($"Unknown exercise type for checking: {exerciseBlock.ExerciseType}");
}
private void ReportWarningIfWrongAnswerVerdictIsNotOk(string waFileName, RunningResults waResult)
{
if (!IsCompiledAndExecuted(waResult))
{
ReportSlideWarning(slide, $"Code verdict of file with wrong answer ({waFileName}) is not OK. RunResult = " + waResult);
}
}
public async Task SaveResult(RunningResults result, Func <UserExerciseSubmission, Task> onSave)
{
log.Info($"Сохраняю информацию о проверке решения {result.Id}");
await workQueueRepo.RemoveByItemId(queueId, result.Id);
using (var transaction = db.Database.BeginTransaction())
{
var submission = await FindSubmissionById(result.Id);
if (submission == null)
{
log.Warn($"Не нашёл в базе данных решение {result.Id}");
return;
}
var aec = await UpdateAutomaticExerciseChecking(submission.AutomaticChecking, result);
await SaveAutomaticExerciseChecking(aec);
await onSave(submission);
await transaction.CommitAsync();
db.ChangeTracker.AcceptAllChanges();
if (!UnhandledSubmissionsWaiter.HandledSubmissions.TryAdd(submission.Id, DateTime.Now))
{
log.Warn($"Не удалось запомнить, что проверка {submission.Id} проверена, а результат сохранен в базу");
}
log.Info($"Есть информация о следующих проверках, которые ещё не записаны в базу клиентом: [{string.Join(", ", UnhandledSubmissionsWaiter.HandledSubmissions.Keys)}]");
}
}
private async Task <UserSolution> UpdateSubmission(UserSolution submission, RunningResults result)
{
var compilationErrorHash = (await textsRepo.AddText(result.CompilationOutput)).Hash;
var outputHash = (await textsRepo.AddText(result.GetOutput().NormalizeEoln())).Hash;
var webRunner = submission.CourseId == "web" && submission.SlideId == "runner";
var exerciseSlide = webRunner ? null : ((ExerciseSlide)courseManager.GetCourse(submission.CourseId).GetSlideById(submission.SlideId));
var updated = new UserSolution
{
Id = submission.Id,
SolutionCodeHash = submission.SolutionCodeHash,
CompilationErrorHash = compilationErrorHash,
CourseId = submission.CourseId,
SlideId = submission.SlideId,
IsCompilationError = result.Verdict == Verdict.CompilationError,
IsRightAnswer = result.Verdict == Verdict.Ok &&
(webRunner || exerciseSlide.Exercise.ExpectedOutput.NormalizeEoln() == result.GetOutput().NormalizeEoln()),
OutputHash = outputHash,
Timestamp = submission.Timestamp,
UserId = submission.UserId,
CodeHash = submission.CodeHash,
Likes = submission.Likes,
ExecutionServiceName = submission.ExecutionServiceName,
Status = SubmissionStatus.Done,
DisplayName = submission.DisplayName,
Elapsed = DateTime.Now - submission.Timestamp
};
return(updated);
}
public async Task SaveResult(RunningResults result, Func <UserExerciseSubmission, Task> onSave)
{
using (var transaction = db.Database.BeginTransaction())
{
log.Info($"Сохраняю информацию о проверке решения {result.Id}");
var submission = FindSubmissionById(result.Id);
if (submission == null)
{
log.Warn($"Не нашёл в базе данных решение {result.Id}");
return;
}
var aec = await UpdateAutomaticExerciseChecking(submission.AutomaticChecking, result).ConfigureAwait(false);
await SaveAll(Enumerable.Repeat(aec, 1)).ConfigureAwait(false);
await onSave(submission).ConfigureAwait(false);
transaction.Commit();
db.ObjectContext().AcceptAllChanges();
if (!handledSubmissions.TryAdd(submission.Id, DateTime.Now))
{
log.Warn($"Не удалось запомнить, что проверка {submission.Id} проверена, а результат сохранен в базу");
}
log.Info($"Есть информация о следующих проверках, которые ещё не записаны в базу клиентом: [{string.Join(", ", handledSubmissions.Keys)}]");
}
}
private async Task SendResultToObservers(UserExerciseSubmission submission, RunningResults result)
{
foreach (var o in resultObservers)
{
await o.ProcessResult(submission, result);
}
}
public async Task SaveResults(RunningResults result)
{
var solution = Find(result.Id);
var updatedSolution = await UpdateSubmission(solution, result);
Save(updatedSolution);
hasHandled = true;
}
public async Task PostResult([FromUri] string token, RunningResults result)
{
if (!ModelState.IsValid)
{
throw new HttpResponseException(HttpStatusCode.BadRequest);
}
CheckRunner(token);
await userSolutionsRepo.SaveResults(result);
}
public async Task ProcessResult(UserExerciseSubmission submission, RunningResults result)
{
if (result.StyleErrors == null || result.StyleErrors.Count == 0)
{
return;
}
if (result.Verdict != Verdict.Ok)
{
return;
}
var checking = submission.AutomaticChecking;
if (!checking.IsRightAnswer)
{
return;
}
var exerciseSlide = (await courseManager.FindCourseAsync(submission.CourseId))
?.FindSlideById(submission.SlideId, true) as ExerciseSlide;
if (exerciseSlide == null)
{
return;
}
if (ulearnBotUserId == null)
{
ulearnBotUserId = await usersRepo.GetUlearnBotUserId();
}
var exerciseMetricId = ExerciseController.GetExerciseMetricId(submission.CourseId, exerciseSlide);
metricSender.SendCount($"exercise.{exerciseMetricId}.StyleViolation");
foreach (var error in result.StyleErrors)
{
await slideCheckingsRepo.AddExerciseCodeReview(
submission.Id,
ulearnBotUserId,
error.Span.StartLinePosition.Line,
error.Span.StartLinePosition.Character,
error.Span.EndLinePosition.Line,
error.Span.EndLinePosition.Character,
error.Message
);
var errorName = error.ErrorType;
metricSender.SendCount("exercise.style_error");
metricSender.SendCount($"exercise.style_error.{errorName}");
metricSender.SendCount($"exercise.{exerciseMetricId}.style_error");
metricSender.SendCount($"exercise.{exerciseMetricId}.style_error.{errorName}");
}
}
private static RunningResults RunDocker(DockerSandboxRunnerSettings settings, DirectoryInfo dir)
{
var name = Guid.NewGuid();
var dockerCommand = BuildDockerCommand(settings, dir, name);
log.Info($"Start process command: docker {dockerCommand}");
using (var dockerShellProcess = BuildShellProcess(dockerCommand))
{
var sw = Stopwatch.StartNew();
dockerShellProcess.Start();
var readErrTask = new AsyncReader(dockerShellProcess.StandardError, settings.OutputLimit).GetDataAsync();
var readOutTask = new AsyncReader(dockerShellProcess.StandardOutput, settings.OutputLimit).GetDataAsync();
var isFinished = Task.WaitAll(new Task[] { readErrTask, readOutTask }, (int)(settings.MaintenanceTimeLimit + settings.TestingTimeLimit).TotalMilliseconds);
var ms = sw.ElapsedMilliseconds;
RunningResults unsuccessfulResult = null;
if (!isFinished)
{
log.Warn($"Не хватило времени ({ms} ms) на работу Docker в папке {dir.FullName}");
unsuccessfulResult = new RunningResults(Verdict.TimeLimit);
}
else if (readErrTask.Result.Length > settings.OutputLimit || readOutTask.Result.Length > settings.OutputLimit)
{
log.Warn("Программа вывела слишком много");
unsuccessfulResult = new RunningResults(Verdict.OutputLimit);
}
else
{
log.Info($"Docker закончил работу за {ms} ms и написал: {readOutTask.Result}");
}
if (!dockerShellProcess.HasExited)
{
GracefullyShutdownDocker(dockerShellProcess, name, settings);
}
if (unsuccessfulResult != null)
{
return(unsuccessfulResult);
}
if (dockerShellProcess.ExitCode != 0)
{
log.Info($"Упал в папке {dir.FullName}");
log.Warn($"Docker написал в stderr:\n{readErrTask.Result}");
return(new RunningResults(Verdict.SandboxError, error: readErrTask.Result));
}
return(new RunningResults(Verdict.Ok, output: readOutTask.Result, error: readErrTask.Result));
}
}
public async void SendResult(RunningResults result)
{
var uri = GetUri("PostResult");
var responce = await httpClient.PostAsJsonAsync(uri, result);
if (responce.IsSuccessStatusCode)
{
return;
}
Console.Error.WriteLine(DateTime.Now.ToString("HH:mm:ss"));
Console.Error.WriteLine(responce.ToString());
Console.Error.WriteLine(result);
}
private void MainLoop(Client client)
{
var serviceKeepAliver = new ServiceKeepAliver(serviceName);
var configuration = ApplicationConfiguration.Read <UlearnConfiguration>();
var keepAliveInterval = TimeSpan.FromSeconds(configuration.KeepAliveInterval ?? 30);
while (!shutdownEvent.WaitOne(0))
{
var unhandledSubmissions = new List <RunnerSubmission>();
try
{
unhandledSubmissions.AddRange(client.TryGetSubmission(supportedSandboxes).Result);
}
catch (Exception e)
{
log.Error(e, $"Не могу получить решения из ulearn. Следующая попытка через {sleep.TotalSeconds} секунд");
Thread.Sleep(sleep);
continue;
}
log.Info($"Получил {unhandledSubmissions.Count} решение(й) со следующими ID: [{string.Join(", ", unhandledSubmissions.Select(s => s.Id))}]");
foreach (var submission in unhandledSubmissions)
{
RunningResults result;
try
{
result = SandboxRunnerClient.Run(submission);
log.Info($"Результат проверки: [{result}]");
}
catch (Exception ex)
{
result = new RunningResults(submission.Id, Verdict.SandboxError, error: ex.ToString());
log.Error(ex);
}
try
{
client.SendResults(result);
}
catch (Exception e)
{
log.Error(e, "Не могу отправить результаты проверки на ulearn");
}
}
serviceKeepAliver.Ping(keepAliveInterval);
Thread.Sleep(sleep);
}
}
public async Task ProcessResult(UserExerciseSubmission submission, RunningResults result)
{
var xQueueSubmission = await xQueueRepo.FindXQueueSubmission(submission);
if (xQueueSubmission == null)
{
return;
}
var watcher = xQueueSubmission.Watcher;
var client = new XQueueClient(watcher.BaseUrl, watcher.UserName, watcher.Password);
await client.Login();
if (await SendSubmissionResultsToQueue(client, xQueueSubmission))
{
await xQueueRepo.MarkXQueueSubmissionThatResultIsSent(xQueueSubmission);
}
}
public async Task ProcessResult(UserExerciseSubmission submission, RunningResults result)
{
var ltiRequestJson = await ltiRequestsRepo.Find(submission.CourseId, submission.UserId, submission.SlideId);
if (ltiRequestJson != null)
{
try
{
var exerciseSlide = (await courseManager.FindCourseAsync(submission.CourseId)).FindSlideById(submission.SlideId, true) as ExerciseSlide;
var score = await visitsRepo.GetScore(submission.CourseId, submission.SlideId, submission.UserId);
await LtiUtils.SubmitScore(exerciseSlide, submission.UserId, score, ltiRequestJson, ltiConsumersRepo);
}
catch (Exception e)
{
log.Error(e, "Мы не смогли отправить баллы на образовательную платформу");
}
}
}
public static void AddCompilationInfo(this RunningResults results, ImmutableArray <Diagnostic> diagnostics)
{
if (diagnostics.Length == 0)
{
return;
}
var sb = new StringBuilder();
var errors = diagnostics.Where(d => d.DefaultSeverity.IsOneOf(DiagnosticSeverity.Error, DiagnosticSeverity.Warning)).ToList();
foreach (var error in errors)
{
sb.Append(error);
}
if (errors.Any(e => e.DefaultSeverity == DiagnosticSeverity.Error))
{
results.Verdict = Verdict.CompilationError;
}
results.CompilationOutput = sb.ToString();
}
public static void AddCompilationInfo(this RunningResults results, CompilerResults assembly)
{
if (assembly.Errors.Count == 0)
{
return;
}
var sb = new StringBuilder();
var errors = assembly.Errors
.Cast <CompilerError>()
.ToList();
foreach (var error in errors)
{
sb.Append(String.Format("({2},{3}): {0} {1}: {4}\n", error.IsWarning ? "warning" : "error", error.ErrorNumber,
error.Line, error.Column,
error.ErrorText));
}
if (assembly.Errors.HasErrors)
{
results.Verdict = Verdict.CompilationError;
}
results.CompilationOutput = sb.ToString();
}
public async Task ProcessResult(UserExerciseSubmission submission, RunningResults result)
{
/* Ignore all verdicts except SandboxError */
if (result.Verdict != Verdict.SandboxError)
{
return;
}
var output = result.GetOutput();
var logs = result.Logs == null ? null : string.Join("\n", result.Logs);
var message = $"<b>Решение #{submission.Id} не запустилось в песочнице {submission.Sandbox} (SandboxError).</b>\n";
if (!string.IsNullOrEmpty(output))
{
message += $"Вывод:\n<pre>{output.EscapeHtml()}</pre>\n";
}
if (!string.IsNullOrEmpty(logs))
{
message += $"Логи:\n<pre>{logs.EscapeHtml()}</pre>\n";
}
await bot.PostToChannelAsync(message, ParseMode.Html).ConfigureAwait(false);
}
private bool IsRightAnswer(RunningResults result, string output, AbstractExerciseBlock exerciseBlock)
{
if (result.Verdict != Verdict.Ok)
{
return(false);
}
/* For sandbox runner */
if (exerciseBlock == null)
{
return(false);
}
if (exerciseBlock.ExerciseType == ExerciseType.CheckExitCode)
{
return(true);
}
if (exerciseBlock.ExerciseType == ExerciseType.CheckOutput)
{
var expectedOutput = exerciseBlock.ExpectedOutput.NormalizeEoln();
return(output.Equals(expectedOutput));
}
if (exerciseBlock.ExerciseType == ExerciseType.CheckPoints)
{
if (!result.Points.HasValue)
{
return(false);
}
const float eps = 0.00001f;
return(exerciseBlock.SmallPointsIsBetter ? result.Points.Value <exerciseBlock.PassingPoints + eps : result.Points.Value> exerciseBlock.PassingPoints - eps);
}
throw new InvalidOperationException($"Unknown exercise type for checking: {exerciseBlock.ExerciseType}");
}
public async Task ProcessResult(UserExerciseSubmission submission, RunningResults result)
{
if (result.StyleErrors == null || result.StyleErrors.Count == 0)
{
return;
}
if (result.Verdict != Verdict.Ok)
{
return;
}
var checking = submission.AutomaticChecking;
if (!checking.IsRightAnswer)
{
return;
}
var exerciseSlide = (await courseManager.FindCourseAsync(submission.CourseId))
?.FindSlideById(submission.SlideId, true) as ExerciseSlide;
if (exerciseSlide == null)
{
return;
}
if (ulearnBotUserId == null)
{
ulearnBotUserId = await usersRepo.GetUlearnBotUserId();
}
var exerciseMetricId = RunnerSetResultController.GetExerciseMetricId(submission.CourseId, exerciseSlide);
await CreateStyleErrorsReviewsForSubmission(submission.Id, result.StyleErrors, exerciseMetricId);
}
private async Task <AutomaticExerciseChecking> UpdateAutomaticExerciseChecking(AutomaticExerciseChecking checking, RunningResults result)
{
var compilationErrorHash = (await textsRepo.AddText(result.CompilationOutput)).Hash;
var output = result.GetOutput().NormalizeEoln();
var outputHash = (await textsRepo.AddText(output)).Hash;
var isWebRunner = checking.CourseId == "web" && checking.SlideId == Guid.Empty;
var exerciseSlide = isWebRunner ? null : (ExerciseSlide)courseManager.GetCourse(checking.CourseId).GetSlideById(checking.SlideId);
var isRightAnswer = IsRightAnswer(result, output, exerciseSlide?.Exercise);
var score = exerciseSlide != null && isRightAnswer ? exerciseSlide.Exercise.CorrectnessScore : 0;
/* For skipped slides score is always 0 */
if (visitsRepo.IsSkipped(checking.CourseId, checking.SlideId, checking.UserId))
{
score = 0;
}
var newChecking = new AutomaticExerciseChecking
{
Id = checking.Id,
CourseId = checking.CourseId,
SlideId = checking.SlideId,
UserId = checking.UserId,
Timestamp = checking.Timestamp,
CompilationErrorHash = compilationErrorHash,
IsCompilationError = result.Verdict == Verdict.CompilationError,
OutputHash = outputHash,
ExecutionServiceName = checking.ExecutionServiceName,
Status = AutomaticExerciseCheckingStatus.Done,
DisplayName = checking.DisplayName,
Elapsed = DateTime.Now - checking.Timestamp,
IsRightAnswer = isRightAnswer,
Score = score,
CheckingAgentName = checking.CheckingAgentName,
};
return(newChecking);
}
private static void HandleInnerException(ref RunningResults results, Exception ex)
{
results.Verdict = Verdict.RuntimeError;
results.Error = ex.ToString();
}
private static void HandleInnerException(ref RunningResults results, MemberAccessException ex)
{
results.Verdict = Verdict.SecurityException;
results.Error = ex.ToString();
}
private static void HandleException(ref RunningResults results, TargetInvocationException ex)
{
HandleInnerException(ref results, (dynamic)ex.InnerException);
}
private static void HandleException(ref RunningResults results, Exception ex)
{
results.Verdict = Verdict.SandboxError;
results.Error = ex.ToString();
}
public static bool IsCompilationError(this RunningResults results)
{
return(results.Verdict == Verdict.CompilationError);
}
public static void HandleException(this RunningResults results, Exception ex)
{
HandleException(ref results, (dynamic)ex);
}
protected static bool IsCompiledAndExecuted(RunningResults result)
{
return(result.Verdict.IsOneOf(Verdict.Ok, Verdict.OutputLimit, Verdict.MemoryLimit, Verdict.TimeLimit));
}
