private void HandleTypeUnittest(CourseProblemCase @case, ExecutionResult result)
{
var caseResult = Item.Results.First(i => i.Case == @case.Id);
var errorText = Context.TmpDir.ErrorFile(@case.Id).ReadAllText();
if (string.IsNullOrEmpty(errorText))
{
if (result.IsTimeOuted)
{
caseResult.Status = ProcessStatus.AnswerCorrectTimeout.Value;
caseResult.Message = ProcessStatus.AnswerCorrectTimeout.Description;
}
else
{
caseResult.Status = ProcessStatus.AnswerCorrect.Value;
caseResult.Message = ProcessStatus.AnswerCorrect.Description;
}
}
else
{
caseResult.Status = ProcessStatus.AnswerWrong.Value;
caseResult.Message = ProcessStatus.AnswerWrong.Description;
caseResult.Messages = errorText.SplitLines();
}
}
private async Task SolveCaseAsync(CourseProblemCase @case)
{
var result = await SolveCaseBaseAsync(@case);
if (result.IsBroken)
{
return;
}
switch (Context.CourseProblem.Type)
{
case ProblemType.Unittest:
HandleTypeUnittest(@case, result);
return;
case ProblemType.LineByLine:
HandleTypeLineByLine(@case, result);
return;
case ProblemType.Program:
HandleTypeProgram(@case, result);
return;
default:
throw new ArgumentOutOfRangeException(nameof(ProblemType));
}
}
private void HandleTypeLineByLine(CourseProblemCase @case, ExecutionResult result)
{
var caseResult = Item.Results.First(i => i.Case == @case.Id);
var referenceFile = Context.ProblemDir.OutputFile(@case.Id);
if (!File.Exists(referenceFile))
{
caseResult.SetStatus(ProcessStatus.Skipped);
caseResult.Message = "Reference file is missing";
caseResult.Messages = new []
{
$"File {referenceFile} does not exists, contact the administrator or a teacher."
};
return;
}
var diffResult = _compareService.CompareFiles(Context, @case);
if (diffResult.isOk)
{
caseResult.SetStatus(result.IsTimeOuted
? ProcessStatus.AnswerCorrectTimeout
: ProcessStatus.AnswerCorrect);
}
else
{
caseResult.SetStatus(result.IsTimeOuted
? ProcessStatus.AnswerWrongTimeout
: ProcessStatus.AnswerWrong);
}
}
public DiffResultComposite CompareFilesComposite(CourseContext Context, CourseProblemCase @case)
{
var generatedFile = Context.TmpDir.OutputFile(@case.Id);
var referenceFile = Context.ProblemDir.OutputFile(@case.Id);
return(CompareFilesComposite(generatedFile, referenceFile));
}
private async Task <ExecutionResult> SolveCaseBaseAsync(CourseProblemCase @case)
{
var caseId = @case.Id;
var subcase = Item.Results.First(i => i.Case == caseId);
// skip global timeouts
if (TimeBank.IsBusted())
{
subcase.SetStatus(ProcessStatus.Skipped);
subcase.TimeLimit = 0;
subcase.Messages = new[] { "No time left" };
return(new ExecutionResult
{
Code = ExecutionStatus.GlobalTimeout,
Status = ExecutionStatus.GlobalTimeout.ToString(),
Duration = 0,
ReturnCode = -1,
});
}
// copy test assets
CopyInDocker($"assets/{caseId}/*");
subcase.Status = ProcessStatus.Running.Value;
subcase.TimeLimit = @case.Timeout.ScaleTo(Context.Language);
var language = Context.Language;
var isUnitTest = Context.CourseProblem.Type == ProblemType.Unittest;
var pipeline = isUnitTest && language.Unittest.Any()
? language.Unittest
: language.Run;
SetPermissions();
var executionResult = ExecuteCommand(new ExecutionCommand
{
Command = $"{string.Join(" ", pipeline)}".ReplaceCommon(Context.MainFileName),
Workdir = Context.DockerTmpWorkdir,
Timeout = @case.Timeout.ScaleTo(language),
Deadline = TimeBank.TimeLeft,
IPath = isUnitTest ? null : $"input/{@case.Id}",
OPath = $"output/{@case.Id}",
EPath = $"error/{@case.Id}"
});
CopyOutputFromDocker(@case);
CopyErrorFromDocker(@case);
CopyFromDocker(".report.json");
CopyFromDocker(".report.simple.json");
foreach (var f in Context.CourseProblem.Export)
{
CopyFromDocker(f);
}
// determine messages and statuses
_evaluationService.EvaluateTypeSolve(executionResult, this, @case);
return(executionResult);
}
public void CopyVerificationFromDocker(CourseProblemCase @case)
{
var dFile = Context.DockerDir.VerificationFile(@case.Id);
var tFile = Context.TmpDir.VerificationFile(@case.Id);
ProcessUtils.Popen($"docker cp \"{ProcessService.ContainerName}:{dFile}.o\" \"{tFile}.o\"");
ProcessUtils.Popen($"docker cp \"{ProcessService.ContainerName}:{dFile}.e\" \"{tFile}.e\"");
}
private void HandleTypeProgram(CourseProblemCase @case, ExecutionResult result)
{
var caseResult = Item.Results.First(i => i.Case == @case.Id);
// reference solution
var reference = Context.CourseProblem.Reference;
var refLanguage = _languageService[reference.Lang];
var args = new List <string>
{
"-v",
"-i", Context.DockerDir.InputFile(@case.Id),
"-o", Context.DockerDir.OutputFile(@case.Id),
};
var pipeline = new List <string>()
.Concat(refLanguage.Run)
.Concat(args)
.ToList();
var verifyResult = ExecuteCommand(new ExecutionCommand
{
Command = $"{string.Join(" ", pipeline)}".ReplaceCommon(reference.Name),
Workdir = Context.DockerDir.VerificationDir,
OPath = $"{@case.Id}.o",
EPath = $"{@case.Id}.e",
});
if (verifyResult.IsBroken)
{
// copy files from tmp
CopyVerificationFromDocker(@case);
var output = File.ReadAllText($"{Context.TmpDir.VerificationDir}/{@case.Id}.o").Trim();
var error = File.ReadAllText($"{Context.TmpDir.VerificationDir}/{@case.Id}.e").Trim();
var messages = new List <string>
{
"Verification failed:"
}
.Concat(output.SplitLines())
.Concat(error.SplitLines())
.Where(i => !string.IsNullOrEmpty(i))
.ToList();
caseResult.Status = ProcessStatus.AnswerWrong.Value;
caseResult.Message = ProcessStatus.AnswerWrong.Description;
caseResult.Messages = messages.ToArray();
}
else
{
caseResult.Status = ProcessStatus.AnswerCorrect.Value;
caseResult.Message = ProcessStatus.AnswerCorrect.Description;
}
}
private async Task <ProcessResult> ProcessCaseMatlabAsync(CourseProblemCase @case, string filename)
{
var uid = Guid.NewGuid().ToString();
var content = _matlabResetTemplate.Replace("{guid}", uid);
var resetFilename = "resetworkspace.m";
await File.WriteAllTextAsync(Context.TmpDir.RootFile(resetFilename), content);
CopyToDocker(resetFilename);
SetPermissions();
return(await _matlabServer.RunFileAsync(filename, uid, resetFilename, Context.DockerTmpWorkdir));
}
private async Task GenerateInputAction(CourseProblemCase @case)
{
var subcase = Item.Results.First(i => i.Case == @case.Id);
if ([email protected]())
{
subcase.Status = ProcessStatus.Skipped.Value;
subcase.Message = "Skipping static input file";
subcase.TimeLimit = 0;
return;
}
// skip global timeouts
if (TimeBank.IsBusted())
{
subcase.SetStatus(ProcessStatus.Skipped);
subcase.Messages = new[] { "No time left" };
subcase.TimeLimit = 0;
return;
}
subcase.Status = ProcessStatus.Running.Value;
subcase.TimeLimit = @case.Timeout.ScaleTo(Context.Language);
var baseCommand = $"{string.Join(" ", Context.Language.Run)}".ReplaceCommon(Context.MainFileName);
var fullCommand = $"{baseCommand} {@case.GetArguments()}";
var generateResult = ExecuteCommand(new ExecutionCommand
{
Command = fullCommand,
Workdir = Context.DockerTmpWorkdir,
Timeout = @case.Timeout.ScaleTo(Context.Language),
Deadline = TimeBank.TimeLeft,
OPath = $"output/{@case.Id}",
EPath = $"error/{@case.Id}",
});
CopyOutputFromDocker(@case);
CopyErrorFromDocker(@case);
CopyToResultDir(Context.TmpDir.OutputDir, Context.ProblemDir.InputDir, false);
_evaluationService.EvaluateTypeGenerate(generateResult, this, @case);
}
public void EvaluateTypeSolve(ExecutionResult executionResult, ProcessItem processItem, CourseProblemCase @case)
{
var context = processItem.Context;
var ccData = processItem.Item;
var subcase = ccData.Results.First(i => i.Case == @case.Id);
subcase.Duration = executionResult.Duration;
subcase.Returncode = executionResult.ReturnCode;
subcase.FullCommand = executionResult.ExecutionCommand.FullCmd;
subcase.Command = executionResult.ExecutionCommand.Command;
processItem.TimeBank.WallTime += executionResult.Duration;
var pythonReportJson = context.TmpDir.RootFile(".report.json");
if (File.Exists(pythonReportJson))
{
var report = PythonReport.FromJson(pythonReportJson.ReadAllText());
ccData.Results.AddRange(
report.Report.Tests.Select(i => new CcDataCaseResult
{
Case = i.Name,
Duration = i.Duration,
Status = i.Outcome == "passed"
? (int)ProcessStatusCodes.AnswerCorrect
: (int)ProcessStatusCodes.AnswerWrong
})
);
File.Delete(pythonReportJson);
}
var simpleReportJson = context.TmpDir.RootFile(".report.simple.json");
if (File.Exists(simpleReportJson))
{
var report = SimpleReport.FromJson(simpleReportJson.ReadAllText());
ccData.Results.AddRange(
report.Tests.Select(i => new CcDataCaseResult
{
Case = i.Name,
Duration = i.Duration,
Message = i.Message,
Status = i.Outcome == "passed"
? (int)ProcessStatusCodes.AnswerCorrect
: (int)ProcessStatusCodes.AnswerWrong
})
);
File.Delete(simpleReportJson);
}
FillBasicInfo(executionResult, processItem, @case);
}
public void EvaluateTypeGenerate(ExecutionResult executionResult, ProcessItem processItem, CourseProblemCase @case)
{
var context = processItem.Context;
var ccData = processItem.Item;
var subcase = ccData.Results.First(i => i.Case == @case.Id);
subcase.Duration = executionResult.Duration;
subcase.Returncode = executionResult.ReturnCode;
subcase.FullCommand = executionResult.ExecutionCommand.FullCmd;
subcase.Command = executionResult.ExecutionCommand.Command;
processItem.TimeBank.WallTime += executionResult.Duration;
var inputFile = context.ProblemDir.InputFile(@case.Id);
switch (executionResult.Code)
{
case ExecutionStatus.Ok:
case ExecutionStatus.OkTimeout:
if (File.Exists(inputFile))
{
subcase.SetStatus(ProcessStatus.Ok);
subcase.Message = $"Input file generated: ({new FileInfo(inputFile).Length} bytes)";
return;
}
subcase.SetStatus(ProcessStatus.ErrorWhileRunning);
subcase.Message = "Program ended with 0 but no input file was generated";
return;
case ExecutionStatus.GlobalTimeout:
subcase.SetStatus(ProcessStatus.GlobalTimeout);
subcase.Messages = new List <string>()
.Concat(executionResult.Messages?.ToArray() ?? new string[] { })
.Concat(context.GetTmpDirErrorMessage(@case.Id).SplitLines())
.ToArray();
return;
case ExecutionStatus.Error:
case ExecutionStatus.ErrorTimeout:
case ExecutionStatus.FatalError:
case ExecutionStatus.NoSuchFile:
subcase.SetStatus(ProcessStatus.ErrorWhileRunning);
subcase.Message = $"[{executionResult.Status}] {executionResult.Message}";
subcase.Messages = new List <string>()
.Concat(executionResult.Messages?.ToArray() ?? new string[] { })
.Concat(context.GetTmpDirErrorMessage(@case.Id).SplitLines())
.ToArray();
return;
default:
throw new Exception(
$"Fatal exception when evaluating result, unknown execution status ${executionResult.Code}");
}
}
private void FillBasicInfo(ExecutionResult executionResult, ProcessItem processItem, CourseProblemCase @case)
{
var language = processItem.Context.Language;
var subcase = processItem.Item.Results.First(i => i.Case == @case.Id);
switch (executionResult.Code)
{
case ExecutionStatus.Ok:
subcase.SetStatus(ProcessStatus.Ok);
return;
case ExecutionStatus.OkTimeout:
subcase.SetStatus(ProcessStatus.Ok);
subcase.Messages = executionResult
.CreateTimeoutMessages(@case.Timeout, language)
.ToArray();
return;
case ExecutionStatus.GlobalTimeout:
subcase.SetStatus(ProcessStatus.GlobalTimeout);
subcase.Messages = executionResult
.CreateDeadlineMessages(processItem.TimeBank)
.ToArray();
return;
case ExecutionStatus.NoSuchFile:
subcase.SetStatus(ProcessStatus.ErrorWhileRunning);
subcase.Messages = new List <string>
{
"Error while starting the process (no such file)", executionResult.Message
}
.ToArray();
return;
case ExecutionStatus.ErrorTimeout:
subcase.SetStatus(ProcessStatus.ErrorWhileRunning);
subcase.Message = executionResult.Message;
subcase.Messages = new List <string>()
.Concat(executionResult.Messages)
.Concat(new[] { "----", "Also, program did not finish in time" })
.Concat(executionResult.CreateTimeoutMessages(@case.Timeout, language))
.ToArray();
return;
case ExecutionStatus.Error:
case ExecutionStatus.FatalError:
subcase.SetStatus(ProcessStatus.ErrorWhileRunning);
subcase.Message = executionResult.Message;
subcase.Messages = executionResult.Messages?.ToArray();
return;
default:
throw new Exception(
$"Fatal exception when evaluating result, unknown execution status ${executionResult.Code}");
}
}
private async Task GenerateOutputAction(CourseProblemCase @case)
{
await SolveCaseBaseAsync(@case);
}
public ProcessResult CopyErrorFromDocker(CourseProblemCase @case)
{
var cpCommand = $"docker cp \"{ProcessService.ContainerName}:{Context.DockerTmpWorkdir}/error/{@case.Id}\" \"{Context.TmpDir.ErrorDir}\"";
return(ProcessUtils.Popen(cpCommand));
}
