public ExecutionResult Execute(ExecutionContext executionContext)
{
throw new NotImplementedException();
/*
var request = new PayloadSettings
{
Id = executionContext.SubmissionId,
TimeLimit = executionContext.TimeLimit,
MemoryLimit = executionContext.MemoryLimit,
Payload = this.PreparePayloadFile(executionContext)
};
var client = new HttpClient();
var message = new HttpRequestMessage(HttpMethod.Post, "http://79.124.67.13:9000/execute");
client.SendAsync(message).ContinueWith(
(response =>
{
var serializer = new JsonSerializer();
serializer.Deserialize<ServerResponse>(
new JsonReader { FloatParseHandling = FloatParseHandling.Decimal });
// TODO: Deserialize response
}));
// TODO: Parse response
*/
}
protected ExecutionResult CompileExecuteAndCheck(ExecutionContext executionContext, Func<CompilerType, string> getCompilerPathFunc, IExecutor executor)
{
var result = new ExecutionResult();
// Compile the file
var compilerPath = getCompilerPathFunc(executionContext.CompilerType);
var compilerResult = this.Compile(executionContext.CompilerType, compilerPath, executionContext.AdditionalCompilerArguments, executionContext.Code);
result.IsCompiledSuccessfully = compilerResult.IsCompiledSuccessfully;
result.CompilerComment = compilerResult.CompilerComment;
if (!compilerResult.IsCompiledSuccessfully)
{
return result;
}
var outputFile = compilerResult.OutputFile;
// Execute and check each test
IChecker checker = Checker.CreateChecker(executionContext.CheckerAssemblyName, executionContext.CheckerTypeName, executionContext.CheckerParameter);
foreach (var test in executionContext.Tests)
{
var processExecutionResult = executor.Execute(outputFile, test.Input, executionContext.TimeLimit, executionContext.MemoryLimit);
var testResult = this.ExecuteAndCheckTest(test, processExecutionResult, checker, processExecutionResult.ReceivedOutput);
result.TestResults.Add(testResult);
}
// Clean our mess
File.Delete(outputFile);
return result;
}
public override ExecutionResult Execute(ExecutionContext executionContext)
{
var result = new ExecutionResult();
// PHP code is not compiled
result.IsCompiledSuccessfully = true;
var codeSavePath = FileHelpers.SaveStringToTempFile(executionContext.Code);
// Process the submission and check each test
var executor = new RestrictedProcessExecutor();
var checker = Checker.CreateChecker(executionContext.CheckerAssemblyName, executionContext.CheckerTypeName, executionContext.CheckerParameter);
result.TestResults = new List<TestResult>();
foreach (var test in executionContext.Tests)
{
var processExecutionResult = executor.Execute(
this.phpCliExecutablePath,
test.Input,
executionContext.TimeLimit,
executionContext.MemoryLimit,
new[] { codeSavePath });
var testResult = this.ExecuteAndCheckTest(test, processExecutionResult, checker, processExecutionResult.ReceivedOutput);
result.TestResults.Add(testResult);
}
// Clean up
File.Delete(codeSavePath);
return result;
}
public ExecutionResult Execute(ExecutionContext executionContext)
{
// User code = user queries - prepare database
// Test input = query to execute
// Test output = expected result from the query
throw new NotImplementedException();
}
public ExecutionResult Execute(ExecutionContext executionContext)
{
return new ExecutionResult
{
CompilerComment = null,
IsCompiledSuccessfully = true,
TestResults = new List<TestResult>(),
};
}
protected override CompileResult DoCompile(ExecutionContext executionContext, string submissionFilePath)
{
var compilerPath = this.GetCompilerPathFunc(executionContext.CompilerType);
// Compile the zip file with user code and sandbox executor
var compilerResult = this.Compile(
executionContext.CompilerType,
compilerPath,
executionContext.AdditionalCompilerArguments,
submissionFilePath);
return compilerResult;
}
protected override string CreateSubmissionFile(ExecutionContext executionContext)
{
var trimmedAllowedFileExtensions = executionContext.AllowedFileExtensions?.Trim();
var allowedFileExtensions = (!trimmedAllowedFileExtensions?.StartsWith(".") ?? false)
? $".{trimmedAllowedFileExtensions}"
: trimmedAllowedFileExtensions;
if (allowedFileExtensions != GlobalConstants.ZipFileExtension)
{
throw new ArgumentException("Submission file is not a zip file!");
}
return this.PrepareSubmissionFile(executionContext.FileContent);
}
protected override List<TestResult> ProcessTests(ExecutionContext executionContext, IExecutor executor, IChecker checker, string codeSavePath)
{
var testResults = new List<TestResult>();
var arguments = new List<string>();
arguments.Add(this.mochaModulePath);
arguments.Add(codeSavePath);
arguments.AddRange(executionContext.AdditionalCompilerArguments.Split(' '));
foreach (var test in executionContext.Tests)
{
var processExecutionResult = executor.Execute(this.NodeJsExecutablePath, test.Input, executionContext.TimeLimit, executionContext.MemoryLimit, arguments);
var mochaResult = JsonExecutionResult.Parse(processExecutionResult.ReceivedOutput);
var testResult = this.ExecuteAndCheckTest(test, processExecutionResult, checker, mochaResult.Passed ? "yes" : string.Format("Unexpected error: {0}", mochaResult.Error));
testResults.Add(testResult);
}
return testResults;
}
private byte[] PreparePayloadFile(ExecutionContext executionContext)
{
var tempFile = Path.GetTempFileName();
tempFile += ".zip"; // TODO: Useless?
var zip = new ZipFile(tempFile);
zip.AddEntry("userCode.deflate", executionContext.FileContent);
zip.AddDirectory("tests");
foreach (var test in executionContext.Tests)
{
zip.AddEntry(string.Format("/tests/{0}", test.Id), test.Input);
}
// TODO: Optimize?
var stream = new MemoryStream();
zip.Save(stream);
stream.Position = 0;
var byteArray = stream.ToArray();
return byteArray;
}
public override ExecutionResult Execute(ExecutionContext executionContext)
{
var result = new ExecutionResult();
// setting the IsCompiledSuccessfully variable to true as in the NodeJS
// execution strategy there is no compilation
result.IsCompiledSuccessfully = true;
// Preprocess the user submission
var codeToExecute = this.PreprocessJsSubmission(this.JsCodeTemplate, executionContext.Code);
// Save the preprocessed submission which is ready for execution
var codeSavePath = this.SaveStringToTempFile(codeToExecute);
// Process the submission and check each test
IExecutor executor = new RestrictedProcessExecutor();
IChecker checker = Checker.CreateChecker(executionContext.CheckerAssemblyName, executionContext.CheckerTypeName, executionContext.CheckerParameter);
result.TestResults = new List<TestResult>();
foreach (var test in executionContext.Tests)
{
var processExecutionResult = executor.Execute(this.nodeJsExecutablePath, test.Input, executionContext.TimeLimit, executionContext.MemoryLimit, new[] { codeSavePath });
var testResult = this.ExecuteAndCheckTest(test, processExecutionResult, checker, processExecutionResult.ReceivedOutput);
result.TestResults.Add(testResult);
}
// Clean up
File.Delete(codeSavePath);
return result;
}
public abstract ExecutionResult Execute(ExecutionContext executionContext);
public override ExecutionResult Execute(ExecutionContext executionContext)
{
var result = new ExecutionResult();
// setting the IsCompiledSuccessfully variable to true as in the NodeJS
// execution strategy there is no compilation
result.IsCompiledSuccessfully = true;
// Preprocess the user submission
var codeToExecute = this.PreprocessJsSubmission(this.JsCodeTemplate, executionContext.Code);
// Save the preprocessed submission which is ready for execution
var codeSavePath = FileHelpers.SaveStringToTempFile(codeToExecute);
// Process the submission and check each test
IExecutor executor = new RestrictedProcessExecutor();
IChecker checker = Checker.CreateChecker(executionContext.CheckerAssemblyName, executionContext.CheckerTypeName, executionContext.CheckerParameter);
result.TestResults = this.ProcessTests(executionContext, executor, checker, codeSavePath);
// Clean up
File.Delete(codeSavePath);
return result;
}
protected virtual List<TestResult> ProcessTests(ExecutionContext executionContext, IExecutor executor, IChecker checker, string codeSavePath)
{
var testResults = new List<TestResult>();
foreach (var test in executionContext.Tests)
{
var processExecutionResult = executor.Execute(this.NodeJsExecutablePath, test.Input, executionContext.TimeLimit, executionContext.MemoryLimit, new[] { codeSavePath });
var testResult = this.ExecuteAndCheckTest(test, processExecutionResult, checker, processExecutionResult.ReceivedOutput);
testResults.Add(testResult);
}
return testResults;
}
protected virtual string CreateSubmissionFile(ExecutionContext executionContext) =>
JavaCodePreprocessorHelper.CreateSubmissionFile(executionContext.Code, this.WorkingDirectory);
protected virtual CompileResult DoCompile(ExecutionContext executionContext, string submissionFilePath)
{
var compilerPath = this.GetCompilerPathFunc(executionContext.CompilerType);
// Compile all source files - sandbox executor and submission file
var compilerResult = this.CompileSourceFiles(
executionContext.CompilerType,
compilerPath,
executionContext.AdditionalCompilerArguments,
new[] { this.SandboxExecutorSourceFilePath, submissionFilePath });
return compilerResult;
}
protected virtual string CreateSubmissionFile(ExecutionContext executionContext)
{
var submissionCode = executionContext.Code;
// Remove existing packages
submissionCode = Regex.Replace(submissionCode, PackageNameRegEx, string.Empty);
// TODO: Remove the restriction for one public class - a non-public Java class can contain the main method!
var classNameMatch = Regex.Match(submissionCode, ClassNameRegEx);
if (!classNameMatch.Success)
{
throw new ArgumentException("No valid public class found!");
}
var className = classNameMatch.Groups[ClassNameRegExGroup].Value;
var submissionFilePath = $"{this.WorkingDirectory}\\{className}";
File.WriteAllText(submissionFilePath, submissionCode);
return submissionFilePath;
}
public override ExecutionResult Execute(ExecutionContext executionContext)
{
var result = new ExecutionResult();
// Copy the sandbox executor source code to a file in the working directory
File.WriteAllText(this.SandboxExecutorSourceFilePath, this.SandboxExecutorCode);
// Create a temp file with the submission code
string submissionFilePath;
try
{
submissionFilePath = this.CreateSubmissionFile(executionContext);
}
catch (ArgumentException exception)
{
result.IsCompiledSuccessfully = false;
result.CompilerComment = exception.Message;
return result;
}
var compilerResult = this.DoCompile(executionContext, submissionFilePath);
// Assign compiled result info to the execution result
result.IsCompiledSuccessfully = compilerResult.IsCompiledSuccessfully;
result.CompilerComment = compilerResult.CompilerComment;
if (!result.IsCompiledSuccessfully)
{
return result;
}
// Prepare execution process arguments and time measurement info
var classPathArgument = $"-classpath \"{this.WorkingDirectory}\"";
var classToExecuteFilePath = compilerResult.OutputFile;
var classToExecute = classToExecuteFilePath
.Substring(
this.WorkingDirectory.Length + 1,
classToExecuteFilePath.Length - this.WorkingDirectory.Length - JavaCompiledFileExtension.Length - 1)
.Replace('\\', '.');
var timeMeasurementFilePath = $"{this.WorkingDirectory}\\{TimeMeasurementFileName}";
// Create an executor and a checker
var executor = new StandardProcessExecutor();
var checker = Checker.CreateChecker(executionContext.CheckerAssemblyName, executionContext.CheckerTypeName, executionContext.CheckerParameter);
// Process the submission and check each test
foreach (var test in executionContext.Tests)
{
var processExecutionResult = executor.Execute(
this.javaExecutablePath,
test.Input,
executionContext.TimeLimit * 2, // Java virtual machine takes more time to start up
executionContext.MemoryLimit,
new[] { classPathArgument, SandboxExecutorClassName, classToExecute, $"\"{timeMeasurementFilePath}\"" });
UpdateExecutionTime(timeMeasurementFilePath, processExecutionResult, executionContext.TimeLimit);
var testResult = this.ExecuteAndCheckTest(test, processExecutionResult, checker, processExecutionResult.ReceivedOutput);
result.TestResults.Add(testResult);
}
return result;
}
public override ExecutionResult Execute(ExecutionContext executionContext)
{
IExecutor executor = new RestrictedProcessExecutor();
var result = this.CompileExecuteAndCheck(executionContext, this.getCompilerPathFunc, executor);
return result;
}
public override ExecutionResult Execute(ExecutionContext executionContext)
{
var result = new ExecutionResult();
// Create a temp file with the submission code
string submissionFilePath;
try
{
submissionFilePath = this.CreateSubmissionFile(executionContext.Code);
}
catch (ArgumentException exception)
{
result.IsCompiledSuccessfully = false;
result.CompilerComment = exception.Message;
return result;
}
// Create a sandbox executor source file in the working directory
var sandboxExecutorSourceFilePath = $"{this.workingDirectory}\\{SandboxExecutorClassName}";
File.WriteAllText(sandboxExecutorSourceFilePath, this.SandboxExecutorCode);
// Compile all source files - sandbox executor and submission file
var compilerPath = this.getCompilerPathFunc(executionContext.CompilerType);
var compilerResult = this.CompileSourceFiles(
executionContext.CompilerType,
compilerPath,
executionContext.AdditionalCompilerArguments,
new[] { submissionFilePath, sandboxExecutorSourceFilePath });
// Assign compiled result info to the execution result
result.IsCompiledSuccessfully = compilerResult.IsCompiledSuccessfully;
result.CompilerComment = compilerResult.CompilerComment;
if (!result.IsCompiledSuccessfully)
{
return result;
}
// Prepare execution process arguments and time measurement info
var classPathArgument = $"-classpath \"{this.workingDirectory}\"";
var submissionFilePathLastIndexOfSlash = submissionFilePath.LastIndexOf('\\');
var classToExecute = submissionFilePath.Substring(submissionFilePathLastIndexOfSlash + 1);
var timeMeasurementFilePath = $"{this.workingDirectory}\\{TimeMeasurementFileName}";
// Create an executor and a checker
var executor = new StandardProcessExecutor();
var checker = Checker.CreateChecker(executionContext.CheckerAssemblyName, executionContext.CheckerTypeName, executionContext.CheckerParameter);
// Process the submission and check each test
foreach (var test in executionContext.Tests)
{
var processExecutionResult = executor.Execute(
this.javaExecutablePath,
test.Input,
executionContext.TimeLimit,
executionContext.MemoryLimit,
new[] { classPathArgument, SandboxExecutorClassName, classToExecute, $"\"{timeMeasurementFilePath}\"" });
UpdateExecutionTime(timeMeasurementFilePath, processExecutionResult, executionContext.TimeLimit);
var testResult = this.ExecuteAndCheckTest(test, processExecutionResult, checker, processExecutionResult.ReceivedOutput);
result.TestResults.Add(testResult);
}
return result;
}
public override ExecutionResult Execute(ExecutionContext executionContext)
{
var result = new ExecutionResult();
// Create a temp file with the submission code
string submissionFilePath;
try
{
submissionFilePath = this.CreateSubmissionFile(executionContext.Code);
}
catch (ArgumentException exception)
{
result.IsCompiledSuccessfully = false;
result.CompilerComment = exception.Message;
return(result);
}
// Create a sandbox executor source file in the working directory
var sandboxExecutorSourceFilePath = string.Format("{0}\\{1}.{2}", this.workingDirectory, SandboxExecutorClassName, CompilerType.Java.GetFileExtension());
File.WriteAllText(sandboxExecutorSourceFilePath, this.SandboxExecutorCode);
// Compile all source files - sandbox executor and submission file(s)
var compilerPath = this.getCompilerPathFunc(executionContext.CompilerType);
var sourceFilesToCompile = new DirectoryInfo(this.workingDirectory).GetFiles(JavaSourceFilesPattern).Select(x => x.FullName);
var compilerResult = this.CompileSourceFiles(
executionContext.CompilerType,
compilerPath,
executionContext.AdditionalCompilerArguments,
sourceFilesToCompile);
// Assign compiled result info to the execution result
result.IsCompiledSuccessfully = compilerResult.IsCompiledSuccessfully;
result.CompilerComment = compilerResult.CompilerComment;
if (!result.IsCompiledSuccessfully)
{
return(result);
}
// Prepare execution process arguments and time measurement info
var classPathArgument = string.Format("-classpath \"{0}\"", this.workingDirectory);
var submissionFilePathLastIndexOfSlash = submissionFilePath.LastIndexOf('\\');
var submissionFilePathLastIndexOfDot = submissionFilePath.LastIndexOf('.');
var classToExecute = submissionFilePath.Substring(
submissionFilePathLastIndexOfSlash + 1,
submissionFilePathLastIndexOfDot - submissionFilePathLastIndexOfSlash - 1);
var timeMeasurementFilePath = string.Format("{0}\\{1}", this.workingDirectory, TimeMeasurementFileName);
// Create an executor and a checker
var executor = new StandardProcessExecutor();
var checker = Checker.CreateChecker(executionContext.CheckerAssemblyName, executionContext.CheckerTypeName, executionContext.CheckerParameter);
// Process the submission and check each test
foreach (var test in executionContext.Tests)
{
var processExecutionResult = executor.Execute(
this.javaExecutablePath,
test.Input,
executionContext.TimeLimit,
executionContext.MemoryLimit,
new[] { classPathArgument, SandboxExecutorClassName, classToExecute, string.Format("\"{0}\"", timeMeasurementFilePath) });
UpdateExecutionTime(timeMeasurementFilePath, processExecutionResult, executionContext.TimeLimit);
var testResult = this.ExecuteAndCheckTest(test, processExecutionResult, checker, processExecutionResult.ReceivedOutput);
result.TestResults.Add(testResult);
}
return(result);
}
protected CompileResult ExecuteCompiling(ExecutionContext executionContext, Func<CompilerType, string> getCompilerPathFunc, ExecutionResult result)
{
var submissionFilePath = string.IsNullOrEmpty(executionContext.AllowedFileExtensions)
? FileHelpers.SaveStringToTempFile(executionContext.Code)
: FileHelpers.SaveByteArrayToTempFile(executionContext.FileContent);
var compilerPath = getCompilerPathFunc(executionContext.CompilerType);
var compilerResult = this.Compile(executionContext.CompilerType, compilerPath, executionContext.AdditionalCompilerArguments, submissionFilePath);
result.IsCompiledSuccessfully = compilerResult.IsCompiledSuccessfully;
result.CompilerComment = compilerResult.CompilerComment;
return compilerResult;
}
private ExecutionResult RunUnitTests(
ExecutionContext executionContext,
IExecutor executor,
IChecker checker,
ExecutionResult result,
string csProjFilePath)
{
var compileDirectory = Path.GetDirectoryName(csProjFilePath);
var fileName = Path.GetFileName(csProjFilePath);
if (!string.IsNullOrEmpty(fileName))
{
fileName = fileName.Replace(ProjectFileExtenstion, DllFileExtension);
}
var targetDll = $"{compileDirectory}\\{CompiledResultFolder}{fileName}";
var arguments = new List <string> {
targetDll
};
arguments.AddRange(executionContext.AdditionalCompilerArguments.Split(' '));
var testedCodePath = $"{compileDirectory}\\{TestedCode}";
var originalTestsPassed = -1;
var count = 0;
foreach (var test in executionContext.Tests)
{
File.WriteAllText(testedCodePath, test.Input);
var project = new Project(csProjFilePath);
var didCompile = project.Build();
project.ProjectCollection.UnloadAllProjects();
if (!didCompile)
{
result.IsCompiledSuccessfully = false;
return(result);
}
var processExecutionResult = executor.Execute(
this.NUnitConsoleRunnerPath,
string.Empty,
executionContext.TimeLimit,
executionContext.MemoryLimit,
arguments);
var totalTests = 0;
var passedTests = 0;
this.ExtractTestResult(processExecutionResult.ReceivedOutput, out passedTests, out totalTests);
var message = "Test Passed!";
if (totalTests == 0)
{
message = "No tests found";
}
else if (passedTests == originalTestsPassed)
{
message = "No functionality covering this test!";
}
if (count == 0)
{
originalTestsPassed = passedTests;
if (totalTests != passedTests)
{
message = "Not all tests passed on the correct solution.";
}
}
var testResult = this.ExecuteAndCheckTest(test, processExecutionResult, checker, message);
File.Delete(testedCodePath);
result.TestResults.Add(testResult);
count++;
}
return(result);
}
private void ProcessSubmission(Submission submission)
{
// TODO: Check for N+1 queries problem
this.logger.InfoFormat("Work on submission №{0} started.", submission.Id);
var executionStrategy = this.CreateExecutionStrategy(submission.SubmissionType.ExecutionStrategyType);
var context = new ExecutionContext
{
SubmissionId = submission.Id,
AdditionalCompilerArguments = submission.SubmissionType.AdditionalCompilerArguments,
CheckerAssemblyName = submission.Problem.Checker.DllFile,
CheckerParameter = submission.Problem.Checker.Parameter,
CheckerTypeName = submission.Problem.Checker.ClassName,
FileContent = submission.Content,
AllowedFileExtensions = submission.SubmissionType.AllowedFileExtensions,
CompilerType = submission.SubmissionType.CompilerType,
MemoryLimit = submission.Problem.MemoryLimit,
TimeLimit = submission.Problem.TimeLimit,
};
context.Tests = submission.Problem.Tests.ToList().Select(x => new TestContext
{
Id = x.Id,
Input = x.InputDataAsString,
Output = x.OutputDataAsString,
IsTrialTest = x.IsTrialTest
});
ExecutionResult executionResult;
try
{
executionResult = executionStrategy.Execute(context);
}
catch (Exception exception)
{
this.logger.ErrorFormat("executionStrategy.Execute on submission №{0} has thrown an exception: {1}", submission.Id, exception);
submission.ProcessingComment = string.Format("Exception in executionStrategy.Execute: {0}", exception.Message);
return;
}
submission.IsCompiledSuccessfully = executionResult.IsCompiledSuccessfully;
submission.CompilerComment = executionResult.CompilerComment;
if (!executionResult.IsCompiledSuccessfully)
{
return;
}
foreach (var testResult in executionResult.TestResults)
{
var testRun = new TestRun
{
CheckerComment = testResult.CheckerDetails.Comment,
ExpectedOutputFragment = testResult.CheckerDetails.ExpectedOutputFragment,
UserOutputFragment = testResult.CheckerDetails.UserOutputFragment,
ExecutionComment = testResult.ExecutionComment,
MemoryUsed = testResult.MemoryUsed,
ResultType = testResult.ResultType,
TestId = testResult.Id,
TimeUsed = testResult.TimeUsed,
};
submission.TestRuns.Add(testRun);
}
this.logger.InfoFormat("Work on submission №{0} ended.", submission.Id);
}
public override ExecutionResult Execute(ExecutionContext executionContext)
{
var solution = executionContext.FileContent;
var result = new ExecutionResult();
var compileResult = this.ExecuteCompiling(executionContext, this.getCompilerPathFunc, result);
if (!compileResult.IsCompiledSuccessfully)
{
return result;
}
var outputAssemblyPath = this.PreprocessAndCompileTestRunner(executionContext, compileResult.OutputFile);
IExecutor executor = new RestrictedProcessExecutor();
var processExecutionResult = executor.Execute(outputAssemblyPath, string.Empty, executionContext.TimeLimit, executionContext.MemoryLimit);
var workingDirectory = compileResult.OutputFile;
if (Directory.Exists(workingDirectory))
{
Directory.Delete(workingDirectory, true);
}
this.ProcessTests(processExecutionResult, executionContext, result);
return result;
}
private string PreprocessAndCompileTestRunner(ExecutionContext executionContext, string outputDirectory)
{
var testStrings = new List<string>();
foreach (var test in executionContext.Tests)
{
testStrings.Add(TestTemplate.Replace("#testInput#", test.Input));
}
var allTests = string.Join(",", testStrings);
var testRunnerCode = TestRunnerTemplate.Replace("#allTests#", allTests);
var compiler = new CSharpCodeProvider();
var compilerParameters = new CompilerParameters();
compilerParameters.ReferencedAssemblies.Add("mscorlib.dll");
compilerParameters.ReferencedAssemblies.Add("System.dll");
compilerParameters.ReferencedAssemblies.Add("System.Core.dll");
compilerParameters.ReferencedAssemblies.AddRange(Directory.GetFiles(outputDirectory).Where(f => f.EndsWith(".dll") || f.EndsWith(".exe")).ToArray());
compilerParameters.GenerateInMemory = false;
compilerParameters.GenerateExecutable = true;
var compilerResult = compiler.CompileAssemblyFromSource(compilerParameters, testRunnerCode);
var outputAssemblyPath = outputDirectory + "\\LocalTestRunner.exe";
File.Move(compilerResult.PathToAssembly, outputAssemblyPath);
return outputAssemblyPath;
}
private void ProcessTests(ProcessExecutionResult processExecutionResult, ExecutionContext executionContext, ExecutionResult result)
{
var jsonResult = JsonExecutionResult.Parse(processExecutionResult.ReceivedOutput, true, true);
var index = 0;
result.TestResults = new List<TestResult>();
foreach (var test in executionContext.Tests)
{
var testResult = new TestResult
{
Id = test.Id,
TimeUsed = (int)processExecutionResult.TimeWorked.TotalMilliseconds,
MemoryUsed = (int)processExecutionResult.MemoryUsed,
};
if (jsonResult.PassingIndexes.Contains(index))
{
testResult.ResultType = TestRunResultType.CorrectAnswer;
}
else
{
testResult.ResultType = TestRunResultType.WrongAnswer;
testResult.CheckerDetails = new CheckerDetails { Comment = "Test failed." };
}
result.TestResults.Add(testResult);
index++;
}
}
private void ProcessSubmission(Submission submission)
{
// TODO: Check for N+1 queries problem
this.logger.InfoFormat("Work on submission №{0} started.", submission.Id);
IExecutionStrategy executionStrategy = this.CreateExecutionStrategy(submission.SubmissionType.ExecutionStrategyType);
var context = new ExecutionContext
{
AdditionalCompilerArguments = submission.SubmissionType.AdditionalCompilerArguments,
CheckerAssemblyName = submission.Problem.Checker.DllFile,
CheckerParameter = submission.Problem.Checker.Parameter,
CheckerTypeName = submission.Problem.Checker.ClassName,
FileContent = submission.Content,
AllowedFileExtensions = submission.SubmissionType.AllowedFileExtensions,
CompilerType = submission.SubmissionType.CompilerType,
MemoryLimit = submission.Problem.MemoryLimit,
TimeLimit = submission.Problem.TimeLimit,
};
context.Tests = submission.Problem.Tests.ToList().Select(x => new TestContext
{
Id = x.Id,
Input = x.InputDataAsString,
Output = x.OutputDataAsString,
IsTrialTest = x.IsTrialTest
});
ExecutionResult executionResult;
try
{
executionResult = executionStrategy.Execute(context);
}
catch (Exception exception)
{
this.logger.ErrorFormat("executionStrategy.Execute on submission №{0} has thrown an exception: {1}", submission.Id, exception);
submission.ProcessingComment = string.Format("Exception in executionStrategy.Execute: {0}", exception.Message);
return;
}
submission.IsCompiledSuccessfully = executionResult.IsCompiledSuccessfully;
submission.CompilerComment = executionResult.CompilerComment;
if (!executionResult.IsCompiledSuccessfully)
{
return;
}
foreach (var testResult in executionResult.TestResults)
{
var testRun = new TestRun
{
CheckerComment = testResult.CheckerComment,
ExecutionComment = testResult.ExecutionComment,
MemoryUsed = testResult.MemoryUsed,
ResultType = testResult.ResultType,
TestId = testResult.Id,
TimeUsed = testResult.TimeUsed,
};
submission.TestRuns.Add(testRun);
}
this.logger.InfoFormat("Work on submission №{0} ended.", submission.Id);
}
protected virtual List <TestResult> ProcessTests(
ExecutionContext executionContext,
IExecutor executor,
IChecker checker,
string codeSavePath,
int numberOfUserTests)
{
var testResults = new List <TestResult>();
var arguments = new List <string>();
arguments.Add(this.MochaModulePath);
arguments.Add(codeSavePath);
arguments.AddRange(executionContext.AdditionalCompilerArguments.Split(' '));
var testCount = 0;
var processExecutionResult = this.ExecuteNodeJsProcess(executionContext, executor, string.Empty, arguments);
var mochaResult = JsonExecutionResult.Parse(processExecutionResult.ReceivedOutput);
var correctSolutionTestPasses = mochaResult.TestsErrors.Take(numberOfUserTests).Count(x => x == null);
var testOffset = numberOfUserTests;
foreach (var test in executionContext.Tests)
{
TestResult testResult = null;
if (testCount == 0)
{
var minTestCount = int.Parse(
Regex.Match(
test.Input,
"<minTestCount>(\\d+)</minTestCount>").Groups[1].Value);
var message = "Test Passed!";
if (numberOfUserTests == 0)
{
message = "The submitted code was either incorrect or contained no tests!";
}
else if (numberOfUserTests < minTestCount)
{
message = $"Insufficient amount of tests, you have to have atleast {minTestCount} tests!";
}
testResult = this.ExecuteAndCheckTest(
test,
processExecutionResult,
checker,
message);
}
else if (testCount == 1)
{
var message = "Test Passed!";
if (numberOfUserTests == 0)
{
message = "The submitted code was either incorrect or contained no tests!";
}
else if (correctSolutionTestPasses != numberOfUserTests)
{
message = "Error: Some tests failed while running the correct solution!";
}
testResult = this.ExecuteAndCheckTest(
test,
processExecutionResult,
checker,
message);
}
else
{
var numberOfPasses = mochaResult.TestsErrors.Skip(testOffset).Take(numberOfUserTests).Count(x => x == null);
var message = "Test Passed!";
if (numberOfPasses >= correctSolutionTestPasses)
{
message = "No unit test covering this functionality!";
mochaResult.TestsErrors
.Take(numberOfUserTests)
.Where(x => x != null)
.ForEach(x => message += x);
}
testResult = this.ExecuteAndCheckTest(
test,
processExecutionResult,
checker,
message);
testOffset += numberOfUserTests;
}
testCount++;
testResults.Add(testResult);
}
return(testResults);
}
