public ExecutionResult Execute(ExecutionContext executionContext) { throw new NotImplementedException(); /* var request = new PayloadSettings { Id = executionContext.SubmissionId, TimeLimit = executionContext.TimeLimit, MemoryLimit = executionContext.MemoryLimit, Payload = this.PreparePayloadFile(executionContext) }; var client = new HttpClient(); var message = new HttpRequestMessage(HttpMethod.Post, "http://79.124.67.13:9000/execute"); client.SendAsync(message).ContinueWith( (response => { var serializer = new JsonSerializer(); serializer.Deserialize<ServerResponse>( new JsonReader { FloatParseHandling = FloatParseHandling.Decimal }); // TODO: Deserialize response })); // TODO: Parse response */ }
protected ExecutionResult CompileExecuteAndCheck(ExecutionContext executionContext, Func<CompilerType, string> getCompilerPathFunc, IExecutor executor) { var result = new ExecutionResult(); // Compile the file var compilerPath = getCompilerPathFunc(executionContext.CompilerType); var compilerResult = this.Compile(executionContext.CompilerType, compilerPath, executionContext.AdditionalCompilerArguments, executionContext.Code); result.IsCompiledSuccessfully = compilerResult.IsCompiledSuccessfully; result.CompilerComment = compilerResult.CompilerComment; if (!compilerResult.IsCompiledSuccessfully) { return result; } var outputFile = compilerResult.OutputFile; // Execute and check each test IChecker checker = Checker.CreateChecker(executionContext.CheckerAssemblyName, executionContext.CheckerTypeName, executionContext.CheckerParameter); foreach (var test in executionContext.Tests) { var processExecutionResult = executor.Execute(outputFile, test.Input, executionContext.TimeLimit, executionContext.MemoryLimit); var testResult = this.ExecuteAndCheckTest(test, processExecutionResult, checker, processExecutionResult.ReceivedOutput); result.TestResults.Add(testResult); } // Clean our mess File.Delete(outputFile); return result; }
public override ExecutionResult Execute(ExecutionContext executionContext) { var result = new ExecutionResult(); // PHP code is not compiled result.IsCompiledSuccessfully = true; var codeSavePath = FileHelpers.SaveStringToTempFile(executionContext.Code); // Process the submission and check each test var executor = new RestrictedProcessExecutor(); var checker = Checker.CreateChecker(executionContext.CheckerAssemblyName, executionContext.CheckerTypeName, executionContext.CheckerParameter); result.TestResults = new List<TestResult>(); foreach (var test in executionContext.Tests) { var processExecutionResult = executor.Execute( this.phpCliExecutablePath, test.Input, executionContext.TimeLimit, executionContext.MemoryLimit, new[] { codeSavePath }); var testResult = this.ExecuteAndCheckTest(test, processExecutionResult, checker, processExecutionResult.ReceivedOutput); result.TestResults.Add(testResult); } // Clean up File.Delete(codeSavePath); return result; }
public ExecutionResult Execute(ExecutionContext executionContext) { // User code = user queries - prepare database // Test input = query to execute // Test output = expected result from the query throw new NotImplementedException(); }
public ExecutionResult Execute(ExecutionContext executionContext) { return new ExecutionResult { CompilerComment = null, IsCompiledSuccessfully = true, TestResults = new List<TestResult>(), }; }
protected override CompileResult DoCompile(ExecutionContext executionContext, string submissionFilePath) { var compilerPath = this.GetCompilerPathFunc(executionContext.CompilerType); // Compile the zip file with user code and sandbox executor var compilerResult = this.Compile( executionContext.CompilerType, compilerPath, executionContext.AdditionalCompilerArguments, submissionFilePath); return compilerResult; }
protected override string CreateSubmissionFile(ExecutionContext executionContext) { var trimmedAllowedFileExtensions = executionContext.AllowedFileExtensions?.Trim(); var allowedFileExtensions = (!trimmedAllowedFileExtensions?.StartsWith(".") ?? false) ? $".{trimmedAllowedFileExtensions}" : trimmedAllowedFileExtensions; if (allowedFileExtensions != GlobalConstants.ZipFileExtension) { throw new ArgumentException("Submission file is not a zip file!"); } return this.PrepareSubmissionFile(executionContext.FileContent); }
protected override List<TestResult> ProcessTests(ExecutionContext executionContext, IExecutor executor, IChecker checker, string codeSavePath) { var testResults = new List<TestResult>(); var arguments = new List<string>(); arguments.Add(this.mochaModulePath); arguments.Add(codeSavePath); arguments.AddRange(executionContext.AdditionalCompilerArguments.Split(' ')); foreach (var test in executionContext.Tests) { var processExecutionResult = executor.Execute(this.NodeJsExecutablePath, test.Input, executionContext.TimeLimit, executionContext.MemoryLimit, arguments); var mochaResult = JsonExecutionResult.Parse(processExecutionResult.ReceivedOutput); var testResult = this.ExecuteAndCheckTest(test, processExecutionResult, checker, mochaResult.Passed ? "yes" : string.Format("Unexpected error: {0}", mochaResult.Error)); testResults.Add(testResult); } return testResults; }
private byte[] PreparePayloadFile(ExecutionContext executionContext) { var tempFile = Path.GetTempFileName(); tempFile += ".zip"; // TODO: Useless? var zip = new ZipFile(tempFile); zip.AddEntry("userCode.deflate", executionContext.FileContent); zip.AddDirectory("tests"); foreach (var test in executionContext.Tests) { zip.AddEntry(string.Format("/tests/{0}", test.Id), test.Input); } // TODO: Optimize? var stream = new MemoryStream(); zip.Save(stream); stream.Position = 0; var byteArray = stream.ToArray(); return byteArray; }
public override ExecutionResult Execute(ExecutionContext executionContext) { var result = new ExecutionResult(); // setting the IsCompiledSuccessfully variable to true as in the NodeJS // execution strategy there is no compilation result.IsCompiledSuccessfully = true; // Preprocess the user submission var codeToExecute = this.PreprocessJsSubmission(this.JsCodeTemplate, executionContext.Code); // Save the preprocessed submission which is ready for execution var codeSavePath = this.SaveStringToTempFile(codeToExecute); // Process the submission and check each test IExecutor executor = new RestrictedProcessExecutor(); IChecker checker = Checker.CreateChecker(executionContext.CheckerAssemblyName, executionContext.CheckerTypeName, executionContext.CheckerParameter); result.TestResults = new List<TestResult>(); foreach (var test in executionContext.Tests) { var processExecutionResult = executor.Execute(this.nodeJsExecutablePath, test.Input, executionContext.TimeLimit, executionContext.MemoryLimit, new[] { codeSavePath }); var testResult = this.ExecuteAndCheckTest(test, processExecutionResult, checker, processExecutionResult.ReceivedOutput); result.TestResults.Add(testResult); } // Clean up File.Delete(codeSavePath); return result; }
public abstract ExecutionResult Execute(ExecutionContext executionContext);
public override ExecutionResult Execute(ExecutionContext executionContext) { var result = new ExecutionResult(); // setting the IsCompiledSuccessfully variable to true as in the NodeJS // execution strategy there is no compilation result.IsCompiledSuccessfully = true; // Preprocess the user submission var codeToExecute = this.PreprocessJsSubmission(this.JsCodeTemplate, executionContext.Code); // Save the preprocessed submission which is ready for execution var codeSavePath = FileHelpers.SaveStringToTempFile(codeToExecute); // Process the submission and check each test IExecutor executor = new RestrictedProcessExecutor(); IChecker checker = Checker.CreateChecker(executionContext.CheckerAssemblyName, executionContext.CheckerTypeName, executionContext.CheckerParameter); result.TestResults = this.ProcessTests(executionContext, executor, checker, codeSavePath); // Clean up File.Delete(codeSavePath); return result; }
protected virtual List<TestResult> ProcessTests(ExecutionContext executionContext, IExecutor executor, IChecker checker, string codeSavePath) { var testResults = new List<TestResult>(); foreach (var test in executionContext.Tests) { var processExecutionResult = executor.Execute(this.NodeJsExecutablePath, test.Input, executionContext.TimeLimit, executionContext.MemoryLimit, new[] { codeSavePath }); var testResult = this.ExecuteAndCheckTest(test, processExecutionResult, checker, processExecutionResult.ReceivedOutput); testResults.Add(testResult); } return testResults; }
protected virtual string CreateSubmissionFile(ExecutionContext executionContext) => JavaCodePreprocessorHelper.CreateSubmissionFile(executionContext.Code, this.WorkingDirectory);
protected virtual CompileResult DoCompile(ExecutionContext executionContext, string submissionFilePath) { var compilerPath = this.GetCompilerPathFunc(executionContext.CompilerType); // Compile all source files - sandbox executor and submission file var compilerResult = this.CompileSourceFiles( executionContext.CompilerType, compilerPath, executionContext.AdditionalCompilerArguments, new[] { this.SandboxExecutorSourceFilePath, submissionFilePath }); return compilerResult; }
protected virtual string CreateSubmissionFile(ExecutionContext executionContext) { var submissionCode = executionContext.Code; // Remove existing packages submissionCode = Regex.Replace(submissionCode, PackageNameRegEx, string.Empty); // TODO: Remove the restriction for one public class - a non-public Java class can contain the main method! var classNameMatch = Regex.Match(submissionCode, ClassNameRegEx); if (!classNameMatch.Success) { throw new ArgumentException("No valid public class found!"); } var className = classNameMatch.Groups[ClassNameRegExGroup].Value; var submissionFilePath = $"{this.WorkingDirectory}\\{className}"; File.WriteAllText(submissionFilePath, submissionCode); return submissionFilePath; }
public override ExecutionResult Execute(ExecutionContext executionContext) { var result = new ExecutionResult(); // Copy the sandbox executor source code to a file in the working directory File.WriteAllText(this.SandboxExecutorSourceFilePath, this.SandboxExecutorCode); // Create a temp file with the submission code string submissionFilePath; try { submissionFilePath = this.CreateSubmissionFile(executionContext); } catch (ArgumentException exception) { result.IsCompiledSuccessfully = false; result.CompilerComment = exception.Message; return result; } var compilerResult = this.DoCompile(executionContext, submissionFilePath); // Assign compiled result info to the execution result result.IsCompiledSuccessfully = compilerResult.IsCompiledSuccessfully; result.CompilerComment = compilerResult.CompilerComment; if (!result.IsCompiledSuccessfully) { return result; } // Prepare execution process arguments and time measurement info var classPathArgument = $"-classpath \"{this.WorkingDirectory}\""; var classToExecuteFilePath = compilerResult.OutputFile; var classToExecute = classToExecuteFilePath .Substring( this.WorkingDirectory.Length + 1, classToExecuteFilePath.Length - this.WorkingDirectory.Length - JavaCompiledFileExtension.Length - 1) .Replace('\\', '.'); var timeMeasurementFilePath = $"{this.WorkingDirectory}\\{TimeMeasurementFileName}"; // Create an executor and a checker var executor = new StandardProcessExecutor(); var checker = Checker.CreateChecker(executionContext.CheckerAssemblyName, executionContext.CheckerTypeName, executionContext.CheckerParameter); // Process the submission and check each test foreach (var test in executionContext.Tests) { var processExecutionResult = executor.Execute( this.javaExecutablePath, test.Input, executionContext.TimeLimit * 2, // Java virtual machine takes more time to start up executionContext.MemoryLimit, new[] { classPathArgument, SandboxExecutorClassName, classToExecute, $"\"{timeMeasurementFilePath}\"" }); UpdateExecutionTime(timeMeasurementFilePath, processExecutionResult, executionContext.TimeLimit); var testResult = this.ExecuteAndCheckTest(test, processExecutionResult, checker, processExecutionResult.ReceivedOutput); result.TestResults.Add(testResult); } return result; }
public override ExecutionResult Execute(ExecutionContext executionContext) { IExecutor executor = new RestrictedProcessExecutor(); var result = this.CompileExecuteAndCheck(executionContext, this.getCompilerPathFunc, executor); return result; }
public override ExecutionResult Execute(ExecutionContext executionContext) { var result = new ExecutionResult(); // Create a temp file with the submission code string submissionFilePath; try { submissionFilePath = this.CreateSubmissionFile(executionContext.Code); } catch (ArgumentException exception) { result.IsCompiledSuccessfully = false; result.CompilerComment = exception.Message; return result; } // Create a sandbox executor source file in the working directory var sandboxExecutorSourceFilePath = $"{this.workingDirectory}\\{SandboxExecutorClassName}"; File.WriteAllText(sandboxExecutorSourceFilePath, this.SandboxExecutorCode); // Compile all source files - sandbox executor and submission file var compilerPath = this.getCompilerPathFunc(executionContext.CompilerType); var compilerResult = this.CompileSourceFiles( executionContext.CompilerType, compilerPath, executionContext.AdditionalCompilerArguments, new[] { submissionFilePath, sandboxExecutorSourceFilePath }); // Assign compiled result info to the execution result result.IsCompiledSuccessfully = compilerResult.IsCompiledSuccessfully; result.CompilerComment = compilerResult.CompilerComment; if (!result.IsCompiledSuccessfully) { return result; } // Prepare execution process arguments and time measurement info var classPathArgument = $"-classpath \"{this.workingDirectory}\""; var submissionFilePathLastIndexOfSlash = submissionFilePath.LastIndexOf('\\'); var classToExecute = submissionFilePath.Substring(submissionFilePathLastIndexOfSlash + 1); var timeMeasurementFilePath = $"{this.workingDirectory}\\{TimeMeasurementFileName}"; // Create an executor and a checker var executor = new StandardProcessExecutor(); var checker = Checker.CreateChecker(executionContext.CheckerAssemblyName, executionContext.CheckerTypeName, executionContext.CheckerParameter); // Process the submission and check each test foreach (var test in executionContext.Tests) { var processExecutionResult = executor.Execute( this.javaExecutablePath, test.Input, executionContext.TimeLimit, executionContext.MemoryLimit, new[] { classPathArgument, SandboxExecutorClassName, classToExecute, $"\"{timeMeasurementFilePath}\"" }); UpdateExecutionTime(timeMeasurementFilePath, processExecutionResult, executionContext.TimeLimit); var testResult = this.ExecuteAndCheckTest(test, processExecutionResult, checker, processExecutionResult.ReceivedOutput); result.TestResults.Add(testResult); } return result; }
public override ExecutionResult Execute(ExecutionContext executionContext) { var result = new ExecutionResult(); // Create a temp file with the submission code string submissionFilePath; try { submissionFilePath = this.CreateSubmissionFile(executionContext.Code); } catch (ArgumentException exception) { result.IsCompiledSuccessfully = false; result.CompilerComment = exception.Message; return(result); } // Create a sandbox executor source file in the working directory var sandboxExecutorSourceFilePath = string.Format("{0}\\{1}.{2}", this.workingDirectory, SandboxExecutorClassName, CompilerType.Java.GetFileExtension()); File.WriteAllText(sandboxExecutorSourceFilePath, this.SandboxExecutorCode); // Compile all source files - sandbox executor and submission file(s) var compilerPath = this.getCompilerPathFunc(executionContext.CompilerType); var sourceFilesToCompile = new DirectoryInfo(this.workingDirectory).GetFiles(JavaSourceFilesPattern).Select(x => x.FullName); var compilerResult = this.CompileSourceFiles( executionContext.CompilerType, compilerPath, executionContext.AdditionalCompilerArguments, sourceFilesToCompile); // Assign compiled result info to the execution result result.IsCompiledSuccessfully = compilerResult.IsCompiledSuccessfully; result.CompilerComment = compilerResult.CompilerComment; if (!result.IsCompiledSuccessfully) { return(result); } // Prepare execution process arguments and time measurement info var classPathArgument = string.Format("-classpath \"{0}\"", this.workingDirectory); var submissionFilePathLastIndexOfSlash = submissionFilePath.LastIndexOf('\\'); var submissionFilePathLastIndexOfDot = submissionFilePath.LastIndexOf('.'); var classToExecute = submissionFilePath.Substring( submissionFilePathLastIndexOfSlash + 1, submissionFilePathLastIndexOfDot - submissionFilePathLastIndexOfSlash - 1); var timeMeasurementFilePath = string.Format("{0}\\{1}", this.workingDirectory, TimeMeasurementFileName); // Create an executor and a checker var executor = new StandardProcessExecutor(); var checker = Checker.CreateChecker(executionContext.CheckerAssemblyName, executionContext.CheckerTypeName, executionContext.CheckerParameter); // Process the submission and check each test foreach (var test in executionContext.Tests) { var processExecutionResult = executor.Execute( this.javaExecutablePath, test.Input, executionContext.TimeLimit, executionContext.MemoryLimit, new[] { classPathArgument, SandboxExecutorClassName, classToExecute, string.Format("\"{0}\"", timeMeasurementFilePath) }); UpdateExecutionTime(timeMeasurementFilePath, processExecutionResult, executionContext.TimeLimit); var testResult = this.ExecuteAndCheckTest(test, processExecutionResult, checker, processExecutionResult.ReceivedOutput); result.TestResults.Add(testResult); } return(result); }
protected CompileResult ExecuteCompiling(ExecutionContext executionContext, Func<CompilerType, string> getCompilerPathFunc, ExecutionResult result) { var submissionFilePath = string.IsNullOrEmpty(executionContext.AllowedFileExtensions) ? FileHelpers.SaveStringToTempFile(executionContext.Code) : FileHelpers.SaveByteArrayToTempFile(executionContext.FileContent); var compilerPath = getCompilerPathFunc(executionContext.CompilerType); var compilerResult = this.Compile(executionContext.CompilerType, compilerPath, executionContext.AdditionalCompilerArguments, submissionFilePath); result.IsCompiledSuccessfully = compilerResult.IsCompiledSuccessfully; result.CompilerComment = compilerResult.CompilerComment; return compilerResult; }
private ExecutionResult RunUnitTests( ExecutionContext executionContext, IExecutor executor, IChecker checker, ExecutionResult result, string csProjFilePath) { var compileDirectory = Path.GetDirectoryName(csProjFilePath); var fileName = Path.GetFileName(csProjFilePath); if (!string.IsNullOrEmpty(fileName)) { fileName = fileName.Replace(ProjectFileExtenstion, DllFileExtension); } var targetDll = $"{compileDirectory}\\{CompiledResultFolder}{fileName}"; var arguments = new List <string> { targetDll }; arguments.AddRange(executionContext.AdditionalCompilerArguments.Split(' ')); var testedCodePath = $"{compileDirectory}\\{TestedCode}"; var originalTestsPassed = -1; var count = 0; foreach (var test in executionContext.Tests) { File.WriteAllText(testedCodePath, test.Input); var project = new Project(csProjFilePath); var didCompile = project.Build(); project.ProjectCollection.UnloadAllProjects(); if (!didCompile) { result.IsCompiledSuccessfully = false; return(result); } var processExecutionResult = executor.Execute( this.NUnitConsoleRunnerPath, string.Empty, executionContext.TimeLimit, executionContext.MemoryLimit, arguments); var totalTests = 0; var passedTests = 0; this.ExtractTestResult(processExecutionResult.ReceivedOutput, out passedTests, out totalTests); var message = "Test Passed!"; if (totalTests == 0) { message = "No tests found"; } else if (passedTests == originalTestsPassed) { message = "No functionality covering this test!"; } if (count == 0) { originalTestsPassed = passedTests; if (totalTests != passedTests) { message = "Not all tests passed on the correct solution."; } } var testResult = this.ExecuteAndCheckTest(test, processExecutionResult, checker, message); File.Delete(testedCodePath); result.TestResults.Add(testResult); count++; } return(result); }
private void ProcessSubmission(Submission submission) { // TODO: Check for N+1 queries problem this.logger.InfoFormat("Work on submission №{0} started.", submission.Id); var executionStrategy = this.CreateExecutionStrategy(submission.SubmissionType.ExecutionStrategyType); var context = new ExecutionContext { SubmissionId = submission.Id, AdditionalCompilerArguments = submission.SubmissionType.AdditionalCompilerArguments, CheckerAssemblyName = submission.Problem.Checker.DllFile, CheckerParameter = submission.Problem.Checker.Parameter, CheckerTypeName = submission.Problem.Checker.ClassName, FileContent = submission.Content, AllowedFileExtensions = submission.SubmissionType.AllowedFileExtensions, CompilerType = submission.SubmissionType.CompilerType, MemoryLimit = submission.Problem.MemoryLimit, TimeLimit = submission.Problem.TimeLimit, }; context.Tests = submission.Problem.Tests.ToList().Select(x => new TestContext { Id = x.Id, Input = x.InputDataAsString, Output = x.OutputDataAsString, IsTrialTest = x.IsTrialTest }); ExecutionResult executionResult; try { executionResult = executionStrategy.Execute(context); } catch (Exception exception) { this.logger.ErrorFormat("executionStrategy.Execute on submission №{0} has thrown an exception: {1}", submission.Id, exception); submission.ProcessingComment = string.Format("Exception in executionStrategy.Execute: {0}", exception.Message); return; } submission.IsCompiledSuccessfully = executionResult.IsCompiledSuccessfully; submission.CompilerComment = executionResult.CompilerComment; if (!executionResult.IsCompiledSuccessfully) { return; } foreach (var testResult in executionResult.TestResults) { var testRun = new TestRun { CheckerComment = testResult.CheckerDetails.Comment, ExpectedOutputFragment = testResult.CheckerDetails.ExpectedOutputFragment, UserOutputFragment = testResult.CheckerDetails.UserOutputFragment, ExecutionComment = testResult.ExecutionComment, MemoryUsed = testResult.MemoryUsed, ResultType = testResult.ResultType, TestId = testResult.Id, TimeUsed = testResult.TimeUsed, }; submission.TestRuns.Add(testRun); } this.logger.InfoFormat("Work on submission №{0} ended.", submission.Id); }
public override ExecutionResult Execute(ExecutionContext executionContext) { var solution = executionContext.FileContent; var result = new ExecutionResult(); var compileResult = this.ExecuteCompiling(executionContext, this.getCompilerPathFunc, result); if (!compileResult.IsCompiledSuccessfully) { return result; } var outputAssemblyPath = this.PreprocessAndCompileTestRunner(executionContext, compileResult.OutputFile); IExecutor executor = new RestrictedProcessExecutor(); var processExecutionResult = executor.Execute(outputAssemblyPath, string.Empty, executionContext.TimeLimit, executionContext.MemoryLimit); var workingDirectory = compileResult.OutputFile; if (Directory.Exists(workingDirectory)) { Directory.Delete(workingDirectory, true); } this.ProcessTests(processExecutionResult, executionContext, result); return result; }
private string PreprocessAndCompileTestRunner(ExecutionContext executionContext, string outputDirectory) { var testStrings = new List<string>(); foreach (var test in executionContext.Tests) { testStrings.Add(TestTemplate.Replace("#testInput#", test.Input)); } var allTests = string.Join(",", testStrings); var testRunnerCode = TestRunnerTemplate.Replace("#allTests#", allTests); var compiler = new CSharpCodeProvider(); var compilerParameters = new CompilerParameters(); compilerParameters.ReferencedAssemblies.Add("mscorlib.dll"); compilerParameters.ReferencedAssemblies.Add("System.dll"); compilerParameters.ReferencedAssemblies.Add("System.Core.dll"); compilerParameters.ReferencedAssemblies.AddRange(Directory.GetFiles(outputDirectory).Where(f => f.EndsWith(".dll") || f.EndsWith(".exe")).ToArray()); compilerParameters.GenerateInMemory = false; compilerParameters.GenerateExecutable = true; var compilerResult = compiler.CompileAssemblyFromSource(compilerParameters, testRunnerCode); var outputAssemblyPath = outputDirectory + "\\LocalTestRunner.exe"; File.Move(compilerResult.PathToAssembly, outputAssemblyPath); return outputAssemblyPath; }
private void ProcessTests(ProcessExecutionResult processExecutionResult, ExecutionContext executionContext, ExecutionResult result) { var jsonResult = JsonExecutionResult.Parse(processExecutionResult.ReceivedOutput, true, true); var index = 0; result.TestResults = new List<TestResult>(); foreach (var test in executionContext.Tests) { var testResult = new TestResult { Id = test.Id, TimeUsed = (int)processExecutionResult.TimeWorked.TotalMilliseconds, MemoryUsed = (int)processExecutionResult.MemoryUsed, }; if (jsonResult.PassingIndexes.Contains(index)) { testResult.ResultType = TestRunResultType.CorrectAnswer; } else { testResult.ResultType = TestRunResultType.WrongAnswer; testResult.CheckerDetails = new CheckerDetails { Comment = "Test failed." }; } result.TestResults.Add(testResult); index++; } }
private void ProcessSubmission(Submission submission) { // TODO: Check for N+1 queries problem this.logger.InfoFormat("Work on submission №{0} started.", submission.Id); IExecutionStrategy executionStrategy = this.CreateExecutionStrategy(submission.SubmissionType.ExecutionStrategyType); var context = new ExecutionContext { AdditionalCompilerArguments = submission.SubmissionType.AdditionalCompilerArguments, CheckerAssemblyName = submission.Problem.Checker.DllFile, CheckerParameter = submission.Problem.Checker.Parameter, CheckerTypeName = submission.Problem.Checker.ClassName, FileContent = submission.Content, AllowedFileExtensions = submission.SubmissionType.AllowedFileExtensions, CompilerType = submission.SubmissionType.CompilerType, MemoryLimit = submission.Problem.MemoryLimit, TimeLimit = submission.Problem.TimeLimit, }; context.Tests = submission.Problem.Tests.ToList().Select(x => new TestContext { Id = x.Id, Input = x.InputDataAsString, Output = x.OutputDataAsString, IsTrialTest = x.IsTrialTest }); ExecutionResult executionResult; try { executionResult = executionStrategy.Execute(context); } catch (Exception exception) { this.logger.ErrorFormat("executionStrategy.Execute on submission №{0} has thrown an exception: {1}", submission.Id, exception); submission.ProcessingComment = string.Format("Exception in executionStrategy.Execute: {0}", exception.Message); return; } submission.IsCompiledSuccessfully = executionResult.IsCompiledSuccessfully; submission.CompilerComment = executionResult.CompilerComment; if (!executionResult.IsCompiledSuccessfully) { return; } foreach (var testResult in executionResult.TestResults) { var testRun = new TestRun { CheckerComment = testResult.CheckerComment, ExecutionComment = testResult.ExecutionComment, MemoryUsed = testResult.MemoryUsed, ResultType = testResult.ResultType, TestId = testResult.Id, TimeUsed = testResult.TimeUsed, }; submission.TestRuns.Add(testRun); } this.logger.InfoFormat("Work on submission №{0} ended.", submission.Id); }
protected virtual List <TestResult> ProcessTests( ExecutionContext executionContext, IExecutor executor, IChecker checker, string codeSavePath, int numberOfUserTests) { var testResults = new List <TestResult>(); var arguments = new List <string>(); arguments.Add(this.MochaModulePath); arguments.Add(codeSavePath); arguments.AddRange(executionContext.AdditionalCompilerArguments.Split(' ')); var testCount = 0; var processExecutionResult = this.ExecuteNodeJsProcess(executionContext, executor, string.Empty, arguments); var mochaResult = JsonExecutionResult.Parse(processExecutionResult.ReceivedOutput); var correctSolutionTestPasses = mochaResult.TestsErrors.Take(numberOfUserTests).Count(x => x == null); var testOffset = numberOfUserTests; foreach (var test in executionContext.Tests) { TestResult testResult = null; if (testCount == 0) { var minTestCount = int.Parse( Regex.Match( test.Input, "<minTestCount>(\\d+)</minTestCount>").Groups[1].Value); var message = "Test Passed!"; if (numberOfUserTests == 0) { message = "The submitted code was either incorrect or contained no tests!"; } else if (numberOfUserTests < minTestCount) { message = $"Insufficient amount of tests, you have to have atleast {minTestCount} tests!"; } testResult = this.ExecuteAndCheckTest( test, processExecutionResult, checker, message); } else if (testCount == 1) { var message = "Test Passed!"; if (numberOfUserTests == 0) { message = "The submitted code was either incorrect or contained no tests!"; } else if (correctSolutionTestPasses != numberOfUserTests) { message = "Error: Some tests failed while running the correct solution!"; } testResult = this.ExecuteAndCheckTest( test, processExecutionResult, checker, message); } else { var numberOfPasses = mochaResult.TestsErrors.Skip(testOffset).Take(numberOfUserTests).Count(x => x == null); var message = "Test Passed!"; if (numberOfPasses >= correctSolutionTestPasses) { message = "No unit test covering this functionality!"; mochaResult.TestsErrors .Take(numberOfUserTests) .Where(x => x != null) .ForEach(x => message += x); } testResult = this.ExecuteAndCheckTest( test, processExecutionResult, checker, message); testOffset += numberOfUserTests; } testCount++; testResults.Add(testResult); } return(testResults); }