public SolutionTest( StringOrFile inputFile, StringOrFile expectedOutputFile, IOutputVerifier outputVerifier, string processArguments, TimeSpan timeLimit, double maxScore, string testGroup ) { if (inputFile == null) { throw new ArgumentNullException(nameof(inputFile)); } if (expectedOutputFile == null) { throw new ArgumentNullException(nameof(expectedOutputFile)); } if (outputVerifier == null) { throw new ArgumentNullException(nameof(outputVerifier)); } if (testGroup == null) { throw new ArgumentNullException(nameof(testGroup)); } ProcessArguments = processArguments; Input = inputFile; ExpectedOutput = expectedOutputFile; OutputVerifier = outputVerifier; TimeLimit = timeLimit; MaxScore = maxScore; TestGroup = testGroup; }
public void TestSum() { string checker = Path.GetFullPath("checker_sum.exe".GetLocalFilePath()); IExecutableOutputVerifier verifier = new ExecutableOutputVerifierMutable { ConsoleApplication = new FileSystemConsoleApplication(checker, CheckerCore.CrashReporting.CrashReportFinder.Instance), Bindings = new IVerifierResultBinder[] { new StdOutContainsBinder("1", new OutputVerificationResult( OutputVerificationResultType.CorrectAnswer, 1 )), new StdOutContainsBinder("0", new OutputVerificationResult( OutputVerificationResultType.WrongAnswer, 0 )) }, Arguments = new VerifierArgumentType[] { VerifierArgumentType.Solution, //number A VerifierArgumentType.Stdout //number B }, Stdin = VerifierArgumentType.ExitCode //number C }; OutputVerificationInfo info = new OutputVerificationInfo( 5,//C null, null, StringOrFile.FromText("2"), //B StringOrFile.FromText("3") //A ); OutputVerificationResult result = verifier.Verify(info); Assert.AreEqual(result.Type, OutputVerificationResultType.CorrectAnswer); ; }
public void TestContinuity() { ProcessRunResult result = ConsoleApplicationRunner.Instance.Run( new FileSystemConsoleApplication(DummyPath, CheckerCore.CrashReporting.CrashReportFinder.Instance), TimeSpan.FromHours(1), StringOrFile.FromText("continuity 100000 end") ); Assert.AreEqual(result.ExitType, ProcessExitType.Graceful); StringOrFile output = result.Output; string[] lines = output.StringValue .Split(new char[] { '\n' }, StringSplitOptions.RemoveEmptyEntries) .Select(x => x.Trim()) .Where(x => !string.IsNullOrWhiteSpace(x)) .ToArray(); int prev = int.Parse(lines.First().Split(' ')[0]); for (int i = 1; i < lines.Length; i++) { int current = int.Parse(lines[i].Split(' ')[0]); Assert.IsTrue(current == prev + 1); prev = current; } ; }
public ProcessRunArguments(IConsoleApplication application, StringOrFile stdIn, TimeSpan maxRuntime, string processArguments, bool allowCrashReports) { Application = application; StandardInput = stdIn; ProcessArguments = processArguments ?? string.Empty; AllowCrashReports = allowCrashReports; MaxRuntime = maxRuntime; }
public void TestCmpfiles(bool okay) { string checker = Path.GetFullPath("checker_cmpfiles.exe".GetLocalFilePath()); Random r = new Random(1337); string file1 = Path.GetTempFileName(); string file2 = Path.GetTempFileName(); File.WriteAllText(file1, "The quick brown fox jumps over the lazy dog"); if (okay) { File.WriteAllText(file2, "The quick brown fox jumps over the lazy dog"); } else { File.WriteAllText(file2, "The quick brown fox jumps over the lazy dog."); } IOutputVerifier verifier = new ExecutableOutputVerifierMutable { ConsoleApplication = new FileSystemConsoleApplication(checker, CheckerCore.CrashReporting.CrashReportFinder.Instance), Arguments = new VerifierArgumentType[] { VerifierArgumentType.FileStdout, VerifierArgumentType.FileSolution }, Bindings = new IVerifierResultBinder[] { new StdOutContainsBinder("OK", new OutputVerificationResult( OutputVerificationResultType.CorrectAnswer, 1 )), new StdOutContainsBinder("NO", new OutputVerificationResult( OutputVerificationResultType.WrongAnswer, 0 )) }, Stdin = VerifierArgumentType.None }; OutputVerificationInfo info = new OutputVerificationInfo( 0, null, null, StringOrFile.FromFile(file1), StringOrFile.FromFile(file2) ); OutputVerificationResult result = verifier.Verify(info); if (okay) { Assert.AreEqual(result.ScoreMultiplier, 1); Assert.AreEqual(result.Type, OutputVerificationResultType.CorrectAnswer); } else { Assert.AreEqual(result.ScoreMultiplier, 0); Assert.AreEqual(result.Type, OutputVerificationResultType.WrongAnswer); } }
public ProcessRunResult(ProcessExitType exitType, ICrashReport crashReport, StringOrFile stdOut, StringOrFile stdErr, double memoryUsed, int exitCode, TimeSpan executionTime) { ExitType = exitType; CrashReport = crashReport; Output = stdOut; StandardError = stdErr; MemoryUsed = memoryUsed; ExitCode = exitCode; ExecutionTime = executionTime; }
public void TestCrash() { ProcessRunResult result = ConsoleApplicationRunner.Instance.Run( new FileSystemConsoleApplication(DummyPath, CheckerCore.CrashReporting.CrashReportFinder.Instance), TimeSpan.FromSeconds(1), StringOrFile.FromText("sum 1 2 crash") ); Assert.AreEqual(result.ExitType, ProcessExitType.Crashed); Assert.IsNotNull(result.CrashReport); }
public void TestExit() { ProcessRunResult result = ConsoleApplicationRunner.Instance.Run( new FileSystemConsoleApplication(DummyPath, CheckerCore.CrashReporting.CrashReportFinder.Instance), TimeSpan.FromHours(1), StringOrFile.FromText("exit 3") ); Assert.AreEqual(result.ExitCode, 3); Assert.AreEqual(result.ExitType, ProcessExitType.Graceful); }
public void TestSlow() { ProcessRunResult result = ConsoleApplicationRunner.Instance.Run( new FileSystemConsoleApplication(DummyPath, CheckerCore.CrashReporting.CrashReportFinder.Instance), TimeSpan.FromSeconds(1), StringOrFile.FromText("wait 200 sum 1 5 wait 10000 exit 0") ); Assert.AreEqual(result.ExitType, ProcessExitType.Timeout); Assert.IsTrue(result.Output.ToString().Contains("6")); }
public OutputVerificationInfo( int exitCode, StringOrFile standardError, StringOrFile standardInput, StringOrFile standardOutput, StringOrFile expectedOutput) { ExitCode = exitCode; StandardError = standardError; StandardInput = standardInput; StandardOutput = standardOutput; ExpectedOutput = expectedOutput; }
public bool TryCreateTestViewModels(out List <SolutionTest> tests) { //TODO: Fix this shit because it's ugly Java-ish practice if (!CanCreateTests) { tests = null; return(false); } string[] inputs = TestInputFiles.PathsArray.ToArray(); string[] expectedOutputs = TestSolutionFiles.PathsArray.ToArray(); if (SortFilenamesAlphabetically) { Array.Sort <string>(inputs, (x, y) => x.CompareTo(y)); Array.Sort <string>(expectedOutputs, (x, y) => x.CompareTo(y)); } ; double totalExplicitScore = Tests .Where(x => !double.IsNaN(x.MaxScore)) .Sum(x => x.MaxScore); int autoScoreCount = Tests .Where(x => double.IsNaN(x.MaxScore)) .Count(); double autoScore = (100.00 - totalExplicitScore) / autoScoreCount; tests = new List <SolutionTest>(); for (int i = 0; i < inputs.Length; i++) { tests.Add(new SolutionTest( inputFile: StringOrFile.FromFile(inputs[i]), expectedOutputFile: StringOrFile.FromFile(expectedOutputs[i]), outputVerifier: Checker.CreateModel(), processArguments: String.Empty, timeLimit: TimeSpan.FromSeconds(TimeLimitSeconds), maxScore: !double.IsNaN(Tests[i].MaxScore) ? Tests[i].MaxScore : autoScore, testGroup: Tests[i].TestGroup ?? string.Empty )); } return(true); }
public void TestOutput() { ProcessRunResult result = ConsoleApplicationRunner.Instance.Run( new FileSystemConsoleApplication(DummyPath, CheckerCore.CrashReporting.CrashReportFinder.Instance), TimeSpan.FromSeconds(0.1), StringOrFile.FromText("sum 1 2 sum 3 5 end") ); Assert.AreEqual(result.ExitType, ProcessExitType.Graceful); StringOrFile output = result.Output; Assert.IsTrue(output.StringValue.IndexOf('3') != -1 && output.StringValue.IndexOf('8') > output.StringValue.IndexOf('3')); ; }
public ProcessRunResult Run( IConsoleApplication application, TimeSpan maxRuntime, StringOrFile stdIn, string processArguments = null, bool allowCrashReports = true ) { ProcessRunArguments args = new ProcessRunArguments( application, stdIn, maxRuntime, processArguments, allowCrashReports ); return(Run(args)); }
private ProcessRunResult Run(ProcessRunArguments args) { IConsoleProcess process = args.Application.StartProcess(args.ProcessArguments); bool timelyExit = TryWaitForTimelyExitAsync(process, args).GetAwaiter().GetResult(); bool crashed = process.TryFindCrashReport(out ICrashReport report); string stdOut = process.GetStandardOutput(); string stdErr = process.GetStandardError(); return(new ProcessRunResult( DeduceExitType(crashed, timelyExit), report, StringOrFile.FromText(stdOut), StringOrFile.FromText(stdErr), 100, process.ExitCode, process.ExecutionTime )); }
private StringOrFile GetVerifierArgument(OutputVerificationInfo info, VerifierArgumentType thing) { switch (thing) { case VerifierArgumentType.ExitCode: return(StringOrFile.FromText(info.ExitCode.ToString())); case VerifierArgumentType.FileSolution: return(StringOrFile.FromText(info.ExpectedOutput.FilePath)); case VerifierArgumentType.FileStderr: return(StringOrFile.FromText(info.StandardError.FilePath)); case VerifierArgumentType.FileStdin: return(StringOrFile.FromText(info.StandardInput.FilePath)); case VerifierArgumentType.FileStdout: return(StringOrFile.FromText(info.StandardOutput.FilePath)); case VerifierArgumentType.Solution: return(info.ExpectedOutput); case VerifierArgumentType.Stderr: return(info.StandardError); case VerifierArgumentType.Stdin: return(info.StandardInput); case VerifierArgumentType.Stdout: return(info.StandardOutput); default: case VerifierArgumentType.None: return(StringOrFile.FromText(string.Empty)); } }
public void TestIsOdd() { string checker = Path.GetFullPath("checker_isodd.exe".GetLocalFilePath()); Random r = new Random(1337); for (int i = 0; i < 12; i++) { int number = r.Next(); IOutputVerifier verifier = new ExecutableOutputVerifierMutable { ConsoleApplication = new FileSystemConsoleApplication(checker, CheckerCore.CrashReporting.CrashReportFinder.Instance), Stdin = VerifierArgumentType.Stdout,//number Bindings = new IVerifierResultBinder[] { new ExitCodeBinder(1, new OutputVerificationResult( OutputVerificationResultType.CorrectAnswer, 1 )), new ExitCodeBinder(0, new OutputVerificationResult( OutputVerificationResultType.CorrectAnswer, 0 )) } }; OutputVerificationInfo info = new OutputVerificationInfo( 0, null, null, StringOrFile.FromText(number.ToString()), null ); OutputVerificationResult result = verifier.Verify(info); Assert.AreEqual(result.Type, OutputVerificationResultType.CorrectAnswer); Assert.AreEqual(result.ScoreMultiplier, number % 2); } }