public async Task <(Success?, Failure?)> Verify(Challenge.Challenge challenge, string yolol) { await Task.CompletedTask; IScore scoreMode = challenge.ScoreMode switch { ScoreMode.BasicScoring => new BasicScoring(), ScoreMode.Approximate => new ApproximateScoring(), ScoreMode.Unknown => throw new InvalidOperationException("Cannot use `Unknown` score mode (challenge is broken - contact Martin#2468)"), _ => throw new NotImplementedException($"Score mode `{challenge.ScoreMode}` is not implemented") }; // Retrieve the test cases for the challenge var(inputs, outputs) = GetTests(challenge); // Check input program fits within 20x70 var lines = yolol.Split("\n"); if (lines.Length > 20 || lines.Any(l => l.Length > 70)) { return(null, new Failure(FailureType.ProgramTooLarge, null)); } // parse the program var result = Parser.ParseProgram(yolol); if (!result.IsOk) { return(null, new Failure(FailureType.ParseFailed, result.Err.ToString())); } var entry = result.Ok; // Get the variable which the program uses to indicate it is ready to move to the next round var state = new MachineState(new DefaultValueDeviceNetwork()); var done = state.GetVariable($":{challenge.CheckIndicator}"); // Begin counting how long it takes to verify (for profiling purposes) var timer = new Stopwatch(); timer.Start(); // Run through test cases one by one var overflowIters = _config.MaxItersOverflow; var totalRuntime = 0u; var pc = 0; for (var i = 0; i < Math.Min(inputs.Count, outputs.Count); i++) { // Set inputs var input = inputs[i]; foreach (var(key, value) in input) { state.GetVariable($":{key}").Value = value; } // Clear completion indicator done.Value = 0; // Run lines until completion indicator is set or execution time limit is exceeded var limit = 0; while (!done.Value.ToBool()) { // Check if this test has exceed it's time limit if (limit++ > _config.MaxTestIters) { //If so use iterations from the overflow pool overflowIters--; //Once the overflow pool is empty too, fail if (overflowIters <= 0) { return(null, new Failure(FailureType.RuntimeTooLong, null)); } } totalRuntime++; try { // If line if blank, just move to the next line if (pc >= entry.Lines.Count) { pc++; } else { pc = entry.Lines[pc].Evaluate(pc, state); } } catch (ExecutionException) { pc++; } // loop around if program counter goes over max if (pc >= 20) { pc = 0; } } // Check this test case with the current scoring mode var scoreFailure = scoreMode.CheckCase(input, outputs[i], state); if (scoreFailure != null) { return(null, scoreFailure); } } Console.WriteLine($"Verified {totalRuntime} ticks, {timer.ElapsedMilliseconds}ms runtime"); // Calculate score var codeLength = yolol.Replace("\n", "").Length; var score = scoreMode.FinalizeScore( (uint)inputs.Count, totalRuntime, codeLength ); return(new Success((uint)score, (uint)totalRuntime, (uint)codeLength), null); }
public async Task <(Success?, Failure?)> Verify(Challenge.Challenge challenge, string yolol) { await Task.CompletedTask; IScore scoreMode = challenge.ScoreMode switch { ScoreMode.BasicScoring => new BasicScoring(), ScoreMode.Approximate => new ApproximateScoring(), ScoreMode.Unknown => throw new InvalidOperationException("Cannot use `Unknown` score mode (challenge is broken - contact Martin#2468)"), _ => throw new NotImplementedException($"Score mode `{challenge.ScoreMode}` is not implemented (contact Martin#2468)") }; if (challenge.Chip == YololChip.Unknown) { throw new InvalidOperationException("Cannot submit to a challenge with `YololChip.Unknown` (challenge is broken - contact Martin#2468)"); } if (!challenge.Intermediate.IsOk) { throw new InvalidOperationException("Cannot submit to a challenge with broken intermediate code (challenge is broken - contact Martin#2468)"); } // Check input program fits within 20x70 var lines = yolol.Split("\n"); if (lines.Length > 20 || lines.Any(l => l.Length > 70)) { return(null, new Failure(FailureType.ProgramTooLarge, null, null)); } // parse the program var parsed = Parser.ParseProgram(yolol); if (!parsed.IsOk) { return(null, new Failure(FailureType.ParseFailed, parsed.Err.ToString(), null)); } // Verify that code is allowed on the given chip level if (challenge.Chip != YololChip.Professional) { var fail = CheckChipLevel(challenge.Chip, parsed.Ok); if (fail != null) { return(null, fail); } } // Prepare a machine state for execution. // Two states are needed - one for user code and one for code supplied by the challenge var executionsContexts = (await _executor.Prepare(new[] { parsed.Ok, challenge.Intermediate.Ok }, $":{challenge.CheckIndicator}")).ToList(); var stateUser = executionsContexts[0]; var stateChallenge = executionsContexts[1]; // Retrieve the test cases for the challenge var(inputs, outputs) = GetTests(challenge); // Begin counting how long it takes to verify (for profiling purposes) var timer = new Stopwatch(); timer.Start(); // Run through test cases one by one var overflowIters = (long)MaxItersOverflow; for (var i = 0; i < Math.Max(inputs.Count, outputs.Count); i++) { // Set inputs in user execution state var input = SetInputs(i < inputs.Count ? inputs[i] : new Dictionary <string, Value>(), stateUser); var output = outputs[i]; var savedState = stateUser.Serialize(output); // Check realtime timeout if (timer.Elapsed.TotalMilliseconds > TimeoutMillisecondsAll) { Console.WriteLine($"Timed out verification: {stateUser.TotalLinesExecuted} ticks, {timer.ElapsedMilliseconds}ms runtime"); return(null, new Failure(FailureType.RuntimeTooLongMilliseconds, $"Execution Timed Out (executed {i} test cases in {timer.Elapsed.TotalMilliseconds}ms)", savedState)); } // Run the user code until completion Failure?failure; (failure, overflowIters) = await RunToDone(stateUser, MaxTestIters, i, inputs.Count, overflowIters, savedState); if (failure != null) { return(null, failure); } // Set the challenge inputs _and_ outputs into the challenge state SetInputs(input, stateChallenge, "input_"); SetInputs(outputs[i], stateChallenge, "output_"); // Run the challenge code (failure, _) = await RunToDone(stateChallenge, MaxTestIters, 0, 0, MaxItersOverflow, savedState); if (failure != null) { return(null, new Failure(FailureType.ChallengeCodeFailed, failure.Hint, savedState)); } // Check if the challenge code has forced a failure var forceFail = stateChallenge.TryGet(new VariableName(":fail")); if (forceFail?.Type == Yolol.Execution.Type.String && forceFail.Value.String.Length != 0) { return(null, new Failure(FailureType.ChallengeForceFail, forceFail.Value.String.ToString(), savedState)); } // Check this test case with the current scoring mode var scoreFailure = scoreMode.CheckCase(input, outputs[i], stateUser, savedState); if (scoreFailure != null) { return(null, scoreFailure); } } Console.WriteLine($"Verified {stateUser.TotalLinesExecuted} ticks, {timer.ElapsedMilliseconds}ms runtime"); // Calculate score var codeLength = yolol.Replace("\n", "").Length; var score = scoreMode.FinalizeScore( (uint)Math.Max(inputs.Count, outputs.Count), (uint)stateUser.TotalLinesExecuted, codeLength ); return(new Success(score, (uint)stateUser.TotalLinesExecuted, (uint)codeLength, scoreMode.Hint, (uint)challenge.Inputs.Count), null); }