private async Task Evaluate() { IExecutionState?previous = null; foreach (var message in _messages) { // Skip forward to the first message with a null state (i.e. needs evaluating) if (message.State != null) { previous = message.State; continue; } previous = await EvaluateSingle(message.Input, message.Response, previous); } async Task <IExecutionState?> EvaluateSingle(IMessage input, IUserMessage output, IExecutionState?prevState) { // Try to get code from message var code = input.Content.ExtractYololCodeBlock(); if (code == null) { await output.ModifyAsync2(@"Failed to parse a yolol program from message - ensure you have enclosed your solution in triple backticks \`\`\`like this\`\`\`"); return(prevState); } // Try to parse code as Yolol var result = Yolol.Grammar.Parser.ParseProgram(code); if (!result.IsOk) { await output.ModifyAsync2(result.Err.ToString()); return(prevState); } // Setup a new state var state = _executor.Prepare(result.Ok); state.TerminateOnPcOverflow = true; prevState?.CopyTo(state); // Execute for 2000 ticks var err = state.Run(2000, TimeSpan.FromMilliseconds(150)); // Print out result if (err != null) { await output.ModifyAsync2(err); } else { await output.ModifyAsync2(embed : state.ToEmbed().Build()); } return(state); } }
private async Task RunYolol(string input, IYololExecutor executor, uint lines) { // Try to parse code as Yolol var result = await Parse(input); if (result == null) { return; } // Prep execution state var compileTimer = new Stopwatch(); compileTimer.Start(); var state = await executor.Prepare(result); compileTimer.Stop(); // Run for N lines, 500ms or until `:done!=0` var exeTimer = new Stopwatch(); exeTimer.Start(); var saved = state.Serialize(); var err = await state.Run(lines, TimeSpan.FromMilliseconds(500)); // Print out the final machine state var embed = state.ToEmbed(embed => { embed.AddField("Setup", $"{compileTimer.ElapsedMilliseconds}ms", true); embed.AddField("Execute", $"{exeTimer.ElapsedMilliseconds}ms", true); if (err != null) { embed.AddField("Error", $"{err}", false); } if (_debuggerUrl != null) { var base64 = saved.Serialize(); embed.WithUrl($"{_debuggerUrl}?state={base64}"); } }); await ReplyAsync(embed : embed.Build()); }
private async Task RunYolol(string input, IYololExecutor executor) { // Try to parse code as Yolol var result = await Parse(input); if (result == null) { return; } // Prep execution state var compileTimer = new Stopwatch(); compileTimer.Start(); var state = executor.Prepare(result); compileTimer.Stop(); // Run for 2000 lines, 500ms or until `:done!=0` var exeTimer = new Stopwatch(); exeTimer.Start(); var err = state.Run(2000, TimeSpan.FromMilliseconds(500)); // Print out error if execution terminated for some reason if (err != null) { await ReplyAsync(err); return; } // Print out the final machine state var embed = state.ToEmbed(embed => { embed.AddField("Setup", $"{compileTimer.ElapsedMilliseconds}ms", true); embed.AddField("Execute", $"{exeTimer.ElapsedMilliseconds}ms", true); }); await ReplyAsync(embed : embed.Build()); }
public async Task <(Success?, Failure?)> Verify(Challenge.Challenge challenge, string yolol) { await Task.CompletedTask; IScore scoreMode = challenge.ScoreMode switch { ScoreMode.BasicScoring => new BasicScoring(), ScoreMode.Approximate => new ApproximateScoring(), ScoreMode.Unknown => throw new InvalidOperationException("Cannot use `Unknown` score mode (challenge is broken - contact Martin#2468)"), _ => throw new NotImplementedException($"Score mode `{challenge.ScoreMode}` is not implemented (contact Martin#2468)") }; if (challenge.Chip == YololChip.Unknown) { throw new InvalidOperationException("Cannot submit to a challenge with `YololChip.Unknown` (challenge is broken - contact Martin#2468)"); } if (!challenge.Intermediate.IsOk) { throw new InvalidOperationException("Cannot submit to a challenge with broken intermediate code (challenge is broken - contact Martin#2468)"); } // Check input program fits within 20x70 var lines = yolol.Split("\n"); if (lines.Length > 20 || lines.Any(l => l.Length > 70)) { return(null, new Failure(FailureType.ProgramTooLarge, null, null)); } // parse the program var parsed = Parser.ParseProgram(yolol); if (!parsed.IsOk) { return(null, new Failure(FailureType.ParseFailed, parsed.Err.ToString(), null)); } // Verify that code is allowed on the given chip level if (challenge.Chip != YololChip.Professional) { var fail = CheckChipLevel(challenge.Chip, parsed.Ok); if (fail != null) { return(null, fail); } } // Prepare a machine state for execution. // Two states are needed - one for user code and one for code supplied by the challenge var executionsContexts = (await _executor.Prepare(new[] { parsed.Ok, challenge.Intermediate.Ok }, $":{challenge.CheckIndicator}")).ToList(); var stateUser = executionsContexts[0]; var stateChallenge = executionsContexts[1]; // Retrieve the test cases for the challenge var(inputs, outputs) = GetTests(challenge); // Begin counting how long it takes to verify (for profiling purposes) var timer = new Stopwatch(); timer.Start(); // Run through test cases one by one var overflowIters = (long)MaxItersOverflow; for (var i = 0; i < Math.Max(inputs.Count, outputs.Count); i++) { // Set inputs in user execution state var input = SetInputs(i < inputs.Count ? inputs[i] : new Dictionary <string, Value>(), stateUser); var output = outputs[i]; var savedState = stateUser.Serialize(output); // Check realtime timeout if (timer.Elapsed.TotalMilliseconds > TimeoutMillisecondsAll) { Console.WriteLine($"Timed out verification: {stateUser.TotalLinesExecuted} ticks, {timer.ElapsedMilliseconds}ms runtime"); return(null, new Failure(FailureType.RuntimeTooLongMilliseconds, $"Execution Timed Out (executed {i} test cases in {timer.Elapsed.TotalMilliseconds}ms)", savedState)); } // Run the user code until completion Failure?failure; (failure, overflowIters) = await RunToDone(stateUser, MaxTestIters, i, inputs.Count, overflowIters, savedState); if (failure != null) { return(null, failure); } // Set the challenge inputs _and_ outputs into the challenge state SetInputs(input, stateChallenge, "input_"); SetInputs(outputs[i], stateChallenge, "output_"); // Run the challenge code (failure, _) = await RunToDone(stateChallenge, MaxTestIters, 0, 0, MaxItersOverflow, savedState); if (failure != null) { return(null, new Failure(FailureType.ChallengeCodeFailed, failure.Hint, savedState)); } // Check if the challenge code has forced a failure var forceFail = stateChallenge.TryGet(new VariableName(":fail")); if (forceFail?.Type == Yolol.Execution.Type.String && forceFail.Value.String.Length != 0) { return(null, new Failure(FailureType.ChallengeForceFail, forceFail.Value.String.ToString(), savedState)); } // Check this test case with the current scoring mode var scoreFailure = scoreMode.CheckCase(input, outputs[i], stateUser, savedState); if (scoreFailure != null) { return(null, scoreFailure); } } Console.WriteLine($"Verified {stateUser.TotalLinesExecuted} ticks, {timer.ElapsedMilliseconds}ms runtime"); // Calculate score var codeLength = yolol.Replace("\n", "").Length; var score = scoreMode.FinalizeScore( (uint)Math.Max(inputs.Count, outputs.Count), (uint)stateUser.TotalLinesExecuted, codeLength ); return(new Success(score, (uint)stateUser.TotalLinesExecuted, (uint)codeLength, scoreMode.Hint, (uint)challenge.Inputs.Count), null); }
public static async Task <IExecutionState> Prepare(this IYololExecutor executor, Yolol.Grammar.AST.Program program, string done = ":done") { return((await executor.Prepare(new[] { program }, done)).Single()); }