private async Task Evaluate()
{
IExecutionState?previous = null;
foreach (var message in _messages)
{
// Skip forward to the first message with a null state (i.e. needs evaluating)
if (message.State != null)
{
previous = message.State;
continue;
}
previous = await EvaluateSingle(message.Input, message.Response, previous);
}
async Task <IExecutionState?> EvaluateSingle(IMessage input, IUserMessage output, IExecutionState?prevState)
{
// Try to get code from message
var code = input.Content.ExtractYololCodeBlock();
if (code == null)
{
await output.ModifyAsync2(@"Failed to parse a yolol program from message - ensure you have enclosed your solution in triple backticks \`\`\`like this\`\`\`");
return(prevState);
}
// Try to parse code as Yolol
var result = Yolol.Grammar.Parser.ParseProgram(code);
if (!result.IsOk)
{
await output.ModifyAsync2(result.Err.ToString());
return(prevState);
}
// Setup a new state
var state = _executor.Prepare(result.Ok);
state.TerminateOnPcOverflow = true;
prevState?.CopyTo(state);
// Execute for 2000 ticks
var err = state.Run(2000, TimeSpan.FromMilliseconds(150));
// Print out result
if (err != null)
{
await output.ModifyAsync2(err);
}
else
{
await output.ModifyAsync2(embed : state.ToEmbed().Build());
}
return(state);
}
}
private async Task RunYolol(string input, IYololExecutor executor, uint lines)
{
// Try to parse code as Yolol
var result = await Parse(input);
if (result == null)
{
return;
}
// Prep execution state
var compileTimer = new Stopwatch();
compileTimer.Start();
var state = await executor.Prepare(result);
compileTimer.Stop();
// Run for N lines, 500ms or until `:done!=0`
var exeTimer = new Stopwatch();
exeTimer.Start();
var saved = state.Serialize();
var err = await state.Run(lines, TimeSpan.FromMilliseconds(500));
// Print out the final machine state
var embed = state.ToEmbed(embed =>
{
embed.AddField("Setup", $"{compileTimer.ElapsedMilliseconds}ms", true);
embed.AddField("Execute", $"{exeTimer.ElapsedMilliseconds}ms", true);
if (err != null)
{
embed.AddField("Error", $"{err}", false);
}
if (_debuggerUrl != null)
{
var base64 = saved.Serialize();
embed.WithUrl($"{_debuggerUrl}?state={base64}");
}
});
await ReplyAsync(embed : embed.Build());
}
private async Task RunYolol(string input, IYololExecutor executor)
{
// Try to parse code as Yolol
var result = await Parse(input);
if (result == null)
{
return;
}
// Prep execution state
var compileTimer = new Stopwatch();
compileTimer.Start();
var state = executor.Prepare(result);
compileTimer.Stop();
// Run for 2000 lines, 500ms or until `:done!=0`
var exeTimer = new Stopwatch();
exeTimer.Start();
var err = state.Run(2000, TimeSpan.FromMilliseconds(500));
// Print out error if execution terminated for some reason
if (err != null)
{
await ReplyAsync(err);
return;
}
// Print out the final machine state
var embed = state.ToEmbed(embed => {
embed.AddField("Setup", $"{compileTimer.ElapsedMilliseconds}ms", true);
embed.AddField("Execute", $"{exeTimer.ElapsedMilliseconds}ms", true);
});
await ReplyAsync(embed : embed.Build());
}
public async Task <(Success?, Failure?)> Verify(Challenge.Challenge challenge, string yolol)
{
await Task.CompletedTask;
IScore scoreMode = challenge.ScoreMode switch {
ScoreMode.BasicScoring => new BasicScoring(),
ScoreMode.Approximate => new ApproximateScoring(),
ScoreMode.Unknown => throw new InvalidOperationException("Cannot use `Unknown` score mode (challenge is broken - contact Martin#2468)"),
_ => throw new NotImplementedException($"Score mode `{challenge.ScoreMode}` is not implemented (contact Martin#2468)")
};
if (challenge.Chip == YololChip.Unknown)
{
throw new InvalidOperationException("Cannot submit to a challenge with `YololChip.Unknown` (challenge is broken - contact Martin#2468)");
}
if (!challenge.Intermediate.IsOk)
{
throw new InvalidOperationException("Cannot submit to a challenge with broken intermediate code (challenge is broken - contact Martin#2468)");
}
// Check input program fits within 20x70
var lines = yolol.Split("\n");
if (lines.Length > 20 || lines.Any(l => l.Length > 70))
{
return(null, new Failure(FailureType.ProgramTooLarge, null, null));
}
// parse the program
var parsed = Parser.ParseProgram(yolol);
if (!parsed.IsOk)
{
return(null, new Failure(FailureType.ParseFailed, parsed.Err.ToString(), null));
}
// Verify that code is allowed on the given chip level
if (challenge.Chip != YololChip.Professional)
{
var fail = CheckChipLevel(challenge.Chip, parsed.Ok);
if (fail != null)
{
return(null, fail);
}
}
// Prepare a machine state for execution.
// Two states are needed - one for user code and one for code supplied by the challenge
var executionsContexts = (await _executor.Prepare(new[] { parsed.Ok, challenge.Intermediate.Ok }, $":{challenge.CheckIndicator}")).ToList();
var stateUser = executionsContexts[0];
var stateChallenge = executionsContexts[1];
// Retrieve the test cases for the challenge
var(inputs, outputs) = GetTests(challenge);
// Begin counting how long it takes to verify (for profiling purposes)
var timer = new Stopwatch();
timer.Start();
// Run through test cases one by one
var overflowIters = (long)MaxItersOverflow;
for (var i = 0; i < Math.Max(inputs.Count, outputs.Count); i++)
{
// Set inputs in user execution state
var input = SetInputs(i < inputs.Count ? inputs[i] : new Dictionary <string, Value>(), stateUser);
var output = outputs[i];
var savedState = stateUser.Serialize(output);
// Check realtime timeout
if (timer.Elapsed.TotalMilliseconds > TimeoutMillisecondsAll)
{
Console.WriteLine($"Timed out verification: {stateUser.TotalLinesExecuted} ticks, {timer.ElapsedMilliseconds}ms runtime");
return(null, new Failure(FailureType.RuntimeTooLongMilliseconds, $"Execution Timed Out (executed {i} test cases in {timer.Elapsed.TotalMilliseconds}ms)", savedState));
}
// Run the user code until completion
Failure?failure;
(failure, overflowIters) = await RunToDone(stateUser, MaxTestIters, i, inputs.Count, overflowIters, savedState);
if (failure != null)
{
return(null, failure);
}
// Set the challenge inputs _and_ outputs into the challenge state
SetInputs(input, stateChallenge, "input_");
SetInputs(outputs[i], stateChallenge, "output_");
// Run the challenge code
(failure, _) = await RunToDone(stateChallenge, MaxTestIters, 0, 0, MaxItersOverflow, savedState);
if (failure != null)
{
return(null, new Failure(FailureType.ChallengeCodeFailed, failure.Hint, savedState));
}
// Check if the challenge code has forced a failure
var forceFail = stateChallenge.TryGet(new VariableName(":fail"));
if (forceFail?.Type == Yolol.Execution.Type.String && forceFail.Value.String.Length != 0)
{
return(null, new Failure(FailureType.ChallengeForceFail, forceFail.Value.String.ToString(), savedState));
}
// Check this test case with the current scoring mode
var scoreFailure = scoreMode.CheckCase(input, outputs[i], stateUser, savedState);
if (scoreFailure != null)
{
return(null, scoreFailure);
}
}
Console.WriteLine($"Verified {stateUser.TotalLinesExecuted} ticks, {timer.ElapsedMilliseconds}ms runtime");
// Calculate score
var codeLength = yolol.Replace("\n", "").Length;
var score = scoreMode.FinalizeScore(
(uint)Math.Max(inputs.Count, outputs.Count),
(uint)stateUser.TotalLinesExecuted,
codeLength
);
return(new Success(score, (uint)stateUser.TotalLinesExecuted, (uint)codeLength, scoreMode.Hint, (uint)challenge.Inputs.Count), null);
}
public static async Task <IExecutionState> Prepare(this IYololExecutor executor, Yolol.Grammar.AST.Program program, string done = ":done")
{
return((await executor.Prepare(new[] { program }, done)).Single());
}