public async Task <(Success?, Failure?)> Verify(Challenge.Challenge challenge, string yolol)
{
await Task.CompletedTask;
IScore scoreMode = challenge.ScoreMode switch {
ScoreMode.BasicScoring => new BasicScoring(),
ScoreMode.Approximate => new ApproximateScoring(),
ScoreMode.Unknown => throw new InvalidOperationException("Cannot use `Unknown` score mode (challenge is broken - contact Martin#2468)"),
_ => throw new NotImplementedException($"Score mode `{challenge.ScoreMode}` is not implemented")
};
// Retrieve the test cases for the challenge
var(inputs, outputs) = GetTests(challenge);
// Check input program fits within 20x70
var lines = yolol.Split("\n");
if (lines.Length > 20 || lines.Any(l => l.Length > 70))
{
return(null, new Failure(FailureType.ProgramTooLarge, null));
}
// parse the program
var result = Parser.ParseProgram(yolol);
if (!result.IsOk)
{
return(null, new Failure(FailureType.ParseFailed, result.Err.ToString()));
}
var entry = result.Ok;
// Get the variable which the program uses to indicate it is ready to move to the next round
var state = new MachineState(new DefaultValueDeviceNetwork());
var done = state.GetVariable($":{challenge.CheckIndicator}");
// Begin counting how long it takes to verify (for profiling purposes)
var timer = new Stopwatch();
timer.Start();
// Run through test cases one by one
var overflowIters = _config.MaxItersOverflow;
var totalRuntime = 0u;
var pc = 0;
for (var i = 0; i < Math.Min(inputs.Count, outputs.Count); i++)
{
// Set inputs
var input = inputs[i];
foreach (var(key, value) in input)
{
state.GetVariable($":{key}").Value = value;
}
// Clear completion indicator
done.Value = 0;
// Run lines until completion indicator is set or execution time limit is exceeded
var limit = 0;
while (!done.Value.ToBool())
{
// Check if this test has exceed it's time limit
if (limit++ > _config.MaxTestIters)
{
//If so use iterations from the overflow pool
overflowIters--;
//Once the overflow pool is empty too, fail
if (overflowIters <= 0)
{
return(null, new Failure(FailureType.RuntimeTooLong, null));
}
}
totalRuntime++;
try
{
// If line if blank, just move to the next line
if (pc >= entry.Lines.Count)
{
pc++;
}
else
{
pc = entry.Lines[pc].Evaluate(pc, state);
}
}
catch (ExecutionException)
{
pc++;
}
// loop around if program counter goes over max
if (pc >= 20)
{
pc = 0;
}
}
// Check this test case with the current scoring mode
var scoreFailure = scoreMode.CheckCase(input, outputs[i], state);
if (scoreFailure != null)
{
return(null, scoreFailure);
}
}
Console.WriteLine($"Verified {totalRuntime} ticks, {timer.ElapsedMilliseconds}ms runtime");
// Calculate score
var codeLength = yolol.Replace("\n", "").Length;
var score = scoreMode.FinalizeScore(
(uint)inputs.Count,
totalRuntime,
codeLength
);
return(new Success((uint)score, (uint)totalRuntime, (uint)codeLength), null);
}
public async Task <(Success?, Failure?)> Verify(Challenge.Challenge challenge, string yolol)
{
await Task.CompletedTask;
IScore scoreMode = challenge.ScoreMode switch {
ScoreMode.BasicScoring => new BasicScoring(),
ScoreMode.Approximate => new ApproximateScoring(),
ScoreMode.Unknown => throw new InvalidOperationException("Cannot use `Unknown` score mode (challenge is broken - contact Martin#2468)"),
_ => throw new NotImplementedException($"Score mode `{challenge.ScoreMode}` is not implemented (contact Martin#2468)")
};
if (challenge.Chip == YololChip.Unknown)
{
throw new InvalidOperationException("Cannot submit to a challenge with `YololChip.Unknown` (challenge is broken - contact Martin#2468)");
}
if (!challenge.Intermediate.IsOk)
{
throw new InvalidOperationException("Cannot submit to a challenge with broken intermediate code (challenge is broken - contact Martin#2468)");
}
// Check input program fits within 20x70
var lines = yolol.Split("\n");
if (lines.Length > 20 || lines.Any(l => l.Length > 70))
{
return(null, new Failure(FailureType.ProgramTooLarge, null, null));
}
// parse the program
var parsed = Parser.ParseProgram(yolol);
if (!parsed.IsOk)
{
return(null, new Failure(FailureType.ParseFailed, parsed.Err.ToString(), null));
}
// Verify that code is allowed on the given chip level
if (challenge.Chip != YololChip.Professional)
{
var fail = CheckChipLevel(challenge.Chip, parsed.Ok);
if (fail != null)
{
return(null, fail);
}
}
// Prepare a machine state for execution.
// Two states are needed - one for user code and one for code supplied by the challenge
var executionsContexts = (await _executor.Prepare(new[] { parsed.Ok, challenge.Intermediate.Ok }, $":{challenge.CheckIndicator}")).ToList();
var stateUser = executionsContexts[0];
var stateChallenge = executionsContexts[1];
// Retrieve the test cases for the challenge
var(inputs, outputs) = GetTests(challenge);
// Begin counting how long it takes to verify (for profiling purposes)
var timer = new Stopwatch();
timer.Start();
// Run through test cases one by one
var overflowIters = (long)MaxItersOverflow;
for (var i = 0; i < Math.Max(inputs.Count, outputs.Count); i++)
{
// Set inputs in user execution state
var input = SetInputs(i < inputs.Count ? inputs[i] : new Dictionary <string, Value>(), stateUser);
var output = outputs[i];
var savedState = stateUser.Serialize(output);
// Check realtime timeout
if (timer.Elapsed.TotalMilliseconds > TimeoutMillisecondsAll)
{
Console.WriteLine($"Timed out verification: {stateUser.TotalLinesExecuted} ticks, {timer.ElapsedMilliseconds}ms runtime");
return(null, new Failure(FailureType.RuntimeTooLongMilliseconds, $"Execution Timed Out (executed {i} test cases in {timer.Elapsed.TotalMilliseconds}ms)", savedState));
}
// Run the user code until completion
Failure?failure;
(failure, overflowIters) = await RunToDone(stateUser, MaxTestIters, i, inputs.Count, overflowIters, savedState);
if (failure != null)
{
return(null, failure);
}
// Set the challenge inputs _and_ outputs into the challenge state
SetInputs(input, stateChallenge, "input_");
SetInputs(outputs[i], stateChallenge, "output_");
// Run the challenge code
(failure, _) = await RunToDone(stateChallenge, MaxTestIters, 0, 0, MaxItersOverflow, savedState);
if (failure != null)
{
return(null, new Failure(FailureType.ChallengeCodeFailed, failure.Hint, savedState));
}
// Check if the challenge code has forced a failure
var forceFail = stateChallenge.TryGet(new VariableName(":fail"));
if (forceFail?.Type == Yolol.Execution.Type.String && forceFail.Value.String.Length != 0)
{
return(null, new Failure(FailureType.ChallengeForceFail, forceFail.Value.String.ToString(), savedState));
}
// Check this test case with the current scoring mode
var scoreFailure = scoreMode.CheckCase(input, outputs[i], stateUser, savedState);
if (scoreFailure != null)
{
return(null, scoreFailure);
}
}
Console.WriteLine($"Verified {stateUser.TotalLinesExecuted} ticks, {timer.ElapsedMilliseconds}ms runtime");
// Calculate score
var codeLength = yolol.Replace("\n", "").Length;
var score = scoreMode.FinalizeScore(
(uint)Math.Max(inputs.Count, outputs.Count),
(uint)stateUser.TotalLinesExecuted,
codeLength
);
return(new Success(score, (uint)stateUser.TotalLinesExecuted, (uint)codeLength, scoreMode.Hint, (uint)challenge.Inputs.Count), null);
}
