private async Task NotifyEnd(Challenge.Challenge challenge, IAsyncEnumerable <RankedSolution> solutions)
{
var top = await solutions.Take(10).ToArrayAsync();
var(othersCount, othersScore) = await solutions.Skip(10).Select(a => (1, a.Solution.Score)).AggregateAsync((0u, 0L), (a, b) => ((uint)(a.Item1 + b.Item1), a.Item1 + b.Score));
EmbedBuilder embed = new EmbedBuilder {
Title = $"Competition `{challenge.Name}` Complete!",
Color = Color.Blue,
Footer = new EmbedFooterBuilder().WithText("A Cylon Project")
};
if (top.Length == 0)
{
embed.Description = "Challenge ended with no entries.";
}
else
{
embed.Description = $"**{await _client.GetUserName(top[0].Solution.UserId)}** is victorious with a score of **{top[0].Solution.Score}**\n\n```{top[0].Solution.Yolol}```";
var leaderboardStr = new StringBuilder();
foreach (var item in top)
{
leaderboardStr.AppendLine($"{item.Rank}. {await _client.GetUserName(item.Solution.UserId)} **{item.Solution.Score}**");
}
if (othersCount > 0)
{
leaderboardStr.AppendLine($"\n{othersCount} other entrants scored {othersScore} combined points.");
}
embed.AddField("Leaderboard", leaderboardStr.ToString());
}
await SendEmbedToSubs(embed.Build());
}
private async Task UpdateLeaderboard(Challenge.Challenge challenge, IAsyncEnumerable <RankedSolution> solutions)
{
// Update trueskill
await _skillUpdate.ApplyChallengeResults(challenge.Id);
const uint maxScore = 10;
var score = maxScore;
// Enumerate through each group of equally ranked users
var ranks = solutions.GroupBy(a => a.Rank).OrderBy(a => a.Key);
await foreach (var rank in ranks)
{
// Award all users at the same rank some points
var count = 0;
await foreach (var solution in rank)
{
count++;
await _leaderboard.AddScore(solution.Solution.UserId, score *(uint)challenge.Difficulty);
}
// If there was only one user in the top rank, award them a bonus
if (count == 1 && score == maxScore)
{
await _leaderboard.AddScore((await rank.SingleAsync()).Solution.UserId, (uint)challenge.Difficulty);
}
// Award at least one point to every entrant
if (score > 1)
{
score--;
}
}
// Find the smallest solution, if there's only one of them (i.e. no tie for smallest) award a bonus point
var smallestGroup = await solutions.GroupBy(a => a.Solution.Yolol.Length).AggregateAsync((a, b) => a.Key < b.Key ? a : b);
if (await smallestGroup.CountAsync() == 1)
{
await _leaderboard.AddScore((await smallestGroup.FirstAsync()).Solution.UserId, (uint)challenge.Difficulty);
}
}
public async Task <(Success?, Failure?)> Verify(Challenge.Challenge challenge, string yolol)
{
await Task.CompletedTask;
IScore scoreMode = challenge.ScoreMode switch {
ScoreMode.BasicScoring => new BasicScoring(),
ScoreMode.Approximate => new ApproximateScoring(),
ScoreMode.Unknown => throw new InvalidOperationException("Cannot use `Unknown` score mode (challenge is broken - contact Martin#2468)"),
_ => throw new NotImplementedException($"Score mode `{challenge.ScoreMode}` is not implemented")
};
// Retrieve the test cases for the challenge
var(inputs, outputs) = GetTests(challenge);
// Check input program fits within 20x70
var lines = yolol.Split("\n");
if (lines.Length > 20 || lines.Any(l => l.Length > 70))
{
return(null, new Failure(FailureType.ProgramTooLarge, null));
}
// parse the program
var result = Parser.ParseProgram(yolol);
if (!result.IsOk)
{
return(null, new Failure(FailureType.ParseFailed, result.Err.ToString()));
}
var entry = result.Ok;
// Get the variable which the program uses to indicate it is ready to move to the next round
var state = new MachineState(new DefaultValueDeviceNetwork());
var done = state.GetVariable($":{challenge.CheckIndicator}");
// Begin counting how long it takes to verify (for profiling purposes)
var timer = new Stopwatch();
timer.Start();
// Run through test cases one by one
var overflowIters = _config.MaxItersOverflow;
var totalRuntime = 0u;
var pc = 0;
for (var i = 0; i < Math.Min(inputs.Count, outputs.Count); i++)
{
// Set inputs
var input = inputs[i];
foreach (var(key, value) in input)
{
state.GetVariable($":{key}").Value = value;
}
// Clear completion indicator
done.Value = 0;
// Run lines until completion indicator is set or execution time limit is exceeded
var limit = 0;
while (!done.Value.ToBool())
{
// Check if this test has exceed it's time limit
if (limit++ > _config.MaxTestIters)
{
//If so use iterations from the overflow pool
overflowIters--;
//Once the overflow pool is empty too, fail
if (overflowIters <= 0)
{
return(null, new Failure(FailureType.RuntimeTooLong, null));
}
}
totalRuntime++;
try
{
// If line if blank, just move to the next line
if (pc >= entry.Lines.Count)
{
pc++;
}
else
{
pc = entry.Lines[pc].Evaluate(pc, state);
}
}
catch (ExecutionException)
{
pc++;
}
// loop around if program counter goes over max
if (pc >= 20)
{
pc = 0;
}
}
// Check this test case with the current scoring mode
var scoreFailure = scoreMode.CheckCase(input, outputs[i], state);
if (scoreFailure != null)
{
return(null, scoreFailure);
}
}
Console.WriteLine($"Verified {totalRuntime} ticks, {timer.ElapsedMilliseconds}ms runtime");
// Calculate score
var codeLength = yolol.Replace("\n", "").Length;
var score = scoreMode.FinalizeScore(
(uint)inputs.Count,
totalRuntime,
codeLength
);
return(new Success((uint)score, (uint)totalRuntime, (uint)codeLength), null);
}
private static (IReadOnlyList <IReadOnlyDictionary <string, Value> >, IReadOnlyList <IReadOnlyDictionary <string, Value> >) GetTests(Challenge.Challenge challenge)
{
if (challenge.ShuffleTests)
{
var shuffled = challenge.Inputs.Zip(challenge.Outputs).Shuffle().ToArray();
return(
shuffled.Select(a => a.First).ToArray(),
shuffled.Select(a => a.Second).ToArray()
);
}
else
{
return(challenge.Inputs, challenge.Outputs);
}
}
private async Task NotifyStart(Challenge.Challenge challenge)
{
await SendEmbedToSubs(challenge.ToEmbed().Build());
}
public async Task <(Success?, Failure?)> Verify(Challenge.Challenge challenge, string yolol)
{
await Task.CompletedTask;
if (challenge.ScoreMode != Challenge.ScoreMode.BasicScoring)
{
throw new NotImplementedException($"Score mode `{challenge.ScoreMode}` is not implemented");
}
// Retrieve the test cases for the challenge
var(inputs, outputs) = GetTests(challenge);
// Check input program is 20x70
var lines = yolol.Split("\n");
if (lines.Length > 20 || lines.Any(l => l.Length > 70))
{
return(null, new Failure(FailureType.ProgramTooLarge, null));
}
// parse the entry program
var result = Parser.ParseProgram(yolol);
if (!result.IsOk)
{
return(null, new Failure(FailureType.ParseFailed, result.Err.ToString()));
}
var entry = result.Ok;
// Get the variable which the program uses to indicate it is ready to move to the next round
var state = new MachineState(new DefaultValueDeviceNetwork());
var done = state.GetVariable($":{challenge.CheckIndicator}");
// Begin counting how long it takes to verify
var timer = new Stopwatch();
timer.Start();
// Run through test cases one by one
var overflowIters = _config.MaxItersOverflow;
var totalRuntime = 0u;
var pc = 0;
for (var i = 0; i < Math.Min(inputs.Count, outputs.Count); i++)
{
// Set inputs
var input = inputs[i];
foreach (var(key, value) in input)
{
state.GetVariable($":{key}").Value = value;
}
// Clear completion indicator
done.Value = 0;
// Run lines until completion indicator is set or execution time limit is exceeded
var limit = 0;
while (!done.Value.ToBool())
{
// Check if this test has exceed it's time limit
if (limit++ > _config.MaxTestIters)
{
//If so use iterations from the overflow pool
overflowIters--;
//Once the overflow pool is empty too, fail
if (overflowIters <= 0)
{
return(null, new Failure(FailureType.RuntimeTooLong, null));
}
}
totalRuntime++;
try
{
// If line if blank, just move to the next line
if (pc >= entry.Lines.Count)
{
pc++;
}
else
{
pc = entry.Lines[pc].Evaluate(pc, state);
}
}
catch (ExecutionException)
{
pc++;
}
// loop around if program counter goes over max
if (pc >= 20)
{
pc = 0;
}
}
// Check outputs
foreach (var(key, value) in outputs[i])
{
var v = state.GetVariable($":{key}");
if ((v.Value != value).ToBool())
{
var ii = string.Join(",", input.Select(b => $"`:{b.Key}={b.Value.ToHumanString()}`"));
var oo = string.Join(",", outputs[i].Select(b => $"`:{b.Key}={b.Value.ToHumanString()}`"));
return(null, new Failure(FailureType.IncorrectResult, $"For inputs {ii} expected outputs {oo}, got `{v.Value.ToHumanString()}` for `:{key}`"));
}
}
}
Console.WriteLine($"Verified {totalRuntime} ticks, {timer.ElapsedMilliseconds}ms runtime");
// Calculate score
var codeLength = yolol.Replace("\n", "").Length;
var score = _score.Score(
(uint)inputs.Count,
totalRuntime,
codeLength
);
return(new Success((uint)score, (uint)totalRuntime, (uint)codeLength), null);
}
public async Task <(Success?, Failure?)> Verify(Challenge.Challenge challenge, string yolol)
{
await Task.CompletedTask;
IScore scoreMode = challenge.ScoreMode switch {
ScoreMode.BasicScoring => new BasicScoring(),
ScoreMode.Approximate => new ApproximateScoring(),
ScoreMode.Unknown => throw new InvalidOperationException("Cannot use `Unknown` score mode (challenge is broken - contact Martin#2468)"),
_ => throw new NotImplementedException($"Score mode `{challenge.ScoreMode}` is not implemented (contact Martin#2468)")
};
if (challenge.Chip == YololChip.Unknown)
{
throw new InvalidOperationException("Cannot submit to a challenge with `YololChip.Unknown` (challenge is broken - contact Martin#2468)");
}
if (!challenge.Intermediate.IsOk)
{
throw new InvalidOperationException("Cannot submit to a challenge with broken intermediate code (challenge is broken - contact Martin#2468)");
}
// Check input program fits within 20x70
var lines = yolol.Split("\n");
if (lines.Length > 20 || lines.Any(l => l.Length > 70))
{
return(null, new Failure(FailureType.ProgramTooLarge, null, null));
}
// parse the program
var parsed = Parser.ParseProgram(yolol);
if (!parsed.IsOk)
{
return(null, new Failure(FailureType.ParseFailed, parsed.Err.ToString(), null));
}
// Verify that code is allowed on the given chip level
if (challenge.Chip != YololChip.Professional)
{
var fail = CheckChipLevel(challenge.Chip, parsed.Ok);
if (fail != null)
{
return(null, fail);
}
}
// Prepare a machine state for execution.
// Two states are needed - one for user code and one for code supplied by the challenge
var executionsContexts = (await _executor.Prepare(new[] { parsed.Ok, challenge.Intermediate.Ok }, $":{challenge.CheckIndicator}")).ToList();
var stateUser = executionsContexts[0];
var stateChallenge = executionsContexts[1];
// Retrieve the test cases for the challenge
var(inputs, outputs) = GetTests(challenge);
// Begin counting how long it takes to verify (for profiling purposes)
var timer = new Stopwatch();
timer.Start();
// Run through test cases one by one
var overflowIters = (long)MaxItersOverflow;
for (var i = 0; i < Math.Max(inputs.Count, outputs.Count); i++)
{
// Set inputs in user execution state
var input = SetInputs(i < inputs.Count ? inputs[i] : new Dictionary <string, Value>(), stateUser);
var output = outputs[i];
var savedState = stateUser.Serialize(output);
// Check realtime timeout
if (timer.Elapsed.TotalMilliseconds > TimeoutMillisecondsAll)
{
Console.WriteLine($"Timed out verification: {stateUser.TotalLinesExecuted} ticks, {timer.ElapsedMilliseconds}ms runtime");
return(null, new Failure(FailureType.RuntimeTooLongMilliseconds, $"Execution Timed Out (executed {i} test cases in {timer.Elapsed.TotalMilliseconds}ms)", savedState));
}
// Run the user code until completion
Failure?failure;
(failure, overflowIters) = await RunToDone(stateUser, MaxTestIters, i, inputs.Count, overflowIters, savedState);
if (failure != null)
{
return(null, failure);
}
// Set the challenge inputs _and_ outputs into the challenge state
SetInputs(input, stateChallenge, "input_");
SetInputs(outputs[i], stateChallenge, "output_");
// Run the challenge code
(failure, _) = await RunToDone(stateChallenge, MaxTestIters, 0, 0, MaxItersOverflow, savedState);
if (failure != null)
{
return(null, new Failure(FailureType.ChallengeCodeFailed, failure.Hint, savedState));
}
// Check if the challenge code has forced a failure
var forceFail = stateChallenge.TryGet(new VariableName(":fail"));
if (forceFail?.Type == Yolol.Execution.Type.String && forceFail.Value.String.Length != 0)
{
return(null, new Failure(FailureType.ChallengeForceFail, forceFail.Value.String.ToString(), savedState));
}
// Check this test case with the current scoring mode
var scoreFailure = scoreMode.CheckCase(input, outputs[i], stateUser, savedState);
if (scoreFailure != null)
{
return(null, scoreFailure);
}
}
Console.WriteLine($"Verified {stateUser.TotalLinesExecuted} ticks, {timer.ElapsedMilliseconds}ms runtime");
// Calculate score
var codeLength = yolol.Replace("\n", "").Length;
var score = scoreMode.FinalizeScore(
(uint)Math.Max(inputs.Count, outputs.Count),
(uint)stateUser.TotalLinesExecuted,
codeLength
);
return(new Success(score, (uint)stateUser.TotalLinesExecuted, (uint)codeLength, scoreMode.Hint, (uint)challenge.Inputs.Count), null);
}
