public async Task InsertRetrieveCurrentNoisesTasks(Round round, Configuration config)
{
// getnoises are started in Q1
double maxRunningTime = config.RoundLengthInSeconds / 4;
var taskStart = round.Quarter3;
int tasksCount = 0;
foreach (var service in config.ActiveServices)
{
tasksCount += service.NoisesPerRound * config.ActiveTeams.Count;
}
if (tasksCount == 0)
{
return;
}
double timeDiff = (maxRunningTime - 2) / tasksCount;
var tasks = new CheckerTask[tasksCount];
int i = 0;
foreach (var service in config.ActiveServices)
{
var checkers = config.Checkers[service.Id];
foreach (var team in config.ActiveTeams)
{
for (int taskIndex = 0; taskIndex < service.NoisesPerRound; taskIndex++)
{
var checkerTask = new CheckerTask(
0,
checkers[i % checkers.Length],
CheckerTaskMethod.getnoise,
team.Address ?? $"team{team.Id}.{config.DnsSuffix}",
service.Id,
service.Name,
team.Id,
team.Name,
round.Id,
round.Id,
null,
taskStart,
(int)(maxRunningTime * 1000),
config.RoundLengthInSeconds,
taskIndex,
taskIndex % service.NoiseVariants,
CheckerResult.INTERNAL_ERROR,
null,
null,
CheckerTaskLaunchStatus.New);
tasks[i] = checkerTask;
taskStart = taskStart.AddSeconds(timeDiff);
i += 1;
}
}
}
// TODO shuffle
await this.InsertCheckerTasks(tasks);
}
public static IDisposable BeginEnoScope(this ILogger logger, CheckerTask checkerTask)
{
return(logger.BeginScope(new Dictionary <string, object>
{
[nameof(CheckerTask)] = checkerTask,
}));
}
public async Task InsertPutFlagsTasks(Round round, Configuration config)
{
// putflags are started in Q1
double maxRunningTime = config.RoundLengthInSeconds / 4;
var taskStart = round.Begin;
int tasksCount = 0;
foreach (var service in config.ActiveServices)
{
tasksCount += service.FlagsPerRound * config.ActiveTeams.Count;
}
if (tasksCount == 0)
{
return;
}
double timeDiff = (maxRunningTime - 2) / tasksCount;
var tasks = new CheckerTask[tasksCount];
int i = 0;
foreach (var service in config.ActiveServices)
{
var checkers = config.Checkers[service.Id];
foreach (var team in config.ActiveTeams)
{
for (int taskIndex = 0; taskIndex < service.FlagsPerRound; taskIndex++)
{
var checkerTask = new CheckerTask(
0,
checkers[i % checkers.Length],
CheckerTaskMethod.putflag,
team.Address ?? $"team{team.Id}.{config.DnsSuffix}",
service.Id,
service.Name,
team.Id,
team.Name,
round.Id,
round.Id,
new Flag(team.Id, service.Id, taskIndex, round.Id, 0).ToString(Encoding.ASCII.GetBytes(config.FlagSigningKey), config.Encoding),
taskStart,
(int)(maxRunningTime * 1000),
config.RoundLengthInSeconds,
taskIndex,
taskIndex % service.FlagVariants,
CheckerResult.INTERNAL_ERROR,
null,
null,
CheckerTaskLaunchStatus.New);
tasks[i] = checkerTask;
taskStart = taskStart.AddSeconds(timeDiff);
i += 1;
}
}
}
// TODO shuffle
await this.InsertCheckerTasks(tasks);
}
internal async Task UpdateDatabaseLoop()
{
try
{
while (!LauncherCancelSource.IsCancellationRequested)
{
CheckerTask[]? results = new CheckerTask[TaskUpdateBatchSize];
int i = 0;
for (; i < TaskUpdateBatchSize; i++)
{
if (ResultsQueue.TryDequeue(out var result))
{
results[i] = result;
}
else
{
break;
}
}
try
{
using (var scope = this.serviceProvider.CreateScope())
{
var db = scope.ServiceProvider.GetRequiredService <IEnoDatabase>();
await db.UpdateTaskCheckerTaskResults(results.AsMemory(0, i));
}
if (i != TaskUpdateBatchSize)
{
await Task.Delay(1);
}
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception e)
{
this.logger.LogInformation($"UpdateDatabase dropping update because: {e.ToFancyStringWithCaller()}");
if (results != null)
{
foreach (var task in results)
{
this.logger.LogCritical(task.ToString());
}
}
}
}
}
catch (TaskCanceledException)
{
}
catch (Exception e)
{
this.logger.LogCritical($"UpdateDatabase failed : {e.ToFancyStringWithCaller()}");
}
}
public async Task <List <CheckerTask> > RetrievePendingCheckerTasks(int maxAmount)
{
var strategy = this.context.Database.CreateExecutionStrategy();
return(await strategy.ExecuteAsync(async() =>
{
using var transaction = this.context.Database.BeginTransaction(IsolationLevel.Serializable);
try
{
var tasks = await this.context.CheckerTasks
.Where(t => t.CheckerTaskLaunchStatus == CheckerTaskLaunchStatus.New)
.OrderBy(t => t.StartTime)
.Take(maxAmount)
.AsNoTracking()
.ToListAsync();
var launchedTasks = new CheckerTask[tasks.Count];
// TODO update launch status without delaying operation
for (int i = 0; i < launchedTasks.Length; i++)
{
launchedTasks[i] = tasks[i] with {
CheckerTaskLaunchStatus = CheckerTaskLaunchStatus.Launched
};
}
this.context.UpdateRange(launchedTasks);
await this.context.SaveChangesAsync();
await transaction.CommitAsync();
return tasks;
}
catch (Exception e)
{
await transaction.RollbackAsync();
this.logger.LogDebug($"RetrievePendingCheckerTasks: Rolling Back Transaction{e.ToFancyStringWithCaller()}");
throw new Exception(e.Message, e.InnerException);
}
}));
}
public async Task LaunchCheckerTask(CheckerTask task)
{
using var scope = this.logger.BeginEnoScope(task);
try
{
this.logger.LogTrace($"LaunchCheckerTask() for task {task.Id} ({task.Method}, currentRound={task.CurrentRoundId}, relatedRound={task.RelatedRoundId})");
var cancelSource = new CancellationTokenSource();
var now = DateTime.UtcNow;
var span = task.StartTime.Subtract(DateTime.UtcNow);
if (span.TotalSeconds < -0.5)
{
this.logger.LogWarning($"Task {task.Id} starts {span.TotalSeconds} late (should: {task.StartTime})");
}
if (task.StartTime > now)
{
this.logger.LogTrace($"Task {task.Id} sleeping: {span}");
await Task.Delay(span);
}
string taskChainId = task.Method switch
{
CheckerTaskMethod.putflag => $"flag_{task.ServiceId}_r{task.RelatedRoundId}_t{task.TeamId}_i{task.UniqueVariantId}",
CheckerTaskMethod.getflag => $"flag_{task.ServiceId}_r{task.RelatedRoundId}_t{task.TeamId}_i{task.UniqueVariantId}",
CheckerTaskMethod.putnoise => $"noise_{task.ServiceId}_r{task.RelatedRoundId}_t{task.TeamId}_i{task.UniqueVariantId}",
CheckerTaskMethod.getnoise => $"noise_{task.ServiceId}_r{task.RelatedRoundId}_t{task.TeamId}_i{task.UniqueVariantId}",
CheckerTaskMethod.havoc => $"havoc_{task.ServiceId}_r{task.RelatedRoundId}_t{task.TeamId}_i{task.UniqueVariantId}",
_ => throw new NotImplementedException(),
};
var content = new StringContent(
JsonSerializer.Serialize(
new CheckerTaskMessage(
task.Id,
task.Method,
task.Address,
task.TeamId,
task.TeamName,
task.CurrentRoundId,
task.RelatedRoundId,
task.Payload,
task.VariantId,
task.MaxRunningTime,
task.RoundLength,
taskChainId),
EnoCoreUtil.CamelCaseEnumConverterOptions),
Encoding.UTF8,
"application/json");
cancelSource.CancelAfter(task.MaxRunningTime);
this.statistics.LogCheckerTaskLaunchMessage(task);
this.logger.LogDebug($"LaunchCheckerTask {task.Id} POSTing {task.Method} to checker");
var response = await Client.PostAsync(task.CheckerUrl, content, cancelSource.Token);
if (response.StatusCode == HttpStatusCode.OK)
{
var responseString = (await response.Content.ReadAsStringAsync()).TrimEnd(Environment.NewLine.ToCharArray());
this.logger.LogDebug($"LaunchCheckerTask received {responseString}");
var resultMessage = JsonSerializer.Deserialize <CheckerResultMessage>(responseString, EnoCoreUtil.CamelCaseEnumConverterOptions);
var checkerResult = resultMessage !.Result;
this.logger.LogDebug($"LaunchCheckerTask {task.Id} returned {checkerResult} with Message {resultMessage.Message}");
CheckerTask updatedTask = task with {
CheckerResult = checkerResult, ErrorMessage = resultMessage.Message, CheckerTaskLaunchStatus = CheckerTaskLaunchStatus.Done
};
this.statistics.LogCheckerTaskFinishedMessage(updatedTask);
ResultsQueue.Enqueue(updatedTask);
return;
}
else
{
this.logger.LogError($"LaunchCheckerTask {task.Id} {task.Method} returned {response.StatusCode} ({(int)response.StatusCode})");
var updatedTask = task with {
CheckerResult = CheckerResult.INTERNAL_ERROR, CheckerTaskLaunchStatus = CheckerTaskLaunchStatus.Done
};
this.statistics.LogCheckerTaskFinishedMessage(updatedTask);
ResultsQueue.Enqueue(updatedTask);
return;
}
}
catch (TaskCanceledException)
{
this.logger.LogError($"{nameof(this.LaunchCheckerTask)} {task.Id} {task.Method} was cancelled because it did not finish");
var updatedTask = task with {
CheckerResult = CheckerResult.OFFLINE, CheckerTaskLaunchStatus = CheckerTaskLaunchStatus.Done
};
this.statistics.LogCheckerTaskFinishedMessage(updatedTask);
ResultsQueue.Enqueue(updatedTask);
}
catch (Exception e)
{
this.logger.LogError($"{nameof(this.LaunchCheckerTask)} {task.Id} failed: {e.ToFancyStringWithCaller()}");
var updatedTask = task with {
CheckerResult = CheckerResult.INTERNAL_ERROR, CheckerTaskLaunchStatus = CheckerTaskLaunchStatus.Done
};
this.statistics.LogCheckerTaskFinishedMessage(updatedTask);
ResultsQueue.Enqueue(updatedTask);
}
}
public async Task InsertRetrieveCurrentNoisesTasks(
Round round,
Team[] activeTeams,
Service[] activeServices,
Configuration configuration)
{
// getnoises are started in Q3
double quarterLength = configuration.RoundLengthInSeconds / 4;
int tasksCount = 0;
foreach (var service in activeServices)
{
tasksCount += (int)service.NoisesPerRound * activeTeams.Length;
}
if (tasksCount == 0)
{
return;
}
var tasks = new List <CheckerTask>(tasksCount);
var taskStart = round.Quarter3;
double teamStepLength = (quarterLength - 2) / activeTeams.Length;
double serviceStepLength = teamStepLength / activeServices.Length;
foreach (var service in activeServices)
{
double serviceOffset = (service.Id - 1) * serviceStepLength;
for (int variantIndex = 0; variantIndex < service.NoisesPerRound; variantIndex++)
{
double variantOffset = variantIndex * (serviceStepLength / service.NoisesPerRound);
int i = 0;
var currentTasks = new List <CheckerTask>();
foreach (var team in activeTeams)
{
double taskOffset = (i * teamStepLength) + serviceOffset + variantOffset;
// this.logger.LogDebug($"InsertRetrieveCurrentNoisesTasks Q3 taskOffset={taskOffset}");
var checkerTask = new CheckerTask(
0,
service.Checkers[i % service.Checkers.Length],
CheckerTaskMethod.getnoise,
team.Address ?? $"team{team.Id}.{configuration.DnsSuffix}",
service.Id,
service.Name,
team.Id,
team.Name,
round.Id,
round.Id,
null,
taskStart.AddSeconds(taskOffset),
(int)(quarterLength * 1000),
configuration.RoundLengthInSeconds,
variantIndex,
variantIndex % service.NoiseVariants,
CheckerResult.INTERNAL_ERROR,
null,
null,
CheckerTaskLaunchStatus.New);
currentTasks.Add(checkerTask);
i += 1;
}
this.FisherYatesShuffleTaskStarts(currentTasks);
tasks.AddRange(currentTasks);
}
}
await this.InsertCheckerTasks(tasks);
}
public async Task InsertRetrieveOldFlagsTasks(
Round round,
Team[] activeTeams,
Service[] activeServices,
Configuration configuration)
{
// getflags for old flags are started in Q2 and Q3
double quarterLength = configuration.RoundLengthInSeconds / 4;
int tasksCount = 0;
int oldRoundsCount = (int)Math.Min(configuration.CheckedRoundsPerRound, round.Id) - 1;
foreach (var service in activeServices)
{
tasksCount += (int)service.FlagsPerRound
* activeTeams.Length
* oldRoundsCount;
}
if (tasksCount == 0)
{
return;
}
var tasks = new List <CheckerTask>(tasksCount);
var taskStart = round.Quarter2;
for (long oldRoundId = round.Id - 1; oldRoundId > (round.Id - configuration.CheckedRoundsPerRound) && oldRoundId > 0; oldRoundId--)
{
double teamStepLength = ((2 * quarterLength) - 2) / activeTeams.Length;
double serviceStepLength = teamStepLength / activeServices.Length;
foreach (var service in activeServices)
{
double serviceOffset = (service.Id - 1) * serviceStepLength;
for (int variantIndex = 0; variantIndex < service.FlagsPerRound; variantIndex++)
{
double variantOffset = variantIndex * (serviceStepLength / service.FlagsPerRound);
int i = 0;
var currentTasks = new List <CheckerTask>();
foreach (var team in activeTeams)
{
double taskOffset = (i * teamStepLength) + serviceOffset + variantOffset;
var checkerTask = new CheckerTask(
0,
service.Checkers[i % service.Checkers.Length],
CheckerTaskMethod.getflag,
team.Address ?? $"team{team.Id}.{configuration.DnsSuffix}",
service.Id,
service.Name,
team.Id,
team.Name,
oldRoundId,
round.Id,
new Flag(team.Id, service.Id, variantIndex, oldRoundId).ToString(configuration.FlagSigningKey, configuration.Encoding),
taskStart.AddSeconds(taskOffset),
(int)(quarterLength * 1000),
configuration.RoundLengthInSeconds,
variantIndex,
variantIndex % service.FlagVariants,
CheckerResult.INTERNAL_ERROR,
null,
null,
CheckerTaskLaunchStatus.New);
currentTasks.Add(checkerTask);
i += 1;
}
this.FisherYatesShuffleTaskStarts(currentTasks);
tasks.AddRange(currentTasks);
}
}
}
await this.InsertCheckerTasks(tasks);
}
public void LogCheckerTaskFinishedMessage(CheckerTask task)
{
var msg = CheckerTaskFinishedMessage.FromCheckerTask(task);
this.queue.Enqueue(PREFIX + JsonSerializer.Serialize(msg) + "\n");
}
public void LogCheckerTaskLaunchMessage(CheckerTask task)
{
var message = CheckerTaskLaunchMessage.FromCheckerTask(task);
this.queue.Enqueue(PREFIX + JsonSerializer.Serialize(message) + "\n");
}
