public async Task CheckScheduledExecutions()
{
bool printedMessage = false;
var stopwatch = new Stopwatch();
ConcurrentDictionary <Guid, DateTime> skipSchedules = new ConcurrentDictionary <Guid, DateTime>();
var lastLatencyCheck = DateTime.Now;
long maxLatency = 0;
while (true)
{
try
{
List <Guid> enableScheduleList = new List <Guid>();
foreach (var skippedSchedule in skipSchedules)
{
if (skippedSchedule.Value < DateTime.UtcNow)
{
enableScheduleList.Add(skippedSchedule.Key);
}
}
foreach (var es in enableScheduleList)
{
skipSchedules.TryRemove(es, out _);
}
// _logger.LogInformation("Starting scheduler loop.");
//Do not run if it is uninitialized
if (ClusterStateService.Initialized && _nodeStateService.Role == ConsensusCore.Domain.Enums.NodeState.Leader)
{
stopwatch.Restart();
var startDate = DateTime.Now;
if (!printedMessage)
{
_logger.LogInformation("Detected I am cluster leader, starting to run schedule up cluster");
printedMessage = true;
}
Guid[] skip = skipSchedules.Select(ss => ss.Key).ToArray();
var pageSize = 10;
var totalSize = _entitiesRepository.Count <ExecutionSchedule>(e => (e.NextRun == null || e.NextRun < DateTime.Now && e.IsDisabled == false) && !skip.Contains(e.Id));
int runTasks = 0;
_logger.LogDebug("Found " + totalSize + " execution schedules to execute.");
for (var i = 0; i < (totalSize + pageSize - 1) / pageSize; i++)
{
_logger.LogDebug("Pulling " + (i * pageSize + 1) + "-" + (i * pageSize + pageSize));
var executionSchedules = await _entitiesRepository.GetAsync <ExecutionSchedule>(e => (e.NextRun == null || e.NextRun < DateTime.Now && e.IsDisabled == false) && !skip.Contains(e.Id), null, null, pageSize, i);
var tasks = executionSchedules.Select((es) => Task.Run(async() =>
{
try
{
await _mediator.Send(new RecalculateExecutionScheduleCommand()
{
Name = es.Name
});
_logger.LogDebug("Executing schedule " + es.Name + " last run " + es.NextRun.ToString("o") + " current run " + DateTime.Now.ToString("o"));
await _mediator.Send(new ExecuteExecutionTemplateCommand()
{
Name = es.ExecutionTemplateName,
ExecutionScheduleId = es.Id,
CreatedBy = SystemUsers.SCHEDULE_MANAGER
});
}
catch (Exception e)
{
_logger.LogError("Failed to run schedule with exception " + e.Message + " skipping the schedule for 5 minutes" + Environment.NewLine + e.StackTrace);
skipSchedules.TryAdd(es.Id, DateTime.UtcNow.AddMinutes(5));
}
Interlocked.Increment(ref runTasks);
}));
await Task.WhenAll(tasks);
}
var totalTime = stopwatch.ElapsedMilliseconds;
_metricManagementService.EnqueueTick(new MetricTick()
{
MetricId = 0,
Date = DateTime.Now,
Value = totalTime
});
if (TimeSpan.FromMilliseconds(totalTime) > TimeSpan.FromSeconds(5))
{
_logger.LogWarning("Scheduler took longer then 5 second to complete a loop, submitted " + runTasks + " steps, took " + (totalTime) + " milliseconds.");
}
if (maxLatency < totalTime)
{
maxLatency = totalTime;
}
if ((DateTime.Now - lastLatencyCheck).TotalMilliseconds > _clusterOptions.Value.MetricsIntervalMs)
{
_metricManagementService.EnqueueTick(new MetricTick()
{
MetricId = (int)MetricIds.SchedulerLatencyMs,
Date = startDate,
Value = totalTime
});
_logger.LogDebug("Adding a scheduler metric with " + totalTime + " for " + DateTime.Now.ToString("o"));
lastLatencyCheck = DateTime.Now;
maxLatency = 0;
}
if (totalTime < 1000)
{
await Task.Delay(1000 - Convert.ToInt32(totalTime));
}
}
else
{
printedMessage = false;
await Task.Delay(3000);
}
}
catch (Exception e)
{
_logger.LogError("Encountered error " + e.Message + " while trying to complete scheduled execution loop." + Environment.NewLine + e.StackTrace);
}
}
}
