public async Task <QueryResult <List <T> > > Handle(GetEntitiesQuery <T> request, CancellationToken cancellationToken)
{
var stopwatch = new Stopwatch();
stopwatch.Start();
System.Linq.Expressions.Expression <Func <T, bool> > expression;
if (request.Expression != null)
{
expression = request.Expression;
}
else
{
expression = (s) => true;
}
var entities = (await _entitiesRepository.GetAsync <T>(expression, request.Exclusions, request.Sort, request.Size, request.Page)).ToList();
stopwatch.Stop();
return(new QueryResult <List <T> >()
{
Result = entities,
Count = _entitiesRepository.Count <T>(expression),
ElapsedMs = stopwatch.ElapsedMilliseconds
});
}
public async Task <QueryResult <ClusterStats> > Handle(GetClusterStatsQuery request, CancellationToken cancellationToken)
{
var stopwatch = new Stopwatch();
stopwatch.Start();
var stats = new ClusterStats()
{
Steps = new StepStats()
{
Suspended = _entitiesRepository.Count <Step>(s => s.Status == StepStatuses.Suspended),
Unassigned = _entitiesRepository.Count <Step>(s => s.Status == StepStatuses.Unassigned),
Assigned = _entitiesRepository.Count <Step>(s => s.Status == StepStatuses.Assigned),
Successful = _entitiesRepository.Count <Step>(s => s.Status == StepStatuses.Successful),
Warning = _entitiesRepository.Count <Step>(s => s.Status == StepStatuses.Warning),
Error = _entitiesRepository.Count <Step>(s => s.Status == StepStatuses.Error),
}
};
return(new QueryResult <ClusterStats>()
{
Result = stats,
ElapsedMs = stopwatch.ElapsedMilliseconds
});
}
public async Task CheckScheduledExecutions()
{
bool printedMessage = false;
var stopwatch = new Stopwatch();
ConcurrentDictionary <Guid, DateTime> skipSchedules = new ConcurrentDictionary <Guid, DateTime>();
var lastLatencyCheck = DateTime.Now;
long maxLatency = 0;
while (true)
{
try
{
List <Guid> enableScheduleList = new List <Guid>();
foreach (var skippedSchedule in skipSchedules)
{
if (skippedSchedule.Value < DateTime.UtcNow)
{
enableScheduleList.Add(skippedSchedule.Key);
}
}
foreach (var es in enableScheduleList)
{
skipSchedules.TryRemove(es, out _);
}
// _logger.LogInformation("Starting scheduler loop.");
//Do not run if it is uninitialized
if (ClusterStateService.Initialized && _nodeStateService.Role == ConsensusCore.Domain.Enums.NodeState.Leader)
{
stopwatch.Restart();
var startDate = DateTime.Now;
if (!printedMessage)
{
_logger.LogInformation("Detected I am cluster leader, starting to run schedule up cluster");
printedMessage = true;
}
Guid[] skip = skipSchedules.Select(ss => ss.Key).ToArray();
var pageSize = 10;
var totalSize = _entitiesRepository.Count <ExecutionSchedule>(e => (e.NextRun == null || e.NextRun < DateTime.Now && e.IsDisabled == false) && !skip.Contains(e.Id));
int runTasks = 0;
_logger.LogDebug("Found " + totalSize + " execution schedules to execute.");
for (var i = 0; i < (totalSize + pageSize - 1) / pageSize; i++)
{
_logger.LogDebug("Pulling " + (i * pageSize + 1) + "-" + (i * pageSize + pageSize));
var executionSchedules = await _entitiesRepository.GetAsync <ExecutionSchedule>(e => (e.NextRun == null || e.NextRun < DateTime.Now && e.IsDisabled == false) && !skip.Contains(e.Id), null, null, pageSize, i);
var tasks = executionSchedules.Select((es) => Task.Run(async() =>
{
try
{
await _mediator.Send(new RecalculateExecutionScheduleCommand()
{
Name = es.Name
});
_logger.LogDebug("Executing schedule " + es.Name + " last run " + es.NextRun.ToString("o") + " current run " + DateTime.Now.ToString("o"));
await _mediator.Send(new ExecuteExecutionTemplateCommand()
{
Name = es.ExecutionTemplateName,
ExecutionScheduleId = es.Id,
CreatedBy = SystemUsers.SCHEDULE_MANAGER
});
}
catch (Exception e)
{
_logger.LogError("Failed to run schedule with exception " + e.Message + " skipping the schedule for 5 minutes" + Environment.NewLine + e.StackTrace);
skipSchedules.TryAdd(es.Id, DateTime.UtcNow.AddMinutes(5));
}
Interlocked.Increment(ref runTasks);
}));
await Task.WhenAll(tasks);
}
var totalTime = stopwatch.ElapsedMilliseconds;
_metricManagementService.EnqueueTick(new MetricTick()
{
MetricId = 0,
Date = DateTime.Now,
Value = totalTime
});
if (TimeSpan.FromMilliseconds(totalTime) > TimeSpan.FromSeconds(5))
{
_logger.LogWarning("Scheduler took longer then 5 second to complete a loop, submitted " + runTasks + " steps, took " + (totalTime) + " milliseconds.");
}
if (maxLatency < totalTime)
{
maxLatency = totalTime;
}
if ((DateTime.Now - lastLatencyCheck).TotalMilliseconds > _clusterOptions.Value.MetricsIntervalMs)
{
_metricManagementService.EnqueueTick(new MetricTick()
{
MetricId = (int)MetricIds.SchedulerLatencyMs,
Date = startDate,
Value = totalTime
});
_logger.LogDebug("Adding a scheduler metric with " + totalTime + " for " + DateTime.Now.ToString("o"));
lastLatencyCheck = DateTime.Now;
maxLatency = 0;
}
if (totalTime < 1000)
{
await Task.Delay(1000 - Convert.ToInt32(totalTime));
}
}
else
{
printedMessage = false;
await Task.Delay(3000);
}
}
catch (Exception e)
{
_logger.LogError("Encountered error " + e.Message + " while trying to complete scheduled execution loop." + Environment.NewLine + e.StackTrace);
}
}
}
