/// <summary>
/// Clear previous jobs
/// </summary>
private void ClearRecurringJobs()
{
try
{
using IStorageConnection connection = JobStorage.Current.GetConnection();
foreach (RecurringJobDto recurringJob in connection.GetRecurringJobs())
{
if (recurringJob != null)
{
RecurringJob.RemoveIfExists(recurringJob.Id);
}
}
IMonitoringApi monitoringApi = JobStorage.Current.GetMonitoringApi();
while (monitoringApi.Servers().Count > 1)
{
var serverToRemove = monitoringApi.Servers().Last();
connection.RemoveServer(serverToRemove.Name);
}
}
catch (Exception e)
{
Console.WriteLine(e);
}
}
private static int serverCountFromHangfire(IMonitoringApi monitor)
{
var runningServerCount = _retry.Execute(() => monitor.Servers().Count);
const int startingServer = 1;
return(runningServerCount + startingServer);
}
public static void RemoveUnusedHangfireServers(this IApplicationBuilder app)
{
IMonitoringApi monitoringApi = JobStorage.Current.GetMonitoringApi();
foreach (var item in monitoringApi.Servers())
{
var name = item.Name.Split(':')[0];
if (!_options.HangfireServerName.Any(x => x.ToLower().Equals(name.ToLower())))
{
JobStorage.Current.GetConnection().RemoveServer(item.Name);
}
}
}
public static void SetupHangfireServers(this IApplicationBuilder app)
{
IMonitoringApi monitoringApi = JobStorage.Current.GetMonitoringApi();
foreach (var item in _options.HangfireServerName)
{
if (!monitoringApi.Servers().Any(x => x.Name.ToLower().Contains(item.ToLower())))
{
var options = new BackgroundJobServerOptions
{
ServerName = $"{item}"
};
app.UseHangfireServer(options);
}
}
}
public bool BackgroundWorkers()
{
IMonitoringApi monitoringApi = JobStorage.Current.GetMonitoringApi();
return(monitoringApi.Servers().Any());
}
/// <summary>
/// Checks various HangFire settings, i.e., failing jobs and a positive number of servers
/// </summary>
/// <param name="context"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public Task <HealthCheckResult> CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default)
{
_logger.LogInformation("Health check HangFire");
var status = HealthStatus.Healthy;
var data = new Dictionary <string, object>();
// Check servers
var servers = _hangFireMonitoringApi.Servers();
if (servers.Count < 1)
{
var message = $"Number of servers is less than expected at time {DateTime.Now}";
_logger.LogWarning(message);
status = HealthStatus.Unhealthy;
data.Add(message, servers.Count);
}
// Check status of recurring jobs
var recurringJobs = _healthCheckHangFireService.GetRecurringJobs();
var noOfActuallyEnabledJobs = recurringJobs.Count(j => j.NextExecution != null);
var noOfExpectedEnabledJobs = _appSettingsConfig.HealthCheckSettings.NoOfEnabledJobs;
if (noOfActuallyEnabledJobs < noOfExpectedEnabledJobs)
{
var message = $"Number of enabled jobs is less than expected at time {DateTime.Now}. Expected:{noOfExpectedEnabledJobs} - Actual:{noOfActuallyEnabledJobs}";
_logger.LogWarning(message);
status = HealthStatus.Unhealthy;
data.Add(message, noOfActuallyEnabledJobs);
var enabledJobIds = _appSettingsConfig.HealthCheckSettings.EnabledJobIds;
foreach (var jobId in enabledJobIds)
{
var job = recurringJobs.First(j => j.Id == jobId);
if (job.NextExecution != null)
{
continue;
}
var jobMessage = $"Job '{jobId}' is not enabled as expected";
_logger.LogWarning(jobMessage);
status = HealthStatus.Unhealthy;
var info = job.LastExecution != null
? $"Last execution time: {job.LastExecution}"
: $"No last execution time for {jobId}";
data.Add(jobMessage, info);
}
}
// Check failing jobs
var failingJobsCount = _hangFireMonitoringApi.FailedCount();
if (failingJobsCount > 0)
{
status = HealthStatus.Unhealthy;
data.Add("failed jobs count", failingJobsCount);
_logger.LogWarning("HangFire has one or more failed jobs");
var jobs = _hangFireMonitoringApi.FailedJobs(0, 10);
foreach (var job in jobs)
{
var key = job.Key;
var val = job.Value;
_logger.LogWarning($"Failed job ID '{key}' - {val.ExceptionDetails} - {val.ExceptionMessage}");
try
{
var serializedVal = JsonSerializer.Serialize(val, val.GetType());
data.Add($"Job ID {key}", serializedVal);
}
catch (Exception e)
{
var errorMessage = $"Error in health check HangFire. {e.Message} - {e.StackTrace}";
_logger.LogError(errorMessage);
// Adding extra details from the failed job to data
var failedJobDetails = $"JOB FAILED: EXCEPTION DETAILS: {val.ExceptionDetails} - EXCEPTION MESSAGE: {val.ExceptionMessage} - EXCEPTION TYPE: {val.ExceptionType} - FAILED AT: {val.FailedAt}";
data.Add($"Job ID {key}. HangFire health check could not serialize class {val.GetType()}", failedJobDetails);
return(Task.FromResult(new HealthCheckResult(
status,
Description,
e,
data)));
}
}
}
if (data.Count == 0)
{
data.Add("State", "HangFire jobs and servers are in expected state");
}
return(Task.FromResult(new HealthCheckResult(
status,
Description,
data: data)));
}
