/// <summary>
/// 移除超时的服务器列表
/// </summary>
/// <param name="timeOut">超时时间</param>
public override int RemoveTimedOutServers(TimeSpan timeOut)
{
var serverNames = RedisClient.SMembers(_storage.GetRedisKey("servers"));
var heartbeats = new Dictionary <string, Tuple <DateTime, DateTime?> >();
var utcNow = DateTime.UtcNow;
foreach (var serverName in serverNames)
{
var srv = RedisClient.HMGet(_storage.GetRedisKey($"server:{serverName}"), new[] { "StartedAt", "Heartbeat" });
heartbeats.Add(serverName, new Tuple <DateTime, DateTime?>(JobHelper.DeserializeDateTime(srv[0]), JobHelper.DeserializeNullableDateTime(srv[1])));
}
var removedServerCount = 0;
foreach (var heartbeat in heartbeats)
{
var maxTime = new DateTime(Math.Max(heartbeat.Value.Item1.Ticks, (heartbeat.Value.Item2 ?? DateTime.MinValue).Ticks));
if (utcNow > maxTime.Add(timeOut))
{
RemoveServer(heartbeat.Key);
removedServerCount++;
}
}
return(removedServerCount);
}
public RecurringJobsPage()
{
RecurringJobs = new List <RecurringJobDto>();
using (var connection = JobStorage.Current.GetConnection())
{
var ids = connection.GetAllItemsFromSet("recurring-jobs");
foreach (var id in ids)
{
var hash = connection.GetAllEntriesFromHash(String.Format("recurring-job:{0}", id));
if (hash == null)
{
RecurringJobs.Add(new RecurringJobDto {
Id = id, Removed = true
});
continue;
}
var dto = new RecurringJobDto {
Id = id
};
dto.Cron = hash["Cron"];
try
{
var invocationData = JobHelper.FromJson <InvocationData>(hash["Job"]);
dto.Job = invocationData.Deserialize();
}
catch (JobLoadException ex)
{
dto.LoadException = ex;
}
if (hash.ContainsKey("NextExecution"))
{
dto.NextExecution = JobHelper.DeserializeDateTime(hash["NextExecution"]);
}
if (hash.ContainsKey("LastJobId"))
{
dto.LastJobId = hash["LastJobId"];
var stateData = connection.GetStateData(dto.LastJobId);
if (stateData != null)
{
dto.LastJobState = stateData.Name;
}
}
if (hash.ContainsKey("LastExecution"))
{
dto.LastExecution = JobHelper.DeserializeDateTime(hash["LastExecution"]);
}
RecurringJobs.Add(dto);
}
}
}
public void OnCreating(CreatingContext context)
{
if (context.Parameters.TryGetValue("RecurringJobId", out var recurringJobId) &&
context.InitialState?.Reason == "Triggered by recurring job scheduler")
{
// the job being created looks like a recurring job instance,
// and triggered by a scheduler (i.e. not manually) at that.
var recurringJob = context.Connection.GetAllEntriesFromHash($"recurring-job:{recurringJobId}");
if (recurringJob != null && recurringJob.TryGetValue("NextExecution", out var nextExecution))
{
// the next execution time of a recurring job is updated AFTER the job instance creation,
// so at the moment it still contains the scheduled execution time from the previous run.
var scheduledTime = JobHelper.DeserializeDateTime(nextExecution);
if (DateTime.UtcNow > scheduledTime + MaxDelay)
{
// the job is created way later than expected
context.Canceled = true;
}
}
}
}
private static DateTime GetLastInstant(IReadOnlyDictionary <string, string> recurringJob, IScheduleInstant instant, DateTime?startDateUTC, DateTime?endDateUTC)
{
DateTime lastInstant;
if (recurringJob.ContainsKey("LastExecution"))
{
lastInstant = DateTime.SpecifyKind(JobHelper.DeserializeDateTime(recurringJob["LastExecution"]), DateTimeKind.Utc);
}
else if (recurringJob.ContainsKey("CreatedAt"))
{
lastInstant = DateTime.SpecifyKind(JobHelper.DeserializeDateTime(recurringJob["CreatedAt"]), DateTimeKind.Utc);
}
else if (recurringJob.ContainsKey("NextExecution"))
{
lastInstant = DateTime.SpecifyKind(JobHelper.DeserializeDateTime(recurringJob["NextExecution"]), DateTimeKind.Utc);
lastInstant = lastInstant.AddSeconds(-1);
}
else
{
lastInstant = instant.NowInstant.AddSeconds(-1);
}
if (startDateUTC.HasValue && lastInstant < startDateUTC.Value)
{
lastInstant = startDateUTC.Value;
}
else if (endDateUTC.HasValue && lastInstant > endDateUTC.Value)
{
lastInstant = endDateUTC.Value;
}
return(lastInstant);
}
public override int RemoveTimedOutServers(TimeSpan timeOut)
{
var serverNames = Redis.SetMembers(RedisStorage.Prefix + "servers");
var heartbeats = new Dictionary <string, Tuple <DateTime, DateTime?> >();
var utcNow = DateTime.UtcNow;
foreach (var serverName in serverNames)
{
var name = serverName;
var srv = Redis.HashGet(string.Format(RedisStorage.Prefix + "server:{0}", name), new RedisValue[] { "StartedAt", "Heartbeat" });
heartbeats.Add(name,
new Tuple <DateTime, DateTime?>(
JobHelper.DeserializeDateTime(srv[0]),
JobHelper.DeserializeNullableDateTime(srv[1])));
}
var removedServerCount = 0;
foreach (var heartbeat in heartbeats)
{
var maxTime = new DateTime(
Math.Max(heartbeat.Value.Item1.Ticks, (heartbeat.Value.Item2 ?? DateTime.MinValue).Ticks));
if (utcNow > maxTime.Add(timeOut))
{
RemoveServer(Redis, heartbeat.Key);
removedServerCount++;
}
}
return(removedServerCount);
}
private static IHtmlString ScheduledRenderer(IDictionary <string, string> stateData)
{
return(new HtmlString(String.Format(
"<dl class=\"dl-horizontal\"><dt>Enqueue at:</dt><dd data-moment=\"{0}\">{1}</dd></dl>",
stateData["EnqueueAt"],
JobHelper.DeserializeDateTime(stateData["EnqueueAt"]))));
}
public async Task <HttpResponseMessage> Get([FromUri] string id)
{
return(await Task.Run(() =>
{
var state = Hangfire.States.AwaitingState.StateName;
var messages = new List <string>();
try
{
var job = JobStorage.Current.GetMonitoringApi().JobDetails(id);
if (job != null)
{
state = (job.History.Where(s => s.StateName == Hangfire.States.SucceededState.StateName).OrderByDescending(s => s.CreatedAt).FirstOrDefault() ??
job.History.Where(s => s.StateName == Hangfire.States.FailedState.StateName).OrderByDescending(s => s.CreatedAt).FirstOrDefault() ??
job.History.Where(s => s.StateName == Hangfire.States.ProcessingState.StateName).OrderByDescending(s => s.CreatedAt).FirstOrDefault())?.StateName ?? state;
}
{
var processingRecord = job.History.LastOrDefault(s => s.StateName == Hangfire.States.ProcessingState.StateName);
if (processingRecord != null)
{
var timestamp = JobHelper.DeserializeDateTime(processingRecord.Data["StartedAt"]);
messages = JobStorage.Current.GetConsoleApi().GetLines(id, timestamp, Hangfire.Console.Monitoring.LineType.Text).OfType <Hangfire.Console.Monitoring.TextLineDto>().Select(s => s.Text).ToList();
}
}
}
catch
{
}
return Request.CreateResponse(new { state = state, errors = messages });
}));
}
public Task <HttpResponseMessage> GetImportStatus([FromUri] string id)
{
var state = Hangfire.States.AwaitingState.StateName;
var messages = Enumerable.Empty <string>();
try
{
var job = JobStorage.Current.GetMonitoringApi().JobDetails(id);
if (job != null)
{
var history = job.History.OrderByDescending(s => s.CreatedAt);
state = (
history.FirstOrDefault(s => s.StateName == Hangfire.States.SucceededState.StateName) ??
history.FirstOrDefault(s => s.StateName == Hangfire.States.FailedState.StateName) ??
history.FirstOrDefault(s => s.StateName == Hangfire.States.ProcessingState.StateName)
)?.StateName ?? state;
var processingRecord = history.FirstOrDefault(s => s.StateName == Hangfire.States.ProcessingState.StateName);
if (processingRecord != null)
{
var timestamp = JobHelper.DeserializeDateTime(processingRecord.Data["StartedAt"]);
messages = JobStorage.Current.GetConsoleApi().GetLines(id, timestamp, Hangfire.Console.Monitoring.LineType.Text).OfType <Hangfire.Console.Monitoring.TextLineDto>().Select(s => s.Text);
}
}
}
catch { }
return(Task.FromResult(Request.CreateResponse(new { state, errors = messages })));
}
private static NonEscapedString ScheduledRenderer(HtmlHelper helper, IDictionary <string, string> stateData)
{
var enqueueAt = JobHelper.DeserializeDateTime(stateData["EnqueueAt"]);
return(new NonEscapedString(
$"<dl class=\"dl-horizontal\"><dt>Enqueue at:</dt><dd data-moment=\"{helper.HtmlEncode(JobHelper.ToTimestamp(enqueueAt).ToString(CultureInfo.InvariantCulture))}\">{helper.HtmlEncode(enqueueAt.ToString(CultureInfo.CurrentUICulture))}</dd></dl>"));
}
private static NonEscapedString ScheduledRenderer(HtmlHelper helper, IDictionary <string, string> stateData)
{
var enqueueAt = JobHelper.DeserializeDateTime(stateData["EnqueueAt"]);
return(new NonEscapedString(
$"<dl class=\"dl-horizontal\"><dt>Enqueue at:</dt><dd data-moment=\"{JobHelper.ToTimestamp(enqueueAt)}\">{enqueueAt}</dd></dl>"));
}
public NonEscapedString Render(HtmlHelper helper, IDictionary <string, string> stateData)
{
var builder = new StringBuilder();
builder.Append("<dl class=\"dl-horizontal\">");
string serverId = null;
if (stateData.ContainsKey("ServerId"))
{
serverId = stateData["ServerId"];
}
else if (stateData.ContainsKey("ServerName"))
{
serverId = stateData["ServerName"];
}
if (serverId != null)
{
builder.Append("<dt>Server:</dt>");
builder.Append($"<dd>{helper.ServerId(serverId)}</dd>");
}
if (stateData.ContainsKey("WorkerId"))
{
builder.Append("<dt>Worker:</dt>");
builder.Append($"<dd>{stateData["WorkerId"].Substring(0, 8)}</dd>");
}
else if (stateData.ContainsKey("WorkerNumber"))
{
builder.Append("<dt>Worker:</dt>");
builder.Append($"<dd>#{stateData["WorkerNumber"]}</dd>");
}
builder.Append("</dl>");
var page = helper.GetPage();
if (page.RequestPath.StartsWith("/jobs/details/"))
{
// We cannot cast page to an internal type JobDetailsPage to get jobId :(
var jobId = page.RequestPath.Substring("/jobs/details/".Length);
var startedAt = JobHelper.DeserializeDateTime(stateData["StartedAt"]);
var consoleId = new ConsoleId(jobId, startedAt);
builder.Append("<div class=\"console-area\">");
builder.AppendFormat("<div class=\"console\" data-id=\"{0}\">", consoleId);
using (var storage = new ConsoleStorage(page.Storage.GetConnection()))
{
ConsoleRenderer.RenderLineBuffer(builder, storage, consoleId, 0);
}
builder.Append("</div>");
builder.Append("</div>");
}
return(new NonEscapedString(builder.ToString()));
}
public override int RemoveTimedOutServers(TimeSpan timeOut)
{
var serverNames = Redis.SetMembers(Prefix + "servers");
var utcNow = DateTime.UtcNow;
var heartbeats = new List <KeyValuePair <string, RedisValue[]> >();
foreach (var serverName in serverNames)
{
var name = serverName;
heartbeats.Add(new KeyValuePair <string, RedisValue[]>(name, Redis.HashGet(String.Format(Prefix + "server:{0}", name), new RedisValue[] { "StartedAt", "Heartbeat" })));
}
var removedServerCount = 0;
foreach (var heartbeat in heartbeats)
{
var maxTime = new DateTime(
Math.Max(JobHelper.DeserializeDateTime(heartbeat.Value[0]).Ticks, (JobHelper.DeserializeNullableDateTime(heartbeat.Value[1]) ?? DateTime.MinValue).Ticks));
if (utcNow > maxTime.Add(timeOut))
{
RemoveServer(Redis, heartbeat.Key);
removedServerCount++;
}
}
return(removedServerCount);
}
public JobList <RetriesJobDto> RetriesJobs(string tagName, int from, int count)
{
return(UseConnection(redis =>
{
var retriesJobIds = redis
.SortedSetRangeByRankWithScores(GetRedisKey(RedisTagsKeyInfo.GetRetryKey(tagName)), from, from + count - 1, order: Order.Descending)
.Select(x => x.Element.ToString())
.ToArray();
return GetJobsWithProperties(
redis,
retriesJobIds,
new[] { "CreatedAt", "RetryCount" },
new[] { "State", "Reason", "EnqueueAt" },
(job, jobData, state) => new RetriesJobDto
{
Job = job,
Reason = state[1],
State = state[0],
EnqueueAt = JobHelper.DeserializeNullableDateTime(state[2]),
CreatedAt = JobHelper.DeserializeDateTime(jobData[0]),
RetryCount = Convert.ToInt32(jobData[1])
});
}));
}
/// <summary>
/// 获取服务器列表
/// </summary>
public IList <ServerDto> Servers() =>
UseConnection(redis =>
{
var serverNames = redis.SMembers(_storage.GetRedisKey("servers"));
if (serverNames.Length == 0)
{
return(new List <ServerDto>());
}
var servers = new List <ServerDto>();
foreach (var serverName in serverNames)
{
var queue = redis.LRange(_storage.GetRedisKey($"server:{serverName}:queues"), 0, -1);
var server = redis.HMGet(_storage.GetRedisKey($"server:{serverName}"), new string[] { "WorkerCount", "StartedAt", "Heartbeat" });
if (server[0] == null)
{
continue; // skip removed server
}
servers.Add(new ServerDto
{
Name = serverName,
WorkersCount = int.Parse(server[0]),
Queues = queue,
StartedAt = JobHelper.DeserializeDateTime(server[1]),
Heartbeat = JobHelper.DeserializeNullableDateTime(server[2])
});
}
return(servers);
});
/// <summary>
/// 是否检查时间超时
/// </summary>
/// <param name="fetchedTimestamp">拉取时间戳</param>
/// <param name="checkedTimestamp">检查时间戳</param>
private bool TimedOutByCheckedTime(string fetchedTimestamp, string checkedTimestamp)
{
if (!string.IsNullOrEmpty(fetchedTimestamp))
{
return(false);
}
return(!string.IsNullOrEmpty(checkedTimestamp) &&
DateTime.UtcNow - JobHelper.DeserializeDateTime(checkedTimestamp) > _options.CheckedTimeout);
}
private static NonEscapedString ScheduledRenderer(IDictionary <string, string> stateData)
{
var enqueueAt = JobHelper.DeserializeDateTime(stateData["EnqueueAt"]);
return(new NonEscapedString(String.Format(
"<dl class=\"dl-horizontal\"><dt>Enqueue at:</dt><dd data-moment=\"{0}\">{1}</dd></dl>",
JobHelper.ToTimestamp(enqueueAt),
enqueueAt)));
}
public JobList <ScheduledJobDto> ScheduledJobs(int from, int count)
{
return(GetJobsOnState(States.ScheduledState.StateName, from, count, (state, job) => new ScheduledJobDto
{
Job = job,
EnqueueAt = JobHelper.DeserializeDateTime(state.Data["EnqueueAt"]),
ScheduledAt = JobHelper.DeserializeDateTime(state.Data["ScheduledAt"])
}));
}
public JobList <ProcessingJobDto> ProcessingJobs(int from, int count)
{
return(GetJobsOnState(States.ProcessingState.StateName, from, count, (state, job) => new ProcessingJobDto
{
Job = job,
ServerId = state.Data.ContainsKey("ServerId") ? state.Data["ServerId"] : state.Data["ServerName"],
StartedAt = JobHelper.DeserializeDateTime(state.Data["StartedAt"])
}));
}
public void Heartbeat()
{
UseConnections((redis, connection) =>
{
connection.Heartbeat("1");
var pong = JobHelper.DeserializeDateTime(redis.HashGet(Prefix + "server:1", "Heartbeat"));
Assert.Equal(0, (int)(pong - DateTime.UtcNow).TotalSeconds);
});
}
public void SerializeData_ReturnsSerializedStateData()
{
var state = CreateState();
var data = state.SerializeData();
Assert.Equal(1, data.Count);
Assert.True(JobHelper.DeserializeDateTime(data["DeletedAt"]) != default(DateTime));
}
public JobList <ScheduledJobDto> ScheduledJobs(int from, int count)
=> GetJobs(from, count,
ScheduledState.StateName,
(sqlJob, job, stateData) => new ScheduledJobDto
{
Job = job,
EnqueueAt = JobHelper.DeserializeDateTime(stateData["EnqueueAt"]),
ScheduledAt = JobHelper.DeserializeDateTime(stateData["ScheduledAt"])
});
public JobList <ProcessingJobDto> ProcessingJobs(int from, int count)
=> GetJobs(from, count,
ProcessingState.StateName,
(sqlJob, job, stateData) => new ProcessingJobDto
{
Job = job,
ServerId = stateData.ContainsKey("ServerId") ? stateData["ServerId"] : stateData["ServerName"],
StartedAt = JobHelper.DeserializeDateTime(stateData["StartedAt"]),
});
private void TryScheduleJob(IStorageConnection connection, string recurringJobId, Dictionary <string, string> recurringJob)
{
var serializedJob = JobHelper.FromJson <InvocationData>(recurringJob["Job"]);
var job = serializedJob.Deserialize();
var cron = recurringJob["Cron"];
var parts = cron.Split(" ".ToCharArray(), StringSplitOptions.RemoveEmptyEntries);
var cronSchedule = CrontabSchedule.Parse(cron, new CrontabSchedule.ParseOptions {
IncludingSeconds = (parts.Length >= 6)
});
try
{
var timeZone = recurringJob.ContainsKey("TimeZoneId")
? TimeZoneInfo.FindSystemTimeZoneById(recurringJob["TimeZoneId"])
: TimeZoneInfo.Utc;
var instant = _instantFactory.GetInstant(cronSchedule, timeZone);
var lastExecutionTime = recurringJob.ContainsKey("LastExecution")
? JobHelper.DeserializeDateTime(recurringJob["LastExecution"])
: (DateTime?)null;
var changedFields = new Dictionary <string, string>();
if (instant.GetNextInstants(lastExecutionTime).Any())
{
var state = new EnqueuedState {
Reason = "Triggered by recurring job scheduler"
};
var jobId = _client.Create(job, state);
if (String.IsNullOrEmpty(jobId))
{
Logger.DebugFormat(
"Recurring job '{0}' execution at '{1}' has been canceled.",
recurringJobId,
instant.NowInstant);
}
changedFields.Add("LastExecution", JobHelper.SerializeDateTime(instant.NowInstant));
changedFields.Add("LastJobId", jobId ?? String.Empty);
}
changedFields.Add("NextExecution", JobHelper.SerializeDateTime(instant.NextInstant));
connection.SetRangeInHash(
String.Format("recurring-job:{0}", recurringJobId),
changedFields);
}
catch (TimeZoneNotFoundException ex)
{
Logger.ErrorException(
String.Format("Recurring job '{0}' was not triggered: {1}.", recurringJobId, ex.Message),
ex);
}
}
public JobDetailsDto JobDetails([NotNull] string jobId)
{
if (jobId == null)
{
throw new ArgumentNullException(nameof(jobId));
}
return(UseConnection(redis =>
{
var entries = redis.HashGetAll(_storage.GetRedisKey($"job:{jobId}"));
if (entries.Length == 0)
{
return null;
}
var jobData = entries.ToStringDictionary();
var history = redis.ListRange(_storage.GetRedisKey($"job:{jobId}:history"));
// history is in wrong order, fix this
Array.Reverse(history);
var stateHistory = new List <StateHistoryDto>(history.Length);
foreach (var row in history)
{
var entry = JobHelper.FromJson <Dictionary <string, string> >(row);
var stateData = new Dictionary <string, string>(entry, StringComparer.OrdinalIgnoreCase);
stateHistory.Add(new StateHistoryDto
{
StateName = stateData.Pull("State", true),
Reason = stateData.Pull("Reason"),
CreatedAt = JobHelper.DeserializeDateTime(stateData.Pull("CreatedAt", true)),
Data = stateData
});
}
// some properties are not pulled,
// but still need to be excluded
jobData.Remove("State");
jobData.Remove("Fetched");
return new JobDetailsDto
{
Job = TryToGetJob(
jobData.Pull("Type"),
jobData.Pull("Method"),
jobData.Pull("ParameterTypes"),
jobData.Pull("Arguments")),
CreatedAt = JobHelper.DeserializeNullableDateTime(jobData.Pull("CreatedAt")),
Properties = jobData,
History = stateHistory
};
}));
}
public JobList <ScheduledJobDto> ScheduledJobs(int from, int count)
{
return(UseConnection(connection => GetJobs(connection, from, count, ScheduledState.StateName,
(sqlJob, job, stateData) => new ScheduledJobDto
{
Job = job,
EnqueueAt = JobHelper.DeserializeDateTime(stateData["EnqueueAt"]),
ScheduledAt = JobHelper.DeserializeDateTime(stateData["ScheduledAt"])
})));
}
public List <DeletedJobDto> GetDeletedJobs(int from, int count)
{
return(GetJobs <DeletedJobDto>(DeletedState.DefaultName, from, count, (job, stateData) =>
{
return new DeletedJobDto()
{
JobInfo = job,
DeletedAt = JobHelper.DeserializeDateTime(stateData["DeletedAt"])
};
}));
}
public JobList <ScheduledJobDto> ScheduledJobs(int from, int count)
{
List <KeyValuePair <string, ScheduledJobDto> > result = GetJobsOnState(States.ScheduledState.StateName, from, count, (state, job) => new ScheduledJobDto
{
Job = job,
EnqueueAt = JobHelper.DeserializeDateTime(state.Data["EnqueueAt"]),
ScheduledAt = JobHelper.DeserializeDateTime(state.Data["ScheduledAt"])
}).OrderByDescending(j => j.Value.ScheduledAt).ToList();
return(new JobList <ScheduledJobDto>(result));
}
public JobList <ProcessingJobDto> ProcessingJobs(int from, int count)
{
List <KeyValuePair <string, ProcessingJobDto> > result = GetJobsOnState(States.ProcessingState.StateName, from, count, (state, job) => new ProcessingJobDto
{
Job = job,
ServerId = state.Data.ContainsKey("ServerId") ? state.Data["ServerId"] : state.Data["ServerName"],
StartedAt = JobHelper.DeserializeDateTime(state.Data["StartedAt"])
}).OrderByDescending(j => j.Value.StartedAt).ToList();
return(new JobList <ProcessingJobDto>(result));
}
public List <AwaitingJobDto> GetAwaitingJobs(int from, int count)
{
return(GetJobs <AwaitingJobDto>(AwaitingState.DefaultName, from, count, (job, stateData) =>
{
return new AwaitingJobDto()
{
JobInfo = job,
CreatedAt = JobHelper.DeserializeDateTime(stateData["CreatedAt"])
};
}));
}
public List <ScheduledJobDto> GetScheduledJobs(int from, int count)
{
return(GetJobs <ScheduledJobDto>(ScheduledState.DefaultName, from, count, (job, stateData) =>
{
return new ScheduledJobDto()
{
JobInfo = job,
EnqueueAt = JobHelper.DeserializeDateTime(stateData["EnqueueAt"]),
ScheduledAt = JobHelper.DeserializeDateTime(stateData["ScheduledAt"])
};
}));
}