public JobDetailsDto JobDetails(string jobId)
{
return(UseConnection(connection =>
{
const string jobQuery = @"
SELECT ID AS Id
,STATE_ID AS StateId
,STATE_NAME AS StateName
,INVOCATION_DATA AS InvocationData
,ARGUMENTS AS Arguments
,CREATED_AT AS CreatedAt
,EXPIRE_AT AS ExpireAt
FROM HF_JOB
WHERE ID = :ID
";
var sqlJob = connection.QuerySingleOrDefault <SqlJob>(jobQuery, new { ID = jobId });
if (sqlJob == null)
{
return null;
}
const string jobParametersQuery = @"
SELECT ID AS Id
,NAME AS Name
,VALUE AS Value
,JOB_ID AS JobId
FROM HF_JOB_PARAMETER
WHERE JOB_ID = :ID
";
var jobParameters = connection.Query <JobParameter>(jobParametersQuery, new { ID = jobId }).ToDictionary(x => x.Name, parameter => parameter.Value);
const string jobStatesQuery = @"
SELECT ID AS Id
,JOB_ID AS JobId
,NAME AS Name
,REASON AS Reason
,CREATED_AT AS CreatedAt
,DATA AS Data
FROM HF_JOB_STATE
WHERE JOB_ID = :ID
ORDER BY ID DESC
";
var jobStates = connection.Query <SqlState>(jobStatesQuery, new { ID = jobId }).Select(x => new StateHistoryDto
{
StateName = x.Name,
CreatedAt = x.CreatedAt,
Reason = x.Reason,
Data = new Dictionary <string, string>(JobHelper.FromJson <Dictionary <string, string> >(x.Data), StringComparer.OrdinalIgnoreCase),
}).ToList();
return new JobDetailsDto
{
CreatedAt = sqlJob.CreatedAt,
ExpireAt = sqlJob.ExpireAt,
Job = DeserializeJob(sqlJob.InvocationData, sqlJob.Arguments),
History = jobStates,
Properties = jobParameters
};
}));
}
private static List <RecurringJobDto> GetRecurringJobDtos(IStorageConnection connection, IEnumerable <string> ids)
{
var result = new List <RecurringJobDto>();
foreach (var id in ids)
{
var hash = connection.GetAllEntriesFromHash($"recurring-job:{id}");
if (hash == null)
{
result.Add(new RecurringJobDto {
Id = id, Removed = true
});
continue;
}
var dto = new RecurringJobDto
{
Id = id,
Cron = hash["Cron"],
Name = hash.ContainsKey("Name") ? hash["Name"] : String.Empty
};
try
{
var invocationData = JobHelper.FromJson <InvocationData>(hash["Job"]);
dto.Job = invocationData.Deserialize();
}
catch (JobLoadException ex)
{
dto.LoadException = ex;
}
if (hash.ContainsKey("NextExecution"))
{
dto.NextExecution = JobHelper.DeserializeDateTime(hash["NextExecution"]);
}
if (hash.ContainsKey("LastJobId") && !string.IsNullOrWhiteSpace(hash["LastJobId"]))
{
dto.LastJobId = hash["LastJobId"];
var stateData = connection.GetStateData(dto.LastJobId);
if (stateData != null)
{
dto.LastJobState = stateData.Name;
}
}
if (hash.ContainsKey("Queue"))
{
dto.Queue = hash["Queue"];
}
if (hash.ContainsKey("LastExecution"))
{
dto.LastExecution = JobHelper.DeserializeDateTime(hash["LastExecution"]);
}
if (hash.ContainsKey("TimeZoneId"))
{
dto.TimeZoneId = hash["TimeZoneId"];
}
if (hash.ContainsKey("CreatedAt"))
{
dto.CreatedAt = JobHelper.DeserializeDateTime(hash["CreatedAt"]);
}
result.Add(dto);
}
return(result);
}
private bool TimedOutByFetchedTime(string fetchedTimestamp)
{
return(!string.IsNullOrEmpty(fetchedTimestamp) &&
(DateTime.UtcNow - JobHelper.DeserializeDateTime(fetchedTimestamp) > _invisibilityTimeout));
}
public override void Heartbeat(string serverId)
{
Redis.HashSet(String.Format(Prefix + "server:{0}", serverId), "Heartbeat", JobHelper.SerializeDateTime(DateTime.UtcNow));
}
private static string BuildJob(string id, Hangfire.Common.Job job)
{
if (job == null)
{
return("<em>Can not find the target method.</em>");
}
var builder = new StringBuilder();
builder.Append("<span class=\"comment\">// #Job ID: ");
builder.Append(id);
builder.Append("</span>");
builder.AppendLine();
builder.AppendLine();
builder.Append(WrapKeyword("using"));
builder.Append(" ");
builder.Append(Encode(job.Type.Namespace));
builder.Append(";");
builder.AppendLine();
builder.AppendLine();
string serviceName = null;
if (!job.Method.IsStatic)
{
serviceName = GetNameWithoutGenericArity(job.Type);
if (job.Type.GetTypeInfo().IsInterface&& serviceName[0] == 'I' && Char.IsUpper(serviceName[1]))
{
serviceName = serviceName.Substring(1);
}
serviceName = Char.ToLower(serviceName[0]) + serviceName.Substring(1);
builder.Append(WrapKeyword("var"));
builder.Append(
$" {Encode(serviceName)} = Activate<{WrapType(Encode(job.Type.ToString()))}>();");
builder.AppendLine();
}
if (job.Method.GetCustomAttribute <AsyncStateMachineAttribute>() != null)
{
builder.Append($"{WrapKeyword("await")} ");
}
builder.Append(!job.Method.IsStatic ? Encode(serviceName) : WrapType(Encode(job.Type.ToString())));
builder.Append(".");
builder.Append(Encode(job.Method.Name));
if (job.Method.IsGenericMethod)
{
var genericArgumentTypes = job.Method.GetGenericArguments()
.Select(x => WrapType(x.Name))
.ToArray();
builder.Append($"<{String.Join(", ", genericArgumentTypes)}>");
}
builder.Append("(");
var parameters = job.Method.GetParameters();
var renderedArguments = new List <string>(parameters.Length);
var renderedArgumentsTotalLength = 0;
const int splitStringMinLength = 100;
for (var i = 0; i < parameters.Length; i++)
{
var parameter = parameters[i];
#pragma warning disable 618
if (i < job.Arguments.Length)
{
var argument = job.Arguments[i]; // TODO: check bounds
#pragma warning restore 618
string renderedArgument;
var enumerableArgument = GetIEnumerableGenericArgument(parameter.ParameterType);
object argumentValue;
bool isJson = true;
try
{
argumentValue = JobHelper.FromJson(argument, parameter.ParameterType);
}
catch (Exception)
{
// If argument value is not encoded as JSON (an old
// way using TypeConverter), we should display it as is.
argumentValue = argument;
isJson = false;
}
if (enumerableArgument == null || argumentValue == null)
{
var argumentRenderer = ArgumentRenderer.GetRenderer(parameter.ParameterType);
renderedArgument = argumentRenderer.Render(isJson, argumentValue?.ToString(), argument);
}
else
{
var renderedItems = new List <string>();
// ReSharper disable once LoopCanBeConvertedToQuery
foreach (var item in (IEnumerable)argumentValue)
{
var argumentRenderer = ArgumentRenderer.GetRenderer(enumerableArgument);
renderedItems.Add(argumentRenderer.Render(isJson, item?.ToString(),
JobHelper.ToJson(item)));
}
// ReSharper disable once UseStringInterpolation
renderedArgument = String.Format(
"{0}{1} {{ {2} }}",
WrapKeyword("new"),
parameter.ParameterType.IsArray ? " []" : "",
String.Join(", ", renderedItems));
}
renderedArguments.Add(renderedArgument);
renderedArgumentsTotalLength += renderedArgument.Length;
}
else
{
renderedArguments.Add(Encode("<NO VALUE>"));
}
}
for (int i = 0; i < renderedArguments.Count; i++)
{
// TODO: be aware of out of range
var parameter = parameters[i];
var tooltipPosition = "top";
var renderedArgument = renderedArguments[i];
if (renderedArgumentsTotalLength > splitStringMinLength)
{
builder.AppendLine();
builder.Append(" ");
tooltipPosition = "left";
}
else if (i > 0)
{
builder.Append(" ");
}
builder.Append($"<span title=\"{parameter.Name}\" data-placement=\"{tooltipPosition}\">");
builder.Append(renderedArgument);
builder.Append("</span>");
if (i < renderedArguments.Count - 1)
{
builder.Append(",");
}
}
builder.Append(");");
return(builder.ToString());
}
private static void SetContinuations(
IStorageConnection connection, string jobId, List <Continuation> continuations)
{
connection.SetJobParameter(jobId, "Continuations", JobHelper.ToJson(continuations));
}
public NonEscapedString MomentTitle(DateTime time, string value)
{
return(Raw($"<span data-moment-title=\"{HtmlEncode(JobHelper.ToTimestamp(time).ToString(CultureInfo.InvariantCulture))}\">{HtmlEncode(value)}</span>"));
}
public IReadOnlyDictionary <string, string> GetChangedFields(out DateTime?nextExecution)
{
var result = new Dictionary <string, string>();
if ((_recurringJob.ContainsKey("Queue") ? _recurringJob["Queue"] : null) != Queue)
{
result.Add("Queue", Queue);
}
if ((_recurringJob.ContainsKey("Cron") ? _recurringJob["Cron"] : null) != Cron)
{
result.Add("Cron", Cron);
}
if ((_recurringJob.ContainsKey("TimeZoneId") ? _recurringJob["TimeZoneId"] : null) != TimeZone.Id)
{
result.Add("TimeZoneId", TimeZone.Id);
}
var serializedJob = InvocationData.SerializeJob(Job).SerializePayload();
if ((_recurringJob.ContainsKey("Job") ? _recurringJob["Job"] : null) != serializedJob)
{
result.Add("Job", serializedJob);
}
var serializedCreatedAt = JobHelper.SerializeDateTime(CreatedAt);
if ((_recurringJob.ContainsKey("CreatedAt") ? _recurringJob["CreatedAt"] : null) != serializedCreatedAt)
{
result.Add("CreatedAt", serializedCreatedAt);
}
var serializedLastExecution = LastExecution.HasValue ? JobHelper.SerializeDateTime(LastExecution.Value) : null;
if ((_recurringJob.ContainsKey("LastExecution") ? _recurringJob["LastExecution"] : null) !=
serializedLastExecution)
{
result.Add("LastExecution", serializedLastExecution ?? String.Empty);
}
TryGetNextExecution(result.ContainsKey("Cron"), out nextExecution, out _);
var serializedNextExecution = nextExecution.HasValue ? JobHelper.SerializeDateTime(nextExecution.Value) : null;
if ((_recurringJob.ContainsKey("NextExecution") ? _recurringJob["NextExecution"] : null) !=
serializedNextExecution)
{
result.Add("NextExecution", serializedNextExecution ?? String.Empty);
}
if ((_recurringJob.ContainsKey("LastJobId") ? _recurringJob["LastJobId"] : null) != LastJobId)
{
result.Add("LastJobId", LastJobId ?? String.Empty);
}
if (!_recurringJob.ContainsKey("V"))
{
result.Add("V", "2");
}
if (_recurringJob.ContainsKey("Error") && !String.IsNullOrEmpty(_recurringJob["Error"]))
{
result.Add("Error", String.Empty);
}
if (_recurringJob.ContainsKey("RetryAttempt") && _recurringJob["RetryAttempt"] != "0")
{
result.Add("RetryAttempt", "0");
}
return(result);
}
public JobDetailsDto JobDetails(string jobId)
{
return(UseConnection(connection =>
{
string sql = string.Format(@"
SELECT id ""Id"", invocationdata ""InvocationData"", arguments ""Arguments"", createdat ""CreatedAt"", expireat ""ExpireAt""
FROM ""{0}.JOB""
WHERE id = @id;", _options.Prefix);
var job = connection.Query <SqlJob>(sql, new { id = jobId }).SingleOrDefault();
if (job == null)
{
return null;
}
sql = string.Format(@"
SELECT jobid ""JobId"", name ""Name"", ""VALUE"" ""Value""
FROM ""{0}.JOBPARAMETER""
WHERE jobid = @id;", _options.Prefix);
var parameters = connection.Query <JobParameter>(sql, new { id = jobId }).ToDictionary(x => x.Name, x => x.Value);
sql = string.Format(@"
SELECT jobid ""JobId"", name ""Name"", reason ""Reason"", createdat ""CreatedAt"", data ""Data""
FROM ""{0}.STATE""
WHERE jobid = @id
ORDER BY id DESC;", _options.Prefix);
var history = connection.Query <SqlState>(sql, new { id = jobId }).ToList()
.Select(x => new StateHistoryDto
{
StateName = x.Name,
CreatedAt = x.CreatedAt,
Reason = x.Reason,
Data = new Dictionary <string, string>(
JobHelper.FromJson <Dictionary <string, string> >(x.Data),
StringComparer.OrdinalIgnoreCase)
})
.ToList();
return new JobDetailsDto
{
CreatedAt = job.CreatedAt,
Job = DeserializeJob(job.InvocationData, job.Arguments),
History = history,
Properties = parameters
};
/*using (var multi = connection.QueryMultiple(sql, new { id = jobId }))
* {
* var job = multi.Read<SqlJob>().SingleOrDefault();
* if (job == null) return null;
*
* var parameters = multi.Read<JobParameter>().ToDictionary(x => x.Name, x => x.Value);
* var history =
* multi.Read<SqlState>()
* .ToList()
* .Select(x => new StateHistoryDto
* {
* StateName = x.Name,
* CreatedAt = x.CreatedAt,
* Reason = x.Reason,
* Data = new Dictionary<string, string>(
* JobHelper.FromJson<Dictionary<string, string>>(x.Data),
* StringComparer.OrdinalIgnoreCase)
* })
* .ToList();
*
* return new JobDetailsDto
* {
* CreatedAt = job.CreatedAt,
* Job = DeserializeJob(job.InvocationData, job.Arguments),
* History = history,
* Properties = parameters
* };
* }*/
}));
}
private JobDto CreateJobInState(HangfireDbContext database, ObjectId jobId, string stateName, Func <JobDto, JobDto> visitor = null)
{
var job = Job.FromExpression(() => SampleMethod("wrong"));
Dictionary <string, string> stateData;
if (stateName == EnqueuedState.StateName)
{
stateData = new Dictionary <string, string> {
["EnqueuedAt"] = $"{DateTime.UtcNow:o}"
};
}
else if (stateName == ProcessingState.StateName)
{
stateData = new Dictionary <string, string>
{
["ServerId"] = Guid.NewGuid().ToString(),
["StartedAt"] = JobHelper.SerializeDateTime(DateTime.UtcNow.Subtract(TimeSpan.FromMilliseconds(500)))
};
}
else if (stateName == FailedState.StateName)
{
stateData = new Dictionary <string, string>
{
["ExceptionDetails"] = "Test_ExceptionDetails",
["ExceptionMessage"] = "Test_ExceptionMessage",
["ExceptionType"] = "Test_ExceptionType",
["FailedAt"] = JobHelper.SerializeDateTime(DateTime.UtcNow.Subtract(TimeSpan.FromMilliseconds(10)))
};
}
else
{
stateData = new Dictionary <string, string>();
}
var jobState = new StateDto()
{
Name = stateName,
Reason = null,
CreatedAt = DateTime.UtcNow,
Data = stateData
};
var jobDto = new JobDto
{
Id = jobId,
InvocationData = JobHelper.ToJson(InvocationData.Serialize(job)),
Arguments = "[\"\\\"Arguments\\\"\"]",
StateName = stateName,
CreatedAt = DateTime.UtcNow,
StateHistory = new[] { jobState }
};
if (visitor != null)
{
jobDto = visitor(jobDto);
}
database.Job.InsertOne(jobDto);
var jobQueueDto = new JobQueueDto
{
FetchedAt = null,
JobId = jobId,
Queue = DefaultQueue
};
if (stateName == FetchedStateName)
{
jobQueueDto.FetchedAt = DateTime.UtcNow;
}
database.JobQueue.InsertOne(jobQueueDto);
return(jobDto);
}
public override void Execute()
{
WriteLiteral("\n");
#line 10 "..\..\Pages\EnqueuedJobsPage.cshtml"
Layout = new LayoutPage
{
Title = Queue.ToUpperInvariant(),
Subtitle = "Enqueued jobs",
Breadcrumbs = new Dictionary <string, string>
{
{ "Queues", Request.LinkTo("/queues") }
}
};
int from, perPage;
int.TryParse(Request.QueryString["from"], out from);
int.TryParse(Request.QueryString["count"], out perPage);
var monitor = JobStorage.Current.GetMonitoringApi();
Pager pager = new Pager(from, perPage, monitor.EnqueuedCount(Queue))
{
BasePageUrl = Request.LinkTo("/queues/" + Queue)
};
JobList <EnqueuedJobDto> enqueuedJobs = monitor
.EnqueuedJobs(Queue, pager.FromRecord, pager.RecordsPerPage);
#line default
#line hidden
WriteLiteral("\n");
#line 36 "..\..\Pages\EnqueuedJobsPage.cshtml"
if (pager.TotalPageCount == 0)
{
#line default
#line hidden
WriteLiteral(" <div class=\"alert alert-info\">\n The queue is empty.\n </div>\n");
#line 41 "..\..\Pages\EnqueuedJobsPage.cshtml"
}
else
{
#line default
#line hidden
WriteLiteral(" <div class=\"js-jobs-list\">\n <div class=\"btn-toolbar btn-toolbar-top\">\n" +
" <button class=\"js-jobs-list-command btn btn-sm btn-default\"\n " +
" data-url=\"");
#line 47 "..\..\Pages\EnqueuedJobsPage.cshtml"
Write(Request.LinkTo("/enqueued/delete"));
#line default
#line hidden
WriteLiteral(@"""
data-loading-text=""Deleting...""
data-confirm=""Do you really want to DELETE ALL selected jobs?"">
<span class=""glyphicon glyphicon-remove""></span>
Delete selected
</button>
");
#line 54 "..\..\Pages\EnqueuedJobsPage.cshtml"
Write(RenderPartial(new PerPageSelector(pager)));
#line default
#line hidden
WriteLiteral(@"
</div>
<table class=""table"">
<thead>
<tr>
<th class=""min-width"">
<input type=""checkbox"" class=""js-jobs-list-select-all"" />
</th>
<th class=""min-width"">Id</th>
<th class=""min-width"">State</th>
<th>Job</th>
<th class=""align-right"">Enqueued</th>
</tr>
</thead>
<tbody>
");
#line 70 "..\..\Pages\EnqueuedJobsPage.cshtml"
foreach (var job in enqueuedJobs)
{
#line default
#line hidden
WriteLiteral(" <tr class=\"js-jobs-list-row hover ");
#line 72 "..\..\Pages\EnqueuedJobsPage.cshtml"
Write(!job.Value.InEnqueuedState ? "obsolete-data" : null);
#line default
#line hidden
WriteLiteral("\">\n <td>\n <input type=\"checkbox" +
"\" class=\"js-jobs-list-checkbox\" name=\"jobs[]\" value=\"");
#line 74 "..\..\Pages\EnqueuedJobsPage.cshtml"
Write(job.Key);
#line default
#line hidden
WriteLiteral("\" />\n </td>\n <td class=\"min-width\">" +
"\n <a href=\"");
#line 77 "..\..\Pages\EnqueuedJobsPage.cshtml"
Write(Request.LinkTo("/job/" + job.Key));
#line default
#line hidden
WriteLiteral("\">\n ");
#line 78 "..\..\Pages\EnqueuedJobsPage.cshtml"
Write(HtmlHelper.JobId(job.Key));
#line default
#line hidden
WriteLiteral("\n </a>\n");
#line 80 "..\..\Pages\EnqueuedJobsPage.cshtml"
if (!job.Value.InEnqueuedState)
{
#line default
#line hidden
WriteLiteral(" <span title=\"Job\'s state has been changed while f" +
"etching data.\" class=\"glyphicon glyphicon-question-sign\"></span>\n");
#line 83 "..\..\Pages\EnqueuedJobsPage.cshtml"
}
#line default
#line hidden
WriteLiteral(" </td>\n <td class=\"min-width\">\n " +
" <span class=\"label label-default\" style=\"");
#line 86 "..\..\Pages\EnqueuedJobsPage.cshtml"
Write(JobHistoryRenderer.ForegroundStateColors.ContainsKey(job.Value.State) ? String.Format("background-color: {0};", JobHistoryRenderer.ForegroundStateColors[job.Value.State]) : null);
#line default
#line hidden
WriteLiteral("\">\n ");
#line 87 "..\..\Pages\EnqueuedJobsPage.cshtml"
Write(job.Value.State);
#line default
#line hidden
WriteLiteral("\n </span>\n </td>\n " +
" <td>\n <a class=\"job-method\" href=\"");
#line 91 "..\..\Pages\EnqueuedJobsPage.cshtml"
Write(Request.LinkTo("/job/" + job.Key));
#line default
#line hidden
WriteLiteral("\">\n ");
#line 92 "..\..\Pages\EnqueuedJobsPage.cshtml"
Write(HtmlHelper.DisplayMethod(job.Value.Job));
#line default
#line hidden
WriteLiteral("\n </a>\n </td>\n " +
" <td class=\"align-right\">\n");
#line 96 "..\..\Pages\EnqueuedJobsPage.cshtml"
if (job.Value.EnqueuedAt.HasValue)
{
#line default
#line hidden
WriteLiteral(" <span data-moment=\"");
#line 98 "..\..\Pages\EnqueuedJobsPage.cshtml"
Write(JobHelper.ToStringTimestamp(job.Value.EnqueuedAt.Value));
#line default
#line hidden
WriteLiteral("\">\n ");
#line 99 "..\..\Pages\EnqueuedJobsPage.cshtml"
Write(job.Value.EnqueuedAt);
#line default
#line hidden
WriteLiteral("\n </span>\n");
#line 101 "..\..\Pages\EnqueuedJobsPage.cshtml"
}
else
{
#line default
#line hidden
WriteLiteral(" <em>n/a</em>\n");
#line 105 "..\..\Pages\EnqueuedJobsPage.cshtml"
}
#line default
#line hidden
WriteLiteral(" </td>\n </tr>\n");
#line 108 "..\..\Pages\EnqueuedJobsPage.cshtml"
}
#line default
#line hidden
WriteLiteral(" </tbody>\n </table>\n </div>\n");
#line 112 "..\..\Pages\EnqueuedJobsPage.cshtml"
#line default
#line hidden
#line 112 "..\..\Pages\EnqueuedJobsPage.cshtml"
Write(RenderPartial(new Paginator(pager)));
#line default
#line hidden
#line 112 "..\..\Pages\EnqueuedJobsPage.cshtml"
}
#line default
#line hidden
}
public JobDetailsDto JobDetails(string jobId)
{
return(UseConnection(redis =>
{
var job = redis.GetAllEntriesFromHash(String.Format("hangfire:job:{0}", jobId));
if (job.Count == 0)
{
return null;
}
var hiddenProperties = new[] { "Type", "Method", "ParameterTypes", "Arguments", "State", "CreatedAt" };
var historyList = redis.GetAllItemsFromList(
String.Format("hangfire:job:{0}:history", jobId));
var history = historyList
.Select(JobHelper.FromJson <Dictionary <string, string> >)
.ToList();
var stateHistory = new List <StateHistoryDto>(history.Count);
foreach (var entry in history)
{
var dto = new StateHistoryDto
{
StateName = entry["State"],
Reason = entry.ContainsKey("Reason") ? entry["Reason"] : null,
CreatedAt = JobHelper.FromStringTimestamp(entry["CreatedAt"]),
};
// Each history item contains all of the information,
// but other code should not know this. We'll remove
// unwanted keys.
var stateData = new Dictionary <string, string>(entry);
stateData.Remove("State");
stateData.Remove("Reason");
stateData.Remove("CreatedAt");
dto.Data = stateData;
stateHistory.Add(dto);
}
// For compatibility
if (!job.ContainsKey("Method"))
{
job.Add("Method", null);
}
if (!job.ContainsKey("ParameterTypes"))
{
job.Add("ParameterTypes", null);
}
return new JobDetailsDto
{
Job = TryToGetJob(job["Type"], job["Method"], job["ParameterTypes"], job["Arguments"]),
CreatedAt =
job.ContainsKey("CreatedAt")
? JobHelper.FromStringTimestamp(job["CreatedAt"])
: (DateTime?)null,
Properties =
job.Where(x => !hiddenProperties.Contains(x.Key)).ToDictionary(x => x.Key, x => x.Value),
History = stateHistory
};
}));
}
public override int RemoveTimedOutServers(TimeSpan timeOut)
{
var serverNames = Redis.SetMembers(Prefix + "servers");
var utcNow = DateTime.UtcNow;
var heartbeats = new List <KeyValuePair <string, RedisValue[]> >();
foreach (var serverName in serverNames)
{
var name = serverName;
heartbeats.Add(new KeyValuePair <string, RedisValue[]>(name, Redis.HashGet(String.Format(Prefix + "server:{0}", name), new RedisValue[] { "StartedAt", "Heartbeat" })));
}
var removedServerCount = 0;
foreach (var heartbeat in heartbeats)
{
var maxTime = new DateTime(
Math.Max(JobHelper.DeserializeDateTime(heartbeat.Value[0]).Ticks, (JobHelper.DeserializeNullableDateTime(heartbeat.Value[1]) ?? DateTime.MinValue).Ticks));
if (utcNow > maxTime.Add(timeOut))
{
RemoveServer(Redis, heartbeat.Key);
removedServerCount++;
}
}
return(removedServerCount);
}
private static List <OnEventJobDto> GetOnEventJobDtos(IStorageConnection connection, IEnumerable <string> ids)
{
var result = new List <OnEventJobDto>();
foreach (var id in ids)
{
var hash = connection.GetAllEntriesFromHash($"on-event-job:{id}");
if (hash == null)
{
result.Add(new OnEventJobDto {
Id = id, Removed = true
});
continue;
}
var dto = new OnEventJobDto
{
Id = id,
TriggerSignal = hash["SignalId"],
Name = hash.ContainsKey("Name") ? hash["Name"] : String.Empty
};
try
{
Expression <Func <Double, Boolean> > expr = JsonConvert.DeserializeObject <Expression <Func <Double, Boolean> > >(hash["TriggerExpr"], OnEventJobManager.jsonSettings.Value);
dto.TriggerExpression = expr.ToString();
}
catch (Exception)
{
}
try
{
var invocationData = JobHelper.FromJson <InvocationData>(hash["Job"]);
dto.Job = invocationData.Deserialize();
}
catch (JobLoadException ex)
{
dto.LoadException = ex;
}
if (hash.ContainsKey("LastJobId") && !string.IsNullOrWhiteSpace(hash["LastJobId"]))
{
dto.LastJobId = hash["LastJobId"];
var stateData = connection.GetStateData(dto.LastJobId);
if (stateData != null)
{
dto.LastJobState = stateData.Name;
}
}
if (hash.ContainsKey("IsActive") &&
!String.IsNullOrEmpty(hash["IsActive"]))
{
Boolean.TryParse(hash["IsActive"], out bool r);
dto.IsActive = r;
}
if (hash.ContainsKey("Queue"))
{
dto.Queue = hash["Queue"];
}
if (hash.ContainsKey("LastExecution"))
{
dto.LastExecution = JobHelper.DeserializeDateTime(hash["LastExecution"]);
}
if (hash.ContainsKey("TimeZoneId"))
{
dto.TimeZoneId = hash["TimeZoneId"];
}
if (hash.ContainsKey("CreatedAt"))
{
dto.CreatedAt = JobHelper.DeserializeDateTime(hash["CreatedAt"]);
}
result.Add(dto);
}
return(result);
}
public NonEscapedString LocalTime(DateTime value)
{
return(Raw($"<span data-moment-local=\"{HtmlEncode(JobHelper.ToTimestamp(value).ToString(CultureInfo.InvariantCulture))}\">{HtmlEncode(value.ToString(CultureInfo.CurrentUICulture))}</span>"));
}
/// <summary>
/// Fetches console lines from storage.
/// </summary>
/// <param name="storage">Console data accessor</param>
/// <param name="consoleId">Console identifier</param>
/// <param name="start">Offset to read lines from</param>
/// <remarks>
/// On completion, <paramref name="start"/> is set to the end of the current batch,
/// and can be used for next requests (or set to -1, if the job has finished processing).
/// </remarks>
private static IEnumerable <ConsoleLine> ReadLines(IConsoleStorage storage, ConsoleId consoleId, ref int start)
{
if (start < 0)
{
return(null);
}
var count = storage.GetLineCount(consoleId);
var result = new List <ConsoleLine>(Math.Max(1, count - start));
if (count > start)
{
// has some new items to fetch
Dictionary <string, ConsoleLine> progressBars = null;
foreach (var entry in storage.GetLines(consoleId, start, count - 1))
{
if (entry.ProgressValue.HasValue)
{
// aggregate progress value updates into single record
if (progressBars != null)
{
ConsoleLine prev;
if (progressBars.TryGetValue(entry.Message, out prev))
{
prev.ProgressValue = entry.ProgressValue;
prev.TextColor = entry.TextColor;
continue;
}
}
else
{
progressBars = new Dictionary <string, ConsoleLine>();
}
progressBars.Add(entry.Message, entry);
}
result.Add(entry);
}
}
if (count <= start || start == 0)
{
// no new items or initial load, check if the job is still performing
var state = storage.GetState(consoleId);
if (state == null)
{
// No state found for a job, probably it was deleted
count = -2;
}
else
{
if (!string.Equals(state.Name, ProcessingState.StateName, StringComparison.OrdinalIgnoreCase) ||
!consoleId.Equals(new ConsoleId(consoleId.JobId, JobHelper.DeserializeDateTime(state.Data["StartedAt"]))))
{
// Job state has changed (either not Processing, or another Processing with different console id)
count = -1;
}
}
}
start = count;
return(result);
}
public void AddOrUpdate_DoesNotReScheduleJob_WhenUpdatingIt()
{
// Arrange
_connection.Setup(x => x.GetAllEntriesFromHash($"recurring-job:{_id}")).Returns(new Dictionary <string, string>
{
{ "Cron", "* * * * *" },
{ "Job", InvocationData.Serialize(_job).SerializePayload() },
{ "CreatedAt", JobHelper.SerializeDateTime(_now.AddMinutes(-3)) },
{ "LastExecution", JobHelper.SerializeDateTime(_now.AddMinutes(-2)) },
{ "NextExecution", JobHelper.SerializeDateTime(_now.AddMinutes(-1)) }
});
var manager = CreateManager();
// Act
manager.AddOrUpdate(_id, _job, "* * * * *");
// Assert
_transaction.Verify(x => x.SetRangeInHash($"recurring-job:{_id}", It.Is <Dictionary <string, string> >(dict =>
!dict.ContainsKey("NextExecution") || dict["NextExecution"] == JobHelper.SerializeDateTime(_now.AddMinutes(-1)))));
_transaction.Verify(x => x.AddToSet("recurring-jobs", _id, JobHelper.ToTimestamp(_now.AddMinutes(-1))));
_transaction.Verify(x => x.Commit());
}
public RecurringJobEntity(
[NotNull] string recurringJobId,
[NotNull] IDictionary <string, string> recurringJob,
[NotNull] ITimeZoneResolver timeZoneResolver,
DateTime now)
{
if (timeZoneResolver == null)
{
throw new ArgumentNullException(nameof(timeZoneResolver));
}
_recurringJob = recurringJob ?? throw new ArgumentNullException(nameof(recurringJob));
_now = now;
RecurringJobId = recurringJobId ?? throw new ArgumentNullException(nameof(recurringJobId));
if (recurringJob.ContainsKey("Queue") && !String.IsNullOrWhiteSpace(recurringJob["Queue"]))
{
Queue = recurringJob["Queue"];
}
try
{
TimeZone = recurringJob.ContainsKey("TimeZoneId") && !String.IsNullOrWhiteSpace(recurringJob["TimeZoneId"])
? timeZoneResolver.GetTimeZoneById(recurringJob["TimeZoneId"])
: TimeZoneInfo.Utc;
}
catch (Exception ex)
{
_errors.Add(ex);
}
if (recurringJob.ContainsKey("Cron") && !String.IsNullOrWhiteSpace(recurringJob["Cron"]))
{
Cron = recurringJob["Cron"];
}
try
{
if (!recurringJob.ContainsKey("Job") || String.IsNullOrWhiteSpace(recurringJob["Job"]))
{
throw new InvalidOperationException("The 'Job' field has a null or empty value");
}
Job = InvocationData.DeserializePayload(recurringJob["Job"]).DeserializeJob();
}
catch (Exception ex)
{
_errors.Add(ex);
}
if (recurringJob.ContainsKey("LastJobId") && !String.IsNullOrWhiteSpace(recurringJob["LastJobId"]))
{
LastJobId = recurringJob["LastJobId"];
}
if (recurringJob.ContainsKey("LastExecution") && !String.IsNullOrWhiteSpace(recurringJob["LastExecution"]))
{
LastExecution = JobHelper.DeserializeDateTime(recurringJob["LastExecution"]);
}
if (recurringJob.ContainsKey("NextExecution") && !String.IsNullOrWhiteSpace(recurringJob["NextExecution"]))
{
NextExecution = JobHelper.DeserializeDateTime(recurringJob["NextExecution"]);
}
if (recurringJob.ContainsKey("CreatedAt") && !String.IsNullOrWhiteSpace(recurringJob["CreatedAt"]))
{
CreatedAt = JobHelper.DeserializeDateTime(recurringJob["CreatedAt"]);
}
else
{
CreatedAt = now;
}
if (recurringJob.ContainsKey("V") && !String.IsNullOrWhiteSpace(recurringJob["V"]))
{
Version = int.Parse(recurringJob["V"], CultureInfo.InvariantCulture);
}
if (recurringJob.TryGetValue("RetryAttempt", out var attemptString) &&
int.TryParse(attemptString, out var retryAttempt))
{
RetryAttempt = retryAttempt;
}
}
public void AddOrUpdate_UsesCurrentTime_InsteadOfLastExecution_ToCalculateNextExecution_WhenChangingCronExpression()
{
// Arrange
_connection.Setup(x => x.GetAllEntriesFromHash($"recurring-job:{_id}")).Returns(new Dictionary <string, string>
{
{ "Cron", "30 12 * * *" },
{ "Job", InvocationData.Serialize(_job).SerializePayload() },
{ "LastExecution", JobHelper.SerializeDateTime(_now.AddHours(-3)) }
});
var manager = CreateManager();
// Act
manager.AddOrUpdate(_id, _job, "30 13 * * *");
// Assert
_transaction.Verify(x => x.SetRangeInHash($"recurring-job:{_id}", It.Is <Dictionary <string, string> >(dict =>
JobHelper.DeserializeDateTime(dict["NextExecution"]) == _now.AddHours(22))));
_transaction.Verify(x => x.AddToSet("recurring-jobs", _id, JobHelper.ToTimestamp(_now.AddHours(22))));
_transaction.Verify(x => x.Commit());
}
public override void Execute()
{
WriteLiteral("\n");
#line 8 "..\..\Pages\FailedJobsPage.cshtml"
Layout = new LayoutPage {
Title = "Failed Jobs"
};
int from, perPage;
int.TryParse(Request.QueryString["from"], out from);
int.TryParse(Request.QueryString["count"], out perPage);
var monitor = JobStorage.Current.GetMonitoringApi();
Pager pager = new Pager(from, perPage, monitor.FailedCount())
{
BasePageUrl = Request.LinkTo("/failed")
};
JobList <FailedJobDto> failedJobs = monitor
.FailedJobs(pager.FromRecord, pager.RecordsPerPage);
#line default
#line hidden
WriteLiteral("\n");
#line 26 "..\..\Pages\FailedJobsPage.cshtml"
if (pager.TotalPageCount == 0)
{
#line default
#line hidden
WriteLiteral(" <div class=\"alert alert-success\">\n You have no failed jobs at the mome" +
"nt.\n </div>\n");
#line 31 "..\..\Pages\FailedJobsPage.cshtml"
}
else
{
#line default
#line hidden
WriteLiteral(" <div class=\"js-jobs-list\">\n <div class=\"btn-toolbar btn-toolbar-top\">\n" +
" <button class=\"js-jobs-list-command btn btn-sm btn-primary\"\n " +
" data-url=\"");
#line 37 "..\..\Pages\FailedJobsPage.cshtml"
Write(Request.LinkTo("/failed/requeue"));
#line default
#line hidden
WriteLiteral(@"""
data-loading-text=""Enqueueing..."">
<span class=""glyphicon glyphicon-repeat""></span>
Requeue jobs
</button>
<button class=""js-jobs-list-command btn btn-sm btn-default""
data-url=""");
#line 44 "..\..\Pages\FailedJobsPage.cshtml"
Write(Request.LinkTo("/failed/delete"));
#line default
#line hidden
WriteLiteral(@"""
data-loading-text=""Deleting...""
data-confirm=""Do you really want to DELETE ALL selected jobs?"">
<span class=""glyphicon glyphicon-remove""></span>
Delete selected
</button>
");
#line 51 "..\..\Pages\FailedJobsPage.cshtml"
Write(RenderPartial(new PerPageSelector(pager)));
#line default
#line hidden
WriteLiteral(@"
</div>
<table class=""table failed-table"">
<thead>
<tr>
<th class=""min-width"">
<input type=""checkbox"" class=""js-jobs-list-select-all"" />
</th>
<th class=""min-width"">Id</th>
<th>Failed</th>
<th>Job</th>
</tr>
</thead>
<tbody>
");
#line 66 "..\..\Pages\FailedJobsPage.cshtml"
var index = 0;
#line default
#line hidden
#line 67 "..\..\Pages\FailedJobsPage.cshtml"
foreach (var job in failedJobs)
{
#line default
#line hidden
WriteLiteral(" <tr class=\"js-jobs-list-row ");
#line 69 "..\..\Pages\FailedJobsPage.cshtml"
Write(!job.Value.InFailedState ? "obsolete-data" : null);
#line default
#line hidden
WriteLiteral(" ");
#line 69 "..\..\Pages\FailedJobsPage.cshtml"
Write(job.Value.InFailedState ? "hover" : null);
#line default
#line hidden
WriteLiteral("\">\n <td rowspan=\"");
#line 70 "..\..\Pages\FailedJobsPage.cshtml"
Write(job.Value.InFailedState ? "2" : "1");
#line default
#line hidden
WriteLiteral("\">\n");
#line 71 "..\..\Pages\FailedJobsPage.cshtml"
if (job.Value.InFailedState)
{
#line default
#line hidden
WriteLiteral(" <input type=\"checkbox\" class=\"js-jobs-list-checkb" +
"ox\" name=\"jobs[]\" value=\"");
#line 73 "..\..\Pages\FailedJobsPage.cshtml"
Write(job.Key);
#line default
#line hidden
WriteLiteral("\" />\n");
#line 74 "..\..\Pages\FailedJobsPage.cshtml"
}
#line default
#line hidden
WriteLiteral(" </td>\n <td class=\"min-width\" rowsp" +
"an=\"");
#line 76 "..\..\Pages\FailedJobsPage.cshtml"
Write(job.Value.InFailedState ? "2" : "1");
#line default
#line hidden
WriteLiteral("\">\n <a href=\"");
#line 77 "..\..\Pages\FailedJobsPage.cshtml"
Write(Request.LinkTo("/job/" + job.Key));
#line default
#line hidden
WriteLiteral("\">\n ");
#line 78 "..\..\Pages\FailedJobsPage.cshtml"
Write(HtmlHelper.JobId(job.Key));
#line default
#line hidden
WriteLiteral("\n </a>\n");
#line 80 "..\..\Pages\FailedJobsPage.cshtml"
if (!job.Value.InFailedState)
{
#line default
#line hidden
WriteLiteral(" <span title=\"Job\'s state has been changed while f" +
"etching data.\" class=\"glyphicon glyphicon-question-sign\"></span>\n");
#line 83 "..\..\Pages\FailedJobsPage.cshtml"
}
#line default
#line hidden
WriteLiteral(" </td>\n <td class=\"min-width\">\n");
#line 86 "..\..\Pages\FailedJobsPage.cshtml"
if (job.Value.FailedAt.HasValue)
{
#line default
#line hidden
WriteLiteral(" <span data-moment=\"");
#line 88 "..\..\Pages\FailedJobsPage.cshtml"
Write(JobHelper.ToTimestamp(job.Value.FailedAt.Value));
#line default
#line hidden
WriteLiteral("\">\n ");
#line 89 "..\..\Pages\FailedJobsPage.cshtml"
Write(job.Value.FailedAt);
#line default
#line hidden
WriteLiteral("\n </span>\n");
#line 91 "..\..\Pages\FailedJobsPage.cshtml"
}
#line default
#line hidden
WriteLiteral(" </td>\n <td>\n " +
" <div>\n <a class=\"job-method\" href=\"");
#line 95 "..\..\Pages\FailedJobsPage.cshtml"
Write(Request.LinkTo("/job/" + job.Key));
#line default
#line hidden
WriteLiteral("\">\n ");
#line 96 "..\..\Pages\FailedJobsPage.cshtml"
Write(HtmlHelper.DisplayMethod(job.Value.Job));
#line default
#line hidden
WriteLiteral("\n </a>\n </div>\n");
#line 99 "..\..\Pages\FailedJobsPage.cshtml"
if (!String.IsNullOrEmpty(job.Value.ExceptionMessage))
{
#line default
#line hidden
WriteLiteral(" <div style=\"color: #888;\">\n " +
" ");
#line 102 "..\..\Pages\FailedJobsPage.cshtml"
Write(job.Value.Reason);
#line default
#line hidden
WriteLiteral(" <a class=\"expander\" href=\"#\">More details...</a>\n " +
" </div>\n");
#line 104 "..\..\Pages\FailedJobsPage.cshtml"
}
#line default
#line hidden
WriteLiteral(" </td>\n </tr>\n");
#line 107 "..\..\Pages\FailedJobsPage.cshtml"
if (job.Value.InFailedState)
{
#line default
#line hidden
WriteLiteral(" <tr>\n <td colspan=\"2\" class=\"f" +
"ailed-job-details\">\n <div class=\"expandable\" styl" +
"e=\"");
#line 111 "..\..\Pages\FailedJobsPage.cshtml"
Write(index++ == 0 ? "display: block;" : null);
#line default
#line hidden
WriteLiteral("\">\n <h4>");
#line 112 "..\..\Pages\FailedJobsPage.cshtml"
Write(job.Value.ExceptionType);
#line default
#line hidden
WriteLiteral("</h4>\n <p>\n " +
" ");
#line 114 "..\..\Pages\FailedJobsPage.cshtml"
Write(job.Value.ExceptionMessage);
#line default
#line hidden
WriteLiteral("\n </p>\n\n");
#line 117 "..\..\Pages\FailedJobsPage.cshtml"
if (!String.IsNullOrEmpty(job.Value.ExceptionDetails))
{
#line default
#line hidden
WriteLiteral(" <pre class=\"stack-trace\">");
#line 119 "..\..\Pages\FailedJobsPage.cshtml"
Write(HtmlHelper.MarkupStackTrace(job.Value.ExceptionDetails));
#line default
#line hidden
WriteLiteral("</pre>\n");
#line 120 "..\..\Pages\FailedJobsPage.cshtml"
}
#line default
#line hidden
WriteLiteral(" </div>\n </td>\n " +
" </tr>\n");
#line 124 "..\..\Pages\FailedJobsPage.cshtml"
}
}
#line default
#line hidden
WriteLiteral(" </tbody>\n </table>\n </div>\n");
#line 129 "..\..\Pages\FailedJobsPage.cshtml"
#line default
#line hidden
#line 130 "..\..\Pages\FailedJobsPage.cshtml"
Write(RenderPartial(new Paginator(pager)));
#line default
#line hidden
#line 130 "..\..\Pages\FailedJobsPage.cshtml"
}
#line default
#line hidden
}
public void Trigger_CanTriggerRecurringJob_WithCronThatNeverFires()
{
// Arrange
_connection.Setup(x => x.GetAllEntriesFromHash($"recurring-job:{_id}"))
.Returns(new Dictionary <string, string>
{
{ "Job", JobHelper.ToJson(InvocationData.Serialize(_job)) },
{ "Cron", "0 0 31 2 *" },
});
var manager = CreateManager();
// Act
manager.Trigger(_id);
// Assert
_stateMachine.Verify(x => x.ApplyState(It.IsAny <ApplyStateContext>()));
_transaction.Verify(x => x.SetRangeInHash($"recurring-job:{_id}", It.Is <Dictionary <string, string> >(dict =>
dict.ContainsKey("LastExecution") && dict["LastExecution"] == JobHelper.SerializeDateTime(_now) &&
!dict.ContainsKey("NextExecution"))));
_transaction.Verify(x => x.AddToSet("recurring-jobs", _id, -1.0D));
_transaction.Verify(x => x.Commit());
}
private static List <Continuation> GetContinuations(IStorageConnection connection, string jobId)
{
return(JobHelper.FromJson <List <Continuation> >(connection.GetJobParameter(
jobId, "Continuations")) ?? new List <Continuation>());
}
public void Trigger_SchedulesNextExecution_DependingOnCurrentTime_ToTheFuture()
{
// Arrange
_connection.Setup(x => x.GetAllEntriesFromHash($"recurring-job:{_id}")).Returns(new Dictionary <string, string>
{
{ "Cron", "* * * * *" },
{ "Job", InvocationData.Serialize(_job).SerializePayload() },
{ "CreatedAt", JobHelper.SerializeDateTime(_now.AddMinutes(-3)) },
{ "LastExecution", JobHelper.SerializeDateTime(_now.AddMinutes(-2)) },
{ "NextExecution", JobHelper.SerializeDateTime(_now.AddMinutes(-1)) }
});
var manager = CreateManager();
// Act
manager.Trigger(_id);
// Assert
_transaction.Verify(x => x.SetRangeInHash($"recurring-job:{_id}", It.Is <Dictionary <string, string> >(dict =>
dict["NextExecution"] == JobHelper.SerializeDateTime(_now.AddMinutes(1)))));
_transaction.Verify(x => x.AddToSet("recurring-jobs", _id, JobHelper.ToTimestamp(_now.AddMinutes(1))));
_transaction.Verify(x => x.Commit());
}
private bool TimedOutByFetchedTime(string fetchedTimestamp)
{
return(!String.IsNullOrEmpty(fetchedTimestamp) &&
(DateTime.UtcNow - JobHelper.FromStringTimestamp(fetchedTimestamp) > _options.JobTimeout));
}
public async Task Dispatch(DashboardContext context)
{
var request = context.Request;
var response = context.Response;
if (!"POST".Equals(request.Method, StringComparison.OrdinalIgnoreCase))
{
response.StatusCode = (int)HttpStatusCode.MethodNotAllowed;
return;
}
var jobIds = await context.Request.GetFormValuesAsync("jobid").ConfigureAwait(false);
var jobId = jobIds.FirstOrDefault()?.Trim();
var urlHelper = new UrlHelper(context);
string link = null;
if (!String.IsNullOrEmpty(jobId))
{
try
{
var monitoringApi = context.Storage.GetMonitoringApi();
var normalizedId = jobId.StartsWith("#") ? jobId.Substring(1) : jobId;
var jobDetails = monitoringApi.JobDetails(normalizedId);
if (jobDetails != null)
{
link = urlHelper.JobDetails(normalizedId);
}
}
catch
{
//
}
if (link == null)
{
using (var connection = context.Storage.GetConnection())
{
var recurringJob = connection.GetAllEntriesFromHash($"recurring-job:{jobId}");
if (recurringJob != null && recurringJob.Count != 0)
{
link = urlHelper.To("/recurring/details/" + jobId);
}
}
}
if (link == null)
{
using (var connection = context.Storage.GetConnection())
{
var batch = connection.GetAllEntriesFromHash($"batch:{jobId}");
if (batch != null && batch.Count != 0)
{
link = urlHelper.To("/batches/" + jobId);
}
}
}
}
response.StatusCode = link != null ? 200 : 404;
#pragma warning disable 618
await response.WriteAsync(JobHelper.ToJson(new Dictionary <string, string>
#pragma warning restore 618
{
{ "location", link }
}));
}
