public void HangfireScheduler_ResumeJob_InSucceededState_Failed()
{
var dataObjectMock = new Mock <IDSFDataObject>();
var values = new Dictionary <string, StringBuilder>
{
{ "resourceID", new StringBuilder("ab04663e-1e09-4338-8f61-a06a7ae5ebab") },
{ "environment", new StringBuilder("NewEnvironment") },
{ "startActivityId", new StringBuilder("4032a11e-4fb3-4208-af48-b92a0602ab4b") },
{ "versionNumber", new StringBuilder("1") },
{ "currentuserprincipal", new StringBuilder(WindowsIdentity.GetCurrent().Name) }
};
var jobstorage = new MemoryStorage();
var client = new BackgroundJobClient(jobstorage);
var scheduler = new Persistence.Drivers.HangfireScheduler(client, jobstorage);
var jobId = scheduler.ScheduleJob(enSuspendOption.SuspendForSeconds, "1", values);
var o = new object();
var state = new SucceededState(o, 1, 1);
client.ChangeState(jobId, state, ScheduledState.StateName);
var result = scheduler.ResumeJob(dataObjectMock.Object, jobId, false, "NewEnvironment");
Assert.AreEqual(GlobalConstants.Failed, result);
}
public void ChangeToDeletedState()
{
#region DeletedState
var client = new BackgroundJobClient();
var jobId = client.Enqueue(() => Console.WriteLine("Hello"));
var state = new DeletedState();
client.ChangeState(jobId, state, EnqueuedState.StateName);
#endregion
}
public void HangfireScheduler_ResumeJob_FromFailedState_Success()
{
var mockStateNotifier = new Mock <IStateNotifier>();
mockStateNotifier.Setup(o => o.LogAdditionalDetail(It.IsAny <Audit>(), "ResumeJob")).Verifiable();
var dataObjectMock = new Mock <IDSFDataObject>();
dataObjectMock.Setup(o => o.StateNotifier).Returns(mockStateNotifier.Object);
var values = new Dictionary <string, StringBuilder>
{
{ "resourceID", new StringBuilder("ab04663e-1e09-4338-8f61-a06a7ae5ebab") },
{ "environment", new StringBuilder("NewEnvironment") },
{ "startActivityId", new StringBuilder("4032a11e-4fb3-4208-af48-b92a0602ab4b") },
{ "versionNumber", new StringBuilder("1") },
{ "currentuserprincipal", new StringBuilder(WindowsIdentity.GetCurrent().Name) }
};
var suspendOption = enSuspendOption.SuspendUntil;
var suspendOptionValue = DateTime.Now.AddDays(1).ToString();
var jobstorage = new MemoryStorage();
var client = new BackgroundJobClient(jobstorage);
var scheduler = new Persistence.Drivers.HangfireScheduler(client, jobstorage);
var jobId = scheduler.ScheduleJob(suspendOption, suspendOptionValue, values);
var state = new EnqueuedState();
client.ChangeState(jobId, state, FailedState.StateName);
var errors = new ErrorResultTO();
var resourceCatalog = new Mock <IResourceCatalog>();
resourceCatalog.Setup(catalog => catalog.GetService(GlobalConstants.ServerWorkspaceID, It.IsAny <Guid>(), "")).Returns(CreateServiceEntry());
CustomContainer.Register(resourceCatalog.Object);
var mockResumableExecutionContainer = new Mock <IResumableExecutionContainer>();
mockResumableExecutionContainer.Setup(o => o.Execute(out errors, 0)).Verifiable();
var mockResumableExecutionContainerFactory = new Mock <IResumableExecutionContainerFactory>();
mockResumableExecutionContainerFactory.Setup(o => o.New(It.IsAny <Guid>(), It.IsAny <ServiceAction>(), It.IsAny <DsfDataObject>()))
.Returns(mockResumableExecutionContainer.Object);
CustomContainer.Register(mockResumableExecutionContainerFactory.Object);
var result = scheduler.ResumeJob(dataObjectMock.Object, jobId, true, "NewEnvironment_Override");
Assert.AreEqual(GlobalConstants.Success, result);
mockResumableExecutionContainer.Verify(o => o.Execute(out errors, 0), Times.Once);
mockStateNotifier.Verify(o => o.LogAdditionalDetail(It.IsAny <Audit>(), "ResumeJob"), Times.Once);
}
public void ChangeToEnqueuedState()
{
var jobId = "someid";
#region EnqueuedState #2
var client = new BackgroundJobClient();
var state = new EnqueuedState(); // Use the default queue
client.ChangeState(jobId, state, FailedState.StateName);
#endregion
}
//Note: this can be done better later
private void Throw(string jobId, string message, string reason)
{
var client = new BackgroundJobClient();
var exception = new Exception(message);
_ = client.ChangeState(jobId, new FailedState(exception)
{
Reason = reason
}, ProcessingState.StateName);
throw exception;
}
public static void AggregationGrainEnd(string aggregateJobId, string lockId, string grainId, PerformContext context)
{
context.Connection.SetRangeInHash(lockId, new[] { new KeyValuePair <string, string>(grainId, "true") });
var grainsData = context.Connection.GetAllEntriesFromHash(lockId);
var shouldStart = grainsData.All(x => x.Value == "true");
if (shouldStart)
{
var client = new BackgroundJobClient();
client.ChangeState(aggregateJobId, new EnqueuedState());
}
}
/// <summary>
/// Marks the specified workflow as inactive and cancels all currently scheduled tasks for that
/// </summary>
/// <param name="id"></param>
public async System.Threading.Tasks.Task <bool> Delete(string id)
{
var workflow = await _context.Workflows.Where(w => w.id == id).FirstOrDefaultAsync();
if (workflow != null)
{
Log.Information("Cancelling Workflow: {0}, and associated Tasks.", id);
var tasks = await _context.Tasks.Where(t => t.workflowID == id).ToListAsync();
foreach (LimsServer.Entities.Task t in tasks)
{
if (t.status == "SCHEDULED")
{
t.status = "CANCELLED";
t.message = "Corresponding workflow cancelled.";
var newState = new Hangfire.States.DeletedState();
Log.Information("Task Cancelled. WorkflowID: {0}, ID: {1}, Hangfire ID: {2}", t.workflowID, t.id, t.taskID);
try
{
BackgroundJobClient backgroundClient = new BackgroundJobClient();
backgroundClient.ChangeState(t.taskID, newState);
}
catch (Exception)
{
Log.Warning("Error unable to change Hangfire background job to deleted state. Task ID: {0}", t.taskID);
}
}
}
Log.Information("Setting LIMS Workflow, ID: {0} to inactive", id);
workflow.active = false;
await _context.SaveChangesAsync();
return(true);
}
else
{
Log.Information("Unable to cancel Workflow: {0}, ID not found.", id);
return(false);
}
}
public static void SetTrigger(this IStorageConnection connection, string triggerName)
{
var jsc = (JobStorageConnection)connection;
var triggerKey = GenerateTriggerKey(triggerName);
var jobIds = jsc.GetAllItemsFromList(triggerKey);
var client = new BackgroundJobClient();
try
{
using (connection.AcquireDistributedLock(triggerKey, TimeSpan.Zero))
{
foreach (var jobId in jobIds)
{
client.ChangeState(jobId, new EnqueuedState());
}
}
}
catch (DistributedLockTimeoutException)
{
// Assume already run
}
}
static DashboardRoutes()
{
Routes = new RouteCollection();
Routes.AddRazorPage("/", x => new HomePage());
Routes.Add("/stats", new JsonStats());
#region Embedded static content
Routes.Add("/js[0-9]{3}", new CombinedResourceDispatcher(
"application/javascript",
GetExecutingAssembly(),
GetContentFolderNamespace("js"),
Javascripts));
Routes.Add("/css[0-9]{3}", new CombinedResourceDispatcher(
"text/css",
GetExecutingAssembly(),
GetContentFolderNamespace("css"),
Stylesheets));
Routes.Add("/fonts/glyphicons-halflings-regular/eot", new EmbeddedResourceDispatcher(
"application/vnd.ms-fontobject",
GetExecutingAssembly(),
GetContentResourceName("fonts", "glyphicons-halflings-regular.eot")));
Routes.Add("/fonts/glyphicons-halflings-regular/svg", new EmbeddedResourceDispatcher(
"image/svg+xml",
GetExecutingAssembly(),
GetContentResourceName("fonts", "glyphicons-halflings-regular.svg")));
Routes.Add("/fonts/glyphicons-halflings-regular/ttf", new EmbeddedResourceDispatcher(
"application/octet-stream",
GetExecutingAssembly(),
GetContentResourceName("fonts", "glyphicons-halflings-regular.ttf")));
Routes.Add("/fonts/glyphicons-halflings-regular/woff", new EmbeddedResourceDispatcher(
"font/woff",
GetExecutingAssembly(),
GetContentResourceName("fonts", "glyphicons-halflings-regular.woff")));
Routes.Add("/fonts/glyphicons-halflings-regular/woff2", new EmbeddedResourceDispatcher(
"font/woff2",
GetExecutingAssembly(),
GetContentResourceName("fonts", "glyphicons-halflings-regular.woff2")));
#endregion
#region Razor pages and commands
Routes.AddRazorPage("/jobs/enqueued", x => new QueuesPage());
Routes.AddRazorPage(
"/jobs/enqueued/fetched/(?<Queue>.+)",
x => new FetchedJobsPage(x.Groups["Queue"].Value));
Routes.AddClientBatchCommand("/jobs/enqueued/delete", (client, jobId) => client.ChangeState(jobId, CreateDeletedState()));
Routes.AddClientBatchCommand("/jobs/enqueued/requeue", (client, jobId) => client.ChangeState(jobId, CreateEnqueuedState()));
Routes.AddRazorPage(
"/jobs/enqueued/(?<Queue>.+)",
x => new EnqueuedJobsPage(x.Groups["Queue"].Value));
Routes.AddRazorPage("/jobs/processing", x => new ProcessingJobsPage());
Routes.AddClientBatchCommand(
"/jobs/processing/delete",
(client, jobId) => client.ChangeState(jobId, CreateDeletedState(), ProcessingState.StateName));
Routes.AddClientBatchCommand(
"/jobs/processing/requeue",
(client, jobId) => client.ChangeState(jobId, CreateEnqueuedState(), ProcessingState.StateName));
Routes.AddRazorPage("/jobs/scheduled", x => new ScheduledJobsPage());
Routes.AddClientBatchCommand(
"/jobs/scheduled/enqueue",
(client, jobId) => client.ChangeState(jobId, CreateEnqueuedState(), ScheduledState.StateName));
Routes.AddClientBatchCommand(
"/jobs/scheduled/delete",
(client, jobId) => client.ChangeState(jobId, CreateDeletedState(), ScheduledState.StateName));
Routes.AddRazorPage("/jobs/succeeded", x => new SucceededJobs());
Routes.AddClientBatchCommand(
"/jobs/succeeded/requeue",
(client, jobId) => client.ChangeState(jobId, CreateEnqueuedState(), SucceededState.StateName));
Routes.AddRazorPage("/jobs/failed", x => new FailedJobsPage());
Routes.AddClientBatchCommand(
"/jobs/failed/requeue",
(client, jobId) => client.ChangeState(jobId, CreateEnqueuedState(), FailedState.StateName));
Routes.AddClientBatchCommand(
"/jobs/failed/delete",
(client, jobId) => client.ChangeState(jobId, CreateDeletedState(), FailedState.StateName));
Routes.AddRazorPage("/jobs/deleted", x => new DeletedJobsPage());
Routes.AddClientBatchCommand(
"/jobs/deleted/requeue",
(client, jobId) => client.ChangeState(jobId, CreateEnqueuedState(), DeletedState.StateName));
Routes.AddRazorPage("/jobs/awaiting", x => new AwaitingJobsPage());
Routes.AddClientBatchCommand("/jobs/awaiting/enqueue", (client, jobId) => client.ChangeState(
jobId, CreateEnqueuedState(), AwaitingState.StateName));
Routes.AddClientBatchCommand("/jobs/awaiting/delete", (client, jobId) => client.ChangeState(
jobId, CreateDeletedState(), AwaitingState.StateName));
Routes.AddCommand(
"/jobs/actions/requeue/(?<JobId>.+)",
context =>
{
var client = new BackgroundJobClient(context.Storage);
return client.ChangeState(context.UriMatch.Groups["JobId"].Value, CreateEnqueuedState());
});
Routes.AddCommand(
"/jobs/actions/delete/(?<JobId>.+)",
context =>
{
var client = new BackgroundJobClient(context.Storage);
return client.ChangeState(context.UriMatch.Groups["JobId"].Value, CreateDeletedState());
});
Routes.AddRazorPage("/jobs/details/(?<JobId>.+)", x => new JobDetailsPage(x.Groups["JobId"].Value));
Routes.AddRazorPage("/recurring", x => new RecurringJobsPage());
Routes.AddRecurringBatchCommand(
"/recurring/remove",
(manager, jobId) => manager.RemoveIfExists(jobId));
Routes.AddRecurringBatchCommand(
"/recurring/trigger",
(manager, jobId) => manager.Trigger(jobId));
Routes.AddRazorPage("/servers", x => new ServersPage());
Routes.AddRazorPage("/retries", x => new RetriesPage());
#endregion
}
static DashboardRoutes()
{
Routes = new RouteCollection();
Routes.AddRazorPage("/", x => new HomePage());
Routes.Add("/stats", new JsonStats());
#region Embedded static content
Routes.Add("/js[0-9]+", new CombinedResourceDispatcher(
"application/javascript",
GetExecutingAssembly(),
GetContentFolderNamespace("js"),
Javascripts));
Routes.Add("/css[0-9]+", new CombinedResourceDispatcher(
"text/css",
GetExecutingAssembly(),
GetContentFolderNamespace("css"),
Stylesheets));
Routes.Add("/fonts/glyphicons-halflings-regular/eot", new EmbeddedResourceDispatcher(
"application/vnd.ms-fontobject",
GetExecutingAssembly(),
GetContentResourceName("fonts", "glyphicons-halflings-regular.eot")));
Routes.Add("/fonts/glyphicons-halflings-regular/svg", new EmbeddedResourceDispatcher(
"image/svg+xml",
GetExecutingAssembly(),
GetContentResourceName("fonts", "glyphicons-halflings-regular.svg")));
Routes.Add("/fonts/glyphicons-halflings-regular/ttf", new EmbeddedResourceDispatcher(
"application/octet-stream",
GetExecutingAssembly(),
GetContentResourceName("fonts", "glyphicons-halflings-regular.ttf")));
Routes.Add("/fonts/glyphicons-halflings-regular/woff", new EmbeddedResourceDispatcher(
"font/woff",
GetExecutingAssembly(),
GetContentResourceName("fonts", "glyphicons-halflings-regular.woff")));
Routes.Add("/fonts/glyphicons-halflings-regular/woff2", new EmbeddedResourceDispatcher(
"font/woff2",
GetExecutingAssembly(),
GetContentResourceName("fonts", "glyphicons-halflings-regular.woff2")));
#endregion
#region Razor pages and commands
Routes.AddRazorPage("/jobs/enqueued", x => new QueuesPage());
Routes.AddRazorPage(
"/jobs/enqueued/fetched/(?<Queue>.+)",
x => new FetchedJobsPage(x.Groups["Queue"].Value));
Routes.AddClientBatchCommand("/jobs/enqueued/delete", (client, jobId) => client.ChangeState(jobId, CreateDeletedState()));
Routes.AddClientBatchCommand("/jobs/enqueued/requeue", (client, jobId) => client.ChangeState(jobId, CreateEnqueuedState()));
Routes.AddRazorPage(
"/jobs/enqueued/(?<Queue>.+)",
x => new EnqueuedJobsPage(x.Groups["Queue"].Value));
Routes.AddRazorPage("/jobs/processing", x => new ProcessingJobsPage());
Routes.AddClientBatchCommand(
"/jobs/processing/delete",
(client, jobId) => client.ChangeState(jobId, CreateDeletedState(), ProcessingState.StateName));
Routes.AddClientBatchCommand(
"/jobs/processing/requeue",
(client, jobId) => client.ChangeState(jobId, CreateEnqueuedState(), ProcessingState.StateName));
Routes.AddRazorPage("/jobs/scheduled", x => new ScheduledJobsPage());
Routes.AddClientBatchCommand(
"/jobs/scheduled/enqueue",
(client, jobId) => client.ChangeState(jobId, CreateEnqueuedState(), ScheduledState.StateName));
Routes.AddClientBatchCommand(
"/jobs/scheduled/delete",
(client, jobId) => client.ChangeState(jobId, CreateDeletedState(), ScheduledState.StateName));
Routes.AddRazorPage("/jobs/succeeded", x => new SucceededJobs());
Routes.AddClientBatchCommand(
"/jobs/succeeded/requeue",
(client, jobId) => client.ChangeState(jobId, CreateEnqueuedState(), SucceededState.StateName));
Routes.AddRazorPage("/jobs/failed", x => new FailedJobsPage());
Routes.AddClientBatchCommand(
"/jobs/failed/requeue",
(client, jobId) => client.ChangeState(jobId, CreateEnqueuedState(), FailedState.StateName));
Routes.AddClientBatchCommand(
"/jobs/failed/delete",
(client, jobId) => client.ChangeState(jobId, CreateDeletedState(), FailedState.StateName));
Routes.AddRazorPage("/jobs/deleted", x => new DeletedJobsPage());
Routes.AddClientBatchCommand(
"/jobs/deleted/requeue",
(client, jobId) => client.ChangeState(jobId, CreateEnqueuedState(), DeletedState.StateName));
Routes.AddRazorPage("/jobs/awaiting", x => new AwaitingJobsPage());
Routes.AddClientBatchCommand("/jobs/awaiting/enqueue", (client, jobId) => client.ChangeState(
jobId, CreateEnqueuedState(), AwaitingState.StateName));
Routes.AddClientBatchCommand("/jobs/awaiting/delete", (client, jobId) => client.ChangeState(
jobId, CreateDeletedState(), AwaitingState.StateName));
Routes.AddCommand(
"/jobs/actions/requeue/(?<JobId>.+)",
context =>
{
var client = new BackgroundJobClient(context.Storage);
return(client.ChangeState(context.UriMatch.Groups["JobId"].Value, CreateEnqueuedState()));
});
Routes.AddCommand(
"/jobs/actions/delete/(?<JobId>.+)",
context =>
{
var client = new BackgroundJobClient(context.Storage);
return(client.ChangeState(context.UriMatch.Groups["JobId"].Value, CreateDeletedState()));
});
Routes.AddRazorPage("/jobs/details/(?<JobId>.+)", x => new JobDetailsPage(x.Groups["JobId"].Value));
Routes.AddRazorPage("/recurring", x => new RecurringJobsPage());
Routes.AddRecurringBatchCommand(
"/recurring/remove",
(manager, jobId) => manager.RemoveIfExists(jobId));
Routes.AddRecurringBatchCommand(
"/recurring/trigger",
(manager, jobId) => manager.Trigger(jobId));
Routes.AddRazorPage("/servers", x => new ServersPage());
Routes.AddRazorPage("/retries", x => new RetriesPage());
#endregion
}
bool IBackgroundJobClient.ChangeState(string jobId, IState state, string expectedState) => Inner.ChangeState(jobId, state, expectedState);
/// <summary>
/// Updates the workflow provided by id
/// </summary>
/// <param name="workflow"></param>
public async System.Threading.Tasks.Task <bool> Update(Workflow _workflow, bool bypass = false)
{
string id = _workflow.id;
var workflow = await _context.Workflows.Where(w => w.id == id).FirstOrDefaultAsync();
if (workflow != null)
{
workflow.Update(_workflow);
await _context.SaveChangesAsync();
Log.Information("Updating Workflow: {0}, and reschduling existing Tasks.", id);
var tasks = await _context.Tasks.Where(t => t.workflowID == id).ToListAsync();
bool taskRunning = false;
foreach (LimsServer.Entities.Task t in tasks)
{
if (t.status == "SCHEDULED" && workflow.active)
{
t.start = DateTime.Now.AddMinutes(workflow.interval);
await _context.SaveChangesAsync();
Log.Information("Task Rescheduled. WorkflowID: {0}, ID: {1}, Hangfire ID: {2}, Input Directory: {3}, Message: {4}", t.workflowID, t.id, t.taskID, workflow.inputFolder, "Workflow updated, task rescheduled to new workflow configuration.");
taskRunning = true;
try
{
var newSchedule = new Hangfire.States.ScheduledState(TimeSpan.FromMinutes(workflow.interval));
BackgroundJobClient backgroundClient = new BackgroundJobClient();
backgroundClient.ChangeState(t.taskID, newSchedule);
}
catch (Exception)
{
Log.Warning("Error rescheduling Hangfire background job. Job ID: {0}", t.taskID);
}
}
else
{
t.status = "CANCELLED";
t.message = "Corresponding workflow updated.";
var newState = new Hangfire.States.DeletedState();
Log.Information("Task Cancelled. WorkflowID: {0}, ID: {1}, Hangfire ID: {2}", t.workflowID, t.id, t.taskID);
try
{
BackgroundJobClient backgroundClient = new BackgroundJobClient();
backgroundClient.ChangeState(t.taskID, newState);
}
catch (Exception)
{
Log.Warning("Error cancelling Hanfire background job. Job ID: {0}", t.taskID);
}
}
}
if (!taskRunning && workflow.active)
{
string newId = System.Guid.NewGuid().ToString();
LimsServer.Entities.Task tsk = new Entities.Task(newId, workflow.id, workflow.interval);
TaskService ts = new TaskService(this._context);
var task = await ts.Create(tsk);
await _context.SaveChangesAsync();
Log.Information("Created new Task for updated Workflow ID: {0}, Updated Task ID: {1}, Hangfire ID: {2}", newId, tsk.id, tsk.taskID);
}
return(true);
}
else
{
Log.Information("Unable to cancel Workflow: {0}, ID not found.", id);
return(false);
}
}
private static void OnVerify(object state)
{
mDetectionTimer.Change(-1, -1);
try
{
using (var connection = JobStorage.Current.GetConnection())
using (var lockStorage = connection.AcquireDistributedLock("JobAgentServer", TimeSpan.FromSeconds(30)))//防止多个server端竞争
{
//拿到有上报的jobId集合
var jobIdList = connection.GetAllItemsFromSet(keyPrefix);
if (jobIdList == null || !jobIdList.Any())
{
return;
}
foreach (var jobId in jobIdList)
{
JobData jobData = connection.GetJobData(jobId);
//拿到真正的运行结果
var hashKey = keyPrefix + jobId;
var result = connection.GetAllEntriesFromHash(hashKey);
using (var tran = connection.CreateWriteTransaction())
{
//job已经不存在了 就直接删除set
if (jobData == null)
{
tran.AddJobState(jobId, new SucceededState(null, 0, 0));
tran.RemoveFromSet(keyPrefix, jobId);
tran.Commit();
continue;
}
double totalMilliseconds = (DateTime.UtcNow - jobData.CreatedAt).TotalMilliseconds;
long latency = (long)totalMilliseconds;
//如果job存在 但是没有拿到hash数据 认为成功
if (result == null || !result.Any())
{
tran.AddJobState(jobId, new SucceededState(null, latency, latency));
tran.RemoveFromSet(keyPrefix, jobId);
tran.RemoveHash(hashKey);
tran.Commit();
continue;
}
var resultOfAgent = result.First();
JobAgentResult resultData = CodingUtil.FromJson <JobAgentResult>(resultOfAgent.Value);
//异常数据 认为成功
if (resultData == null)
{
tran.AddJobState(jobId, new SucceededState(null, latency, latency));
tran.RemoveFromSet(keyPrefix, jobId);
tran.RemoveHash(hashKey);
tran.Commit();
continue;
}
//jobagent实际上运行的时长
long.TryParse(resultOfAgent.Key, out var realTotalMilliseconds);
if (realTotalMilliseconds < 1)
{
realTotalMilliseconds = latency;
}
var isSuccess = resultData.R == "ok";
tran.RemoveFromSet(keyPrefix, jobId);
tran.RemoveHash(hashKey);
// latency 代表的是 从开始调度 到 实际结束 总共的时长
// realTotalMilliseconds 代表的是 jobagent开始执行 到 实际结束的 总共的时长
if (isSuccess)
{
var currentState = connection.GetStateData(jobId);
if (currentState != null && !string.IsNullOrEmpty(currentState.Name) &&
currentState.Name.Equals("Failed"))
{
tran.AddJobState(jobId, new SucceededState(null, latency, realTotalMilliseconds));
}
else
{
backgroundJobClient.ChangeState(jobId, new SucceededState(null, latency, realTotalMilliseconds));
}
}
else
{
var jobItem = jobData.Job.Args.FirstOrDefault() as HttpJobItem;
var ex = new AgentJobException(jobItem.AgentClass, resultData.E);
backgroundJobClient.ChangeState(jobId, new FailedState(ex));
HttpJob.SendFail(jobId, jobItem, "AgentJobFail", ex);
}
//如果是stop上报过来的时候 记录这个job最后的执行id
if (!string.IsNullOrEmpty(resultData.Action) && resultData.Action.Equals("stop") && !string.IsNullOrEmpty(resultData.RunId))
{
var jobItem = jobData.Job.Args.FirstOrDefault() as HttpJobItem;
var jobKeyName =
$"recurring-job:{(!string.IsNullOrEmpty(jobItem.RecurringJobIdentifier) ? jobItem.RecurringJobIdentifier : jobItem.JobName)}";
tran.SetRangeInHash(jobKeyName, new List <KeyValuePair <string, string> > {
new KeyValuePair <string, string>("LastJobId", resultData.RunId)
});
}
//出错的话 需要走通用的出错流程
tran.Commit();
}
}
}
}
catch (Exception e)
{
Logger.ErrorException("agent reporter fail", e);
}
finally
{
mDetectionTimer.Change(1000 * 2, 1000 * 2);
}
}
private static void OnVerify(object state)
{
mDetectionTimer.Change(-1, -1);
if (string.IsNullOrEmpty(ProcessMonitor.CurrentServerId))
{
mDetectionTimer.Change(1000 * 1, 1000 * 1);
return;
}
try
{
var api = JobStorage.Current.GetMonitoringApi();
var totalCount = api.ProcessingCount();
if (totalCount < 1)
{
return;
}
var pageSize = 50;
var totalPages = (totalCount - 1) / pageSize + 1;
for (int i = 0; i < totalPages; i++)
{
var list = api.ProcessingJobs(pageSize * i, pageSize);
using (var connection = JobStorage.Current.GetConnection())
{
foreach (var job in list)
{
if (job.Value.StartedAt != null && (DateTime.UtcNow - job.Value.StartedAt.Value).TotalMinutes < 5)
{
//给5分钟缓冲
continue;
}
var jobItem = job.Value.Job.Args.FirstOrDefault() as HttpJobItem;
//如果是起多个server的情况 检查 job.Value.ServerId 是否还存活
var servers = JobStorage.Current.GetMonitoringApi().Servers();
var targetServer = servers.FirstOrDefault(r => r.Name.Equals(job.Value.ServerId));
//如果server不存在了 或者 server的最后心跳已经是10分钟之前了
if (targetServer == null || (targetServer.Heartbeat != null && (DateTime.UtcNow - targetServer.Heartbeat.Value).TotalMinutes > 10))
{
//hangfire的server挂了导致滞留在processing的job
//转移到error里面去
var ex1 = new HangfireServerShutDownError();
backClient.ChangeState(job.Key, new FailedState(ex1));
if (jobItem != null)
{
HttpJob.SendFail(job.Key, jobItem, "AgentJobFail", ex1);
}
continue;
}
//查看是否是agentJob
if (jobItem == null || string.IsNullOrEmpty(jobItem.AgentClass))
{
continue;
}
//查看这个job的运行的参数是哪个agentserverId
var agentServerId = SerializationHelper.Deserialize <string>(connection.GetJobParameter(job.Key, "agentServerId"), SerializationOption.User);
if (string.IsNullOrEmpty(agentServerId))
{
continue;
}
//查看这个agent最新的 agentserverId
var uri = new Uri(jobItem.Url);
var key = uri.Host + ":" + uri.Port;
var lastAgentServerHash = connection.GetAllEntriesFromHash("activeAgent:" + key);
if (lastAgentServerHash == null || lastAgentServerHash.Count < 1)
{
continue;
}
var lastAgentServer = lastAgentServerHash.First().Value;
if (string.IsNullOrEmpty(lastAgentServer))
{
continue;
}
var currentAgentServer = SerializationHelper.Deserialize <AgentServerInfo>(lastAgentServer);
if (currentAgentServer == null)
{
continue;
}
if (!string.IsNullOrEmpty(currentAgentServer.Id) && currentAgentServer.Id == agentServerId)
{
//虽然没有变化 但是agent其实已经挂了 （其实已经最少挂了15分钟了）
if (currentAgentServer.Time != DateTime.MinValue && (DateTime.UtcNow - currentAgentServer.Time).TotalMinutes > 10)
{
var ex2 = new HangfireAgentShutDownError();
backClient.ChangeState(job.Key, new FailedState(ex2));
HttpJob.SendFail(job.Key, jobItem, "AgentJobFail", ex2);
}
continue;
}
//说明这个agent已经挂了 /说明这个agent重启了
var ex = new HangfireAgentShutDownError();
backClient.ChangeState(job.Key, new FailedState(ex));
HttpJob.SendFail(job.Key, jobItem, "AgentJobFail", ex);
}
}
}
}
catch
{
//ignore
}
finally
{
mDetectionTimer.Change(1000 * 5, 1000 * 5);
}
}
public async System.Threading.Tasks.Task RunTask(string id)
{
var task = await _context.Tasks.SingleAsync(t => t.id == id);
Log.Information("Executing Task. WorkflowID: {0}, ID: {1}, Hangfire ID: {2}", task.workflowID, task.id, task.taskID);
// Step 1: If status!="SCHEDULED" cancel task
if (!task.status.Equals("SCHEDULED"))
{
Log.Information("Task Cancelled. WorkflowID: {0}, ID: {1}, Hangfire ID: {2}, Current Status: {3}, Message: {4}", task.workflowID, task.id, task.taskID, task.status, "Task status is not marked as schedulled.");
await this.UpdateStatus(task.id, "CANCELLED", "Task status was set to: " + task.status);
return;
}
// Step 2: Change status to "STARTING"
await this.UpdateStatus(task.id, "STARTING", "");
var workflow = await _context.Workflows.Where(w => w.id == task.workflowID).FirstOrDefaultAsync();
if (workflow == null)
{
Log.Information("Task Cancelled. WorkflowID: {0}, ID: {1}, Hangfire ID: {2}, Message: {3}", task.workflowID, task.id, task.taskID, "Unable to find Workflow for the Task.");
await this.UpdateStatus(task.id, "CANCELLED", "Error attempting to get workflow of this task. Workflow ID: " + task.workflowID);
return;
}
// Step 3: Check source directory for files
List <string> files = new List <string>();
string dirFileMessage = "";
if (new DirectoryInfo(@workflow.inputFolder).Exists)
{
files = Directory.GetFiles(@workflow.inputFolder).ToList();
}
else
{
dirFileMessage = String.Format("Input directory {0} not found", workflow.inputFolder);
Log.Information(dirFileMessage);
}
// Step 4: If directory or files do not exist reschedule task
if (files.Count == 0)
{
dirFileMessage = (dirFileMessage.Length > 0) ? dirFileMessage : String.Format("No files found in directory: {0}", workflow.inputFolder);
await this.UpdateStatus(task.id, "SCHEDULED", dirFileMessage);
var newSchedule = new Hangfire.States.ScheduledState(TimeSpan.FromMinutes(workflow.interval));
task.start = DateTime.Now.AddMinutes(workflow.interval);
await _context.SaveChangesAsync();
try
{
BackgroundJobClient backgroundClient = new BackgroundJobClient();
backgroundClient.ChangeState(task.taskID, newSchedule);
Log.Information("Task Rescheduled. WorkflowID: {0}, ID: {1}, Hangfire ID: {2}, Input Directory: {3}, Message: {4}", task.workflowID, task.id, task.taskID, workflow.inputFolder, "No files found in input directory.");
}
catch (Exception)
{
Log.Warning("Error attempting to reschedule Hangfire job. Workflow ID: {0}, task ID: {1}", task.workflowID, task.id);
}
return;
}
// Step 5: If file does exist, update task inputFile then compare against previous Tasks.
task.inputFile = files.First();
task.status = "PROCESSING";
await _context.SaveChangesAsync();
bool alreadyCompleted = await this.InputFileCheck(task.inputFile, task.workflowID);
if (alreadyCompleted)
{
Log.Information("Hash input file match for WorkflowID: {0}, ID: {1}, Hangfire ID: {2}, Input File: {3}, Message: {4}", task.workflowID, task.id, task.taskID, task.inputFile, "Rerunning task after removing file.");
try
{
File.Delete(task.inputFile);
Log.Information("Hash input file match successfully deleted. WorkflowID: {0}, ID: {1}, Input File: {2}", task.workflowID, task.id, task.inputFile);
}
catch (FileNotFoundException ex)
{
Log.Warning("Error unable to delete input file after hash file match with previous input file. Workflow ID: {0}, ID: {1}", task.workflowID, task.id);
}
string statusMessage = String.Format("Input file: {0} matches previously processed input file", task.inputFile);
await this.UpdateStatus(task.id, "SCHEDULED", statusMessage);
await this.RunTask(task.id);
return;
}
ProcessorManager pm = new ProcessorManager();
string config = "./app_files/processors";
ProcessorDTO processor = pm.GetProcessors(config).Find(p => p.Name.ToLower() == workflow.processor.ToLower());
if (processor == null)
{
Log.Information("Task Cancelled. WorkflowID: {0}, ID: {1}, Hangfire ID: {2}, Message: {3}, Processor: {4}", task.workflowID, task.id, task.taskID, "Unable to find Processor for the Task.", workflow.processor);
await this.UpdateStatus(task.id, "CANCELLED", "Error, invalid processor name. Name: " + workflow.processor);
return;
}
try
{
// Step 6: Run processor on source file
if (!new DirectoryInfo(@workflow.outputFolder).Exists)
{
Directory.CreateDirectory(@workflow.outputFolder);
}
}
catch (UnauthorizedAccessException ex)
{
Log.Warning("Task unable to create output directory, unauthorized access exception. WorkflowID: {0}, ID: {1}, Hangfire ID: {2}, Output Directory: {3}", task.workflowID, task.id, task.taskID, workflow.outputFolder);
}
Dictionary <string, ResponseMessage> outputs = new Dictionary <string, ResponseMessage>();
string file = task.inputFile;
DataTableResponseMessage result = pm.ExecuteProcessor(processor.Path, processor.Name, file);
GC.Collect();
GC.WaitForPendingFinalizers();
if (result.ErrorMessage == null && result.TemplateData != null)
{
var output = pm.WriteTemplateOutputFile(workflow.outputFolder, result.TemplateData);
outputs.Add(file, output);
}
else
{
string errorMessage = "";
string logMessage = "";
if (result.TemplateData == null)
{
errorMessage = "Processor results template data is null. ";
}
if (result.ErrorMessage != null)
{
errorMessage = errorMessage + result.ErrorMessage;
logMessage = errorMessage;
}
if (result.LogMessage != null)
{
logMessage = result.LogMessage;
}
await this.UpdateStatus(task.id, "CANCELLED", "Error processing data: " + errorMessage);
Log.Information("Task Cancelled. WorkflowID: {0}, ID: {1}, Hangfire ID: {2}, Message: {3}", task.workflowID, task.id, task.taskID, logMessage);
return;
}
// Step 7: Check if output file exists
bool processed = false;
for (int i = 0; i < outputs.Count; i++)
{
var output = outputs.ElementAt(i);
string outputPath = output.Value.OutputFile;
// Step 8: If output file does exist, update task outputFile and delete input file
if (File.Exists(outputPath))
{
processed = true;
string fileName = System.IO.Path.GetFileName(output.Key);
string inputPath = System.IO.Path.Combine(workflow.inputFolder, fileName);
DataBackup dbBackup = new DataBackup();
dbBackup.DumpData(id, inputPath, outputPath);
try
{
File.Delete(inputPath);
}
catch (Exception ex)
{
Log.Warning("Error unable to delete input file after successful processing. Workflow ID: {0}, ID: {1}", task.workflowID, task.id);
}
task.outputFile = outputPath;
await _context.SaveChangesAsync();
}
else
{
await this.UpdateStatus(task.id, "SCHEDULED", "Error unable to export output. Error Messages: " + output.Value.ErrorMessage);
}
}
// Step 9: Change task status to COMPLETED
// Step 10: Create new Task and schedule
string newStatus = "";
if (processed)
{
newStatus = "COMPLETED";
Log.Information("Task Completed. WorkflowID: {0}, ID: {1}, Hangfire ID: {2}", task.workflowID, task.id, task.taskID);
try
{
if (files.Count > 1)
{
await this.CreateNewTask(workflow.id, 0);
}
else
{
await this.CreateNewTask(workflow.id, workflow.interval);
}
}
catch (Exception)
{
Log.Warning("Error creating new Hangfire job after successful job. Workflow ID: {0}, ID: {1}", task.workflowID, task.id);
}
}
else
{
newStatus = "CANCELLED";
Log.Information("Task Cancelled. WorkflowID: {0}, ID: {1}, Hangfire ID: {2}, Message: {3}", task.workflowID, task.id, task.taskID, "Failed to process input file.");
}
await this.UpdateStatus(task.id, newStatus);
}
