private void ReportJobRunFailure(IJobRun jobRun, Exception exception, bool hasRetries, string customNote)
{
var session = EntityHelper.GetSession(jobRun);
var errHeader = string.Format("=========================== Error {0} ======================================"
+ Environment.NewLine, App.TimeService.UtcNow);
var errMsg = errHeader + exception.ToLogString();
if (!string.IsNullOrEmpty(customNote))
{
errMsg += Environment.NewLine + customNote;
}
// If it is not final run, log it as an incident
var job = jobRun.Job;
if (hasRetries && _incidentLog != null)
{
_incidentLog.LogIncident("JobRunFailed", message: exception.Message, key1: job.Name, keyId1: job.Id, notes: errMsg);
}
else
{
_errorLog.LogError(exception, session.Context);
}
//Update job log
jobRun.Log += errMsg + Environment.NewLine;
}
private void UpdateFailedJobRun(IEntitySession session, IJobRun jobRun, JobRunContext jobContext, Exception exception)
{
var utcNow = session.Context.App.TimeService.UtcNow;
jobRun.EndedOn = utcNow;
// current run failed; mark as error
jobRun.Status = jobContext.Status = JobRunStatus.Error;
string customNote = null;
// if exception is soft exc (validation failure) - do not retry
bool isOpAbort = exception is OperationAbortException;
if (isOpAbort)
{
//This will apear in log
customNote = "Attempt resulted in OperationAbort exception, no retries are scheduled.";
}
var hasRetries = !isOpAbort;
if (hasRetries)
{
// get wait time for next attempt
var waitMinutes = GetWaitInterval(jobRun.Job.RetryIntervals, jobRun.AttemptNumber + 1);
hasRetries &= waitMinutes > 0;
if (hasRetries)
{
// create job run for retry
var nextTry = CreateRetryRun(jobRun, waitMinutes);
}
}
ReportJobRunFailure(jobRun, exception, hasRetries, customNote);
}
} //method
private JobRunContext StartJobRun(IJobRun jobRun)
{
_activitiesCounter.Increment();
//create job context
var jobCtx = new JobRunContext(jobRun);
RegisterJobRun(jobCtx);
OnJobNotify(jobCtx, JobNotificationType.Starting);
jobCtx.StartInfo = CreateJobStartInfo(jobRun, jobCtx);
var runServ = App.GetService <IBackgroundTaskService>();
runServ.QueueBackgroundWorkItem(async() => await StartPoolJobRunAsync(jobCtx));
/*
*/
/*
* if(jobCtx.StartInfo.ThreadType == JobThreadType.Background) {
* jobCtx.Thread = new Thread(StartBackgroundJobRun);
* jobCtx.Thread.Start(jobCtx);
* } else {
* Task.Run(() => StartPoolJobRunAsync(jobCtx));
* }
*/
return(jobCtx);
}
public IJobRun CreateRetryRun(IJobRun jobRun, int waitMinutes)
{
var session = EntityHelper.GetSession(jobRun);
var baseTime = jobRun.EndedOn ?? jobRun.StartedOn ?? App.TimeService.UtcNow;
var retryOn = baseTime.AddMinutes(waitMinutes);
var retryRun = NewJobRun(jobRun.Job, JobRunType.Retry, retryOn, jobRun.DataId, jobRun.Data, jobRun.HostName);
retryRun.AttemptNumber = jobRun.AttemptNumber + 1;
retryRun.UserId = jobRun.UserId;
retryRun.Status = JobRunStatus.Pending;
return(retryRun);
}
//internal Thread Thread; //background thread for long-running jobs
internal JobRunContext(IJobRun jobRun)
{
var session = EntityHelper.GetSession(jobRun);
var app = session.Context.App;
UserInfo user = (jobRun.UserId == null) ? UserInfo.System : new UserInfo(jobRun.UserId.Value, null);
OperationContext = new OperationContext(app, user);
StartedOn = app.TimeService.UtcNow;
JobName = jobRun.Job.Name;
JobRunId = jobRun.Id;
RunType = jobRun.RunType;
AttemptNumber = jobRun.AttemptNumber;
var job = jobRun.Job;
JobId = job.Id;
_progress = jobRun.Progress;
IsPersisted = true;
DataId = jobRun.DataId;
Data = jobRun.Data;
}
private JobStartInfo CreateJobStartInfo(IJobRun jobRun, JobRunContext jobContext)
{
var app = jobContext.App;
var startInfo = new JobStartInfo();
var job = jobRun.Job;
startInfo.ThreadType = job.ThreadType;
startInfo.DeclaringType = ReflectionHelper.GetLoadedType(job.DeclaringType, throwIfNotFound: true); //it will throw if cannot find it
var methName = job.MethodName;
var argCount = job.MethodParameterCount;
startInfo.Method = ReflectionHelper.FindMethod(startInfo.DeclaringType, job.MethodName, job.MethodParameterCount);
// if Method is not static, it must be Module instance - this is a convention
if (!startInfo.Method.IsStatic)
{
startInfo.Object = app.GetGlobalObject(startInfo.DeclaringType); //throws if not found.
}
//arguments
var prms = startInfo.Method.GetParameters();
var serArgs = job.Arguments;
var arrStrArgs = serArgs.Split(new[] { JsonArgsDelimiter }, StringSplitOptions.None);
Util.Check(prms.Length == arrStrArgs.Length, "Serialized arg count ({0}) in job entity " +
"does not match the number of target method parameters ({1}); target method: {2}.",
arrStrArgs.Length, prms.Length, startInfo.Method.Name);
// Deserialize arguments
startInfo.Arguments = new object[arrStrArgs.Length];
for (int i = 0; i < arrStrArgs.Length; i++)
{
var paramType = prms[i].ParameterType;
if (paramType == typeof(JobRunContext))
{
startInfo.Arguments[i] = jobContext;
}
else
{
startInfo.Arguments[i] = Deserialize(paramType, arrStrArgs[i]);
}
}
return(startInfo);
}
private void OnJobScheduleSaving(IJobSchedule sched)
{
var session = EntityHelper.GetSession(sched);
var job = sched.Job;
var utcNow = session.Context.App.TimeService.UtcNow;
var nextRunId = sched.NextRunId;
IJobRun nextRun = (nextRunId == null) ? null : session.GetEntity <IJobRun>(nextRunId.Value);
switch (sched.Status)
{
case JobScheduleStatus.Stopped:
// if there is a pending run in the future, kill it
if (nextRun != null && nextRun.Status == JobRunStatus.Pending && nextRun.StartOn > utcNow.AddMinutes(1))
{
nextRun.Status = JobRunStatus.Killed;
}
break;
case JobScheduleStatus.Active:
// Create or adjust JobRun entity for next run
var nextStartOn = sched.GetNextStartAfter(utcNow);
if (nextStartOn != null)
{
if (nextRun == null || nextRun.Status != JobRunStatus.Pending)
{
nextRun = NewJobRun(job, JobRunType.Schedule, nextStartOn, hostName: sched.HostName);
sched.NextRunId = nextRun.Id;
}
else
{
nextRun.StartOn = nextStartOn.Value;
}
}
else
{
//nextSTartOn == null
sched.NextRunId = null;
}
break;
} //switch
} //method
public void Enqueue(IJobRun jobRun)
{
_queuedRuns.Enqueue(jobRun);
ProcessQueue();
}
