private void ReportJobRunFailure(IJobRun jobRun, Exception exception, bool hasRetries, string customNote) { var session = EntityHelper.GetSession(jobRun); var errHeader = string.Format("=========================== Error {0} ======================================" + Environment.NewLine, App.TimeService.UtcNow); var errMsg = errHeader + exception.ToLogString(); if (!string.IsNullOrEmpty(customNote)) { errMsg += Environment.NewLine + customNote; } // If it is not final run, log it as an incident var job = jobRun.Job; if (hasRetries && _incidentLog != null) { _incidentLog.LogIncident("JobRunFailed", message: exception.Message, key1: job.Name, keyId1: job.Id, notes: errMsg); } else { _errorLog.LogError(exception, session.Context); } //Update job log jobRun.Log += errMsg + Environment.NewLine; }
private void UpdateFailedJobRun(IEntitySession session, IJobRun jobRun, JobRunContext jobContext, Exception exception) { var utcNow = session.Context.App.TimeService.UtcNow; jobRun.EndedOn = utcNow; // current run failed; mark as error jobRun.Status = jobContext.Status = JobRunStatus.Error; string customNote = null; // if exception is soft exc (validation failure) - do not retry bool isOpAbort = exception is OperationAbortException; if (isOpAbort) { //This will apear in log customNote = "Attempt resulted in OperationAbort exception, no retries are scheduled."; } var hasRetries = !isOpAbort; if (hasRetries) { // get wait time for next attempt var waitMinutes = GetWaitInterval(jobRun.Job.RetryIntervals, jobRun.AttemptNumber + 1); hasRetries &= waitMinutes > 0; if (hasRetries) { // create job run for retry var nextTry = CreateRetryRun(jobRun, waitMinutes); } } ReportJobRunFailure(jobRun, exception, hasRetries, customNote); }
} //method private JobRunContext StartJobRun(IJobRun jobRun) { _activitiesCounter.Increment(); //create job context var jobCtx = new JobRunContext(jobRun); RegisterJobRun(jobCtx); OnJobNotify(jobCtx, JobNotificationType.Starting); jobCtx.StartInfo = CreateJobStartInfo(jobRun, jobCtx); var runServ = App.GetService <IBackgroundTaskService>(); runServ.QueueBackgroundWorkItem(async() => await StartPoolJobRunAsync(jobCtx)); /* */ /* * if(jobCtx.StartInfo.ThreadType == JobThreadType.Background) { * jobCtx.Thread = new Thread(StartBackgroundJobRun); * jobCtx.Thread.Start(jobCtx); * } else { * Task.Run(() => StartPoolJobRunAsync(jobCtx)); * } */ return(jobCtx); }
public IJobRun CreateRetryRun(IJobRun jobRun, int waitMinutes) { var session = EntityHelper.GetSession(jobRun); var baseTime = jobRun.EndedOn ?? jobRun.StartedOn ?? App.TimeService.UtcNow; var retryOn = baseTime.AddMinutes(waitMinutes); var retryRun = NewJobRun(jobRun.Job, JobRunType.Retry, retryOn, jobRun.DataId, jobRun.Data, jobRun.HostName); retryRun.AttemptNumber = jobRun.AttemptNumber + 1; retryRun.UserId = jobRun.UserId; retryRun.Status = JobRunStatus.Pending; return(retryRun); }
//internal Thread Thread; //background thread for long-running jobs internal JobRunContext(IJobRun jobRun) { var session = EntityHelper.GetSession(jobRun); var app = session.Context.App; UserInfo user = (jobRun.UserId == null) ? UserInfo.System : new UserInfo(jobRun.UserId.Value, null); OperationContext = new OperationContext(app, user); StartedOn = app.TimeService.UtcNow; JobName = jobRun.Job.Name; JobRunId = jobRun.Id; RunType = jobRun.RunType; AttemptNumber = jobRun.AttemptNumber; var job = jobRun.Job; JobId = job.Id; _progress = jobRun.Progress; IsPersisted = true; DataId = jobRun.DataId; Data = jobRun.Data; }
private JobStartInfo CreateJobStartInfo(IJobRun jobRun, JobRunContext jobContext) { var app = jobContext.App; var startInfo = new JobStartInfo(); var job = jobRun.Job; startInfo.ThreadType = job.ThreadType; startInfo.DeclaringType = ReflectionHelper.GetLoadedType(job.DeclaringType, throwIfNotFound: true); //it will throw if cannot find it var methName = job.MethodName; var argCount = job.MethodParameterCount; startInfo.Method = ReflectionHelper.FindMethod(startInfo.DeclaringType, job.MethodName, job.MethodParameterCount); // if Method is not static, it must be Module instance - this is a convention if (!startInfo.Method.IsStatic) { startInfo.Object = app.GetGlobalObject(startInfo.DeclaringType); //throws if not found. } //arguments var prms = startInfo.Method.GetParameters(); var serArgs = job.Arguments; var arrStrArgs = serArgs.Split(new[] { JsonArgsDelimiter }, StringSplitOptions.None); Util.Check(prms.Length == arrStrArgs.Length, "Serialized arg count ({0}) in job entity " + "does not match the number of target method parameters ({1}); target method: {2}.", arrStrArgs.Length, prms.Length, startInfo.Method.Name); // Deserialize arguments startInfo.Arguments = new object[arrStrArgs.Length]; for (int i = 0; i < arrStrArgs.Length; i++) { var paramType = prms[i].ParameterType; if (paramType == typeof(JobRunContext)) { startInfo.Arguments[i] = jobContext; } else { startInfo.Arguments[i] = Deserialize(paramType, arrStrArgs[i]); } } return(startInfo); }
private void OnJobScheduleSaving(IJobSchedule sched) { var session = EntityHelper.GetSession(sched); var job = sched.Job; var utcNow = session.Context.App.TimeService.UtcNow; var nextRunId = sched.NextRunId; IJobRun nextRun = (nextRunId == null) ? null : session.GetEntity <IJobRun>(nextRunId.Value); switch (sched.Status) { case JobScheduleStatus.Stopped: // if there is a pending run in the future, kill it if (nextRun != null && nextRun.Status == JobRunStatus.Pending && nextRun.StartOn > utcNow.AddMinutes(1)) { nextRun.Status = JobRunStatus.Killed; } break; case JobScheduleStatus.Active: // Create or adjust JobRun entity for next run var nextStartOn = sched.GetNextStartAfter(utcNow); if (nextStartOn != null) { if (nextRun == null || nextRun.Status != JobRunStatus.Pending) { nextRun = NewJobRun(job, JobRunType.Schedule, nextStartOn, hostName: sched.HostName); sched.NextRunId = nextRun.Id; } else { nextRun.StartOn = nextStartOn.Value; } } else { //nextSTartOn == null sched.NextRunId = null; } break; } //switch } //method
public void Enqueue(IJobRun jobRun) { _queuedRuns.Enqueue(jobRun); ProcessQueue(); }