public IStateMachine Create(IStorageConnection connection) { if (connection == null) throw new ArgumentNullException("connection"); var process = new DefaultStateChangeProcess(_handlers); return new StateMachine(connection, process); }
/// <summary> /// Initializes a new instance of the <see cref="BackgroundJobClient"/> class /// with a specified job storage and job creation process. /// </summary> /// /// <exception cref="ArgumentNullException"><paramref name="storage"/> argument is null.</exception> /// <exception cref="ArgumentNullException"><paramref name="process"/> argument is null.</exception> public BackgroundJobClient(JobStorage storage, IJobCreationProcess process) { if (storage == null) throw new ArgumentNullException("storage"); if (process == null) throw new ArgumentNullException("process"); _connection = storage.GetConnection(); _process = process; }
public void ElectState(IStorageConnection connection, ElectStateContext context) { var filterInfo = GetFilters(context.Job); foreach (var filter in filterInfo.ElectStateFilters) { filter.OnStateElection(context); } }
public StateMachine(IStorageConnection connection, IStateChangeProcess stateChangeProcess) { if (connection == null) throw new ArgumentNullException("connection"); if (stateChangeProcess == null) throw new ArgumentNullException("stateChangeProcess"); _connection = connection; _stateChangeProcess = stateChangeProcess; }
private static List<RecurringJobDto> GetRecurringJobDtos(IStorageConnection connection, IEnumerable<string> ids) { var result = new List<RecurringJobDto>(); foreach (var id in ids) { var hash = connection.GetAllEntriesFromHash(String.Format("recurring-job:{0}", id)); if (hash == null) { result.Add(new RecurringJobDto { Id = id, Removed = true }); continue; } var dto = new RecurringJobDto { Id = id }; dto.Cron = hash["Cron"]; try { var invocationData = JobHelper.FromJson<InvocationData>(hash["Job"]); dto.Job = invocationData.Deserialize(); } catch (JobLoadException ex) { dto.LoadException = ex; } if (hash.ContainsKey("NextExecution")) { dto.NextExecution = JobHelper.DeserializeDateTime(hash["NextExecution"]); } if (hash.ContainsKey("LastJobId") && !string.IsNullOrWhiteSpace(hash["LastJobId"])) { dto.LastJobId = hash["LastJobId"]; var stateData = connection.GetStateData(dto.LastJobId); if (stateData != null) { dto.LastJobState = stateData.Name; } } if (hash.ContainsKey("LastExecution")) { dto.LastExecution = JobHelper.DeserializeDateTime(hash["LastExecution"]); } if (hash.ContainsKey("TimeZoneId")) { dto.TimeZoneId = hash["TimeZoneId"]; } result.Add(dto); } return result; }
public CreateContext(IStorageConnection connection, Job job, IState initialState) { if (connection == null) throw new ArgumentNullException("connection"); if (job == null) throw new ArgumentNullException("job"); Connection = connection; Job = job; InitialState = initialState; Items = new Dictionary<string, object>(); }
public StateContext(string jobId, Job job, DateTime createdAt, IStorageConnection connection) { if (connection == null) throw new ArgumentNullException("connection"); if (String.IsNullOrEmpty(jobId)) throw new ArgumentNullException("jobId"); JobId = jobId; Job = job; CreatedAt = createdAt; Connection = connection; }
internal StateMachine( IStorageConnection connection, IEnumerable<StateHandler> handlers, IEnumerable<object> filters) : this(connection) { if (handlers == null) throw new ArgumentNullException("handlers"); if (filters == null) throw new ArgumentNullException("filters"); _getFiltersThunk = md => filters.Select(f => new JobFilter(f, JobFilterScope.Type, null)); _getHandlersThunk = c => handlers; }
public ApplyStateContext( IStorageConnection connection, StateContext context, State newState, string oldStateName) : base(context) { if (connection == null) throw new ArgumentNullException("connection"); _connection = connection; OldStateName = oldStateName; NewState = newState; JobExpirationTimeout = TimeSpan.FromDays(1); }
public ServerJobCancellationToken( [NotNull] string jobId, [NotNull] IStorageConnection connection, [NotNull] WorkerContext workerContext, CancellationToken shutdownToken) { if (jobId == null) throw new ArgumentNullException("jobId"); if (connection == null) throw new ArgumentNullException("connection"); if (workerContext == null) throw new ArgumentNullException("workerContext"); _jobId = jobId; _shutdownToken = shutdownToken; _connection = connection; _workerContext = workerContext; }
public ServerJobCancellationToken( [NotNull] IStorageConnection connection, [NotNull] string jobId, [NotNull] string serverId, [NotNull] string workerId, CancellationToken shutdownToken) { if (jobId == null) throw new ArgumentNullException(nameof(jobId)); if (serverId == null) throw new ArgumentNullException(nameof(serverId)); if (workerId == null) throw new ArgumentNullException(nameof(workerId)); if (connection == null) throw new ArgumentNullException(nameof(connection)); _jobId = jobId; _serverId = serverId; _workerId = workerId; _connection = connection; _shutdownToken = shutdownToken; }
internal ApplyStateContext( [NotNull] JobStorage storage, [NotNull] IStorageConnection connection, [NotNull] IWriteOnlyTransaction transaction, [NotNull] BackgroundJob backgroundJob, [NotNull] IState newState, [CanBeNull] string oldStateName, [NotNull] IProfiler profiler) { if (storage == null) { throw new ArgumentNullException(nameof(storage)); } if (connection == null) { throw new ArgumentNullException(nameof(connection)); } if (transaction == null) { throw new ArgumentNullException(nameof(transaction)); } if (backgroundJob == null) { throw new ArgumentNullException(nameof(backgroundJob)); } if (newState == null) { throw new ArgumentNullException(nameof(newState)); } BackgroundJob = backgroundJob; Storage = storage; Connection = connection; Transaction = transaction; OldStateName = oldStateName; NewState = newState; JobExpirationTimeout = storage.JobExpirationTimeout; Profiler = profiler; }
/// <summary> /// Initializes a new instance of the <see cref="BackgroundJobClient"/> class /// with a specified job storage and job creation process. /// </summary> /// /// <exception cref="ArgumentNullException"><paramref name="storage"/> argument is null.</exception> /// <exception cref="ArgumentNullException"><paramref name="process"/> argument is null.</exception> public BackgroundJobClient( JobStorage storage, IStateMachineFactory stateMachineFactory, IJobCreationProcess process) { if (storage == null) { throw new ArgumentNullException("storage"); } if (stateMachineFactory == null) { throw new ArgumentNullException("stateMachineFactory"); } if (process == null) { throw new ArgumentNullException("process"); } _connection = storage.GetConnection(); _stateMachineFactory = stateMachineFactory; _process = process; }
private bool AddFingerprintIfNotExists(IStorageConnection connection, Job job) { using var lck = connection.AcquireDistributedLock(GetFingerprintLockKey(job), LockTimeout); string fingerprintKey = GetFingerprintKey(job); Dictionary <string, string> fingerprint = connection.GetAllEntriesFromHash(fingerprintKey); if (fingerprint != null) { if (fingerprint.ContainsKey("Timestamp") && DateTimeOffset.TryParse(fingerprint["Timestamp"], null, DateTimeStyles.RoundtripKind, out DateTimeOffset timestamp)) { if (this.FingerprintTimeoutMinutes > 0) { var timestampWithTimeout = timestamp.Add(TimeSpan.FromMinutes(FingerprintTimeoutMinutes)); if (DateTimeOffset.UtcNow <= timestampWithTimeout) { return(false); } } else { return(false); } } } // Fingerprint does not exist, it is invalid (no `Timestamp` key), // or it is not actual (timeout expired). connection.SetRangeInHash(fingerprintKey, new Dictionary <string, string> { { "Timestamp", DateTimeOffset.UtcNow.ToString("o") }, { "MethodName", job.Method.Name }, { "TypeName", job.Type.Name } }); return(true); }
internal JobActivatorContext( [NotNull] IStorageConnection connection, [NotNull] BackgroundJob backgroundJob, [NotNull] IJobCancellationToken cancellationToken) { if (connection == null) { throw new ArgumentNullException(nameof(connection)); } if (backgroundJob == null) { throw new ArgumentNullException(nameof(backgroundJob)); } if (cancellationToken == null) { throw new ArgumentNullException(nameof(cancellationToken)); } Connection = connection; BackgroundJob = backgroundJob; CancellationToken = cancellationToken; }
internal StateChangeContext( [NotNull] JobStorage storage, [NotNull] IStorageConnection connection, [NotNull] string backgroundJobId, [NotNull] IState newState, [CanBeNull] IEnumerable <string> expectedStates, CancellationToken cancellationToken, [NotNull] IProfiler profiler) { if (storage == null) { throw new ArgumentNullException(nameof(storage)); } if (connection == null) { throw new ArgumentNullException(nameof(connection)); } if (backgroundJobId == null) { throw new ArgumentNullException(nameof(backgroundJobId)); } if (newState == null) { throw new ArgumentNullException(nameof(newState)); } if (profiler == null) { throw new ArgumentNullException(nameof(profiler)); } Storage = storage; Connection = connection; BackgroundJobId = backgroundJobId; NewState = newState; ExpectedStates = expectedStates; CancellationToken = cancellationToken; Profiler = profiler; }
private void TryScheduleJob(IStorageConnection connection, string recurringJobId, Dictionary <string, string> recurringJob) { var serializedJob = JobHelper.FromJson <InvocationData>(recurringJob["Job"]); var job = serializedJob.Deserialize(); var cron = recurringJob["Cron"]; var cronSchedule = CrontabSchedule.Parse(cron); var instant = _instantFactory.GetInstant(cronSchedule); var lastExecutionTime = recurringJob.ContainsKey("LastExecution") ? JobHelper.DeserializeDateTime(recurringJob["LastExecution"]) : (DateTime?)null; if (instant.GetMatches(lastExecutionTime).Any()) { var state = new EnqueuedState { Reason = "Triggered by recurring job scheduler" }; var jobId = _client.Create(job, state); connection.SetRangeInHash( String.Format("recurring-job:{0}", recurringJobId), new Dictionary <string, string> { { "LastExecution", JobHelper.SerializeDateTime(instant.UtcTime) }, { "LastJobId", jobId }, }); } connection.SetRangeInHash( String.Format("recurring-job:{0}", recurringJobId), new Dictionary <string, string> { { "NextExecution", JobHelper.SerializeDateTime(instant.NextOccurrence) } }); }
/// <summary> /// /// </summary> /// <param name="storage"></param> /// <param name="connection"></param> /// <param name="transaction"></param> /// <param name="backgroundJob"></param> /// <param name="newState">新状态</param> /// <param name="oldStateName"></param> public ApplyStateContext( [NotNull] JobStorage storage, [NotNull] IStorageConnection connection, [NotNull] IWriteOnlyTransaction transaction, [NotNull] BackgroundJob backgroundJob, [NotNull] IState newState, [CanBeNull] string oldStateName) { if (storage == null) { throw new ArgumentNullException(nameof(storage)); } if (connection == null) { throw new ArgumentNullException(nameof(connection)); } if (transaction == null) { throw new ArgumentNullException(nameof(transaction)); } if (backgroundJob == null) { throw new ArgumentNullException(nameof(backgroundJob)); } if (newState == null) { throw new ArgumentNullException(nameof(newState)); } BackgroundJob = backgroundJob; Storage = storage; Connection = connection; Transaction = transaction; OldStateName = oldStateName; NewState = newState; JobExpirationTimeout = TimeSpan.FromDays(1); }
public virtual async Task SyncCronJobsCoreAsync( IStorageConnection connection, CronJobRegistry.Entry[] entries, CronJob[] currentJobs) { if (entries.Length != 0) { // Add or update jobs foreach (var entry in entries) { var cronJob = currentJobs.FirstOrDefault(j => j.Name == entry.Name); var updating = cronJob != null; if (!updating) { cronJob = new CronJob { Name = entry.Name, TypeName = entry.JobType.AssemblyQualifiedName, Cron = entry.Cron, LastRun = DateTime.MinValue }; await connection.StoreCronJobAsync(cronJob); } else { cronJob.TypeName = entry.JobType.AssemblyQualifiedName; cronJob.Cron = entry.Cron; await connection.UpdateCronJobAsync(cronJob); } } } // Delete old jobs foreach (var oldJob in currentJobs.Where(j => !entries.Any(e => e.Name == j.Name))) { await connection.RemoveCronJobAsync(oldJob.Name); } }
public static void SetTrigger(this IStorageConnection connection, string triggerName) { var jsc = (JobStorageConnection)connection; var triggerKey = GenerateTriggerKey(triggerName); var jobIds = jsc.GetAllItemsFromList(triggerKey); var client = new BackgroundJobClient(); try { using (connection.AcquireDistributedLock(triggerKey, TimeSpan.Zero)) { foreach (var jobId in jobIds) { client.ChangeState(jobId, new EnqueuedState()); } } } catch (DistributedLockTimeoutException) { // Assume already run } }
public virtual JobInfo GetJobInfo(string jobId, CancellationToken cancellationToken) { if (jobId == null) { throw new ArgumentNullException(nameof(jobId)); } using IStorageConnection connection = JobStorage.Current.GetConnection(); JobData jobData = connection.GetJobData(jobId); if (jobData == null) { throw new AppException(BitMetadataBuilder.JobNotFound); } return(new JobInfo { Id = jobId, CreatedAt = jobData.CreatedAt, State = jobData.State }); }
public ServerJobCancellationToken( [NotNull] IStorageConnection connection, [NotNull] string jobId, [NotNull] string serverId, [NotNull] string workerId, CancellationToken shutdownToken) { _jobId = jobId ?? throw new ArgumentNullException(nameof(jobId)); _serverId = serverId ?? throw new ArgumentNullException(nameof(serverId)); _workerId = workerId ?? throw new ArgumentNullException(nameof(workerId)); _connection = connection ?? throw new ArgumentNullException(nameof(connection)); _shutdownToken = shutdownToken; _cancellationTokenHolder = new Lazy <CancellationTokenHolder>( () => new CancellationTokenHolder(_shutdownToken), LazyThreadSafetyMode.None); if (WatchedServers.TryGetValue(_serverId, out _watchedTokens)) { _watchedTokens.TryAdd(this, null); } }
internal PerformContext( [NotNull] IStorageConnection connection, [NotNull] BackgroundJob backgroundJob, [NotNull] IJobCancellationToken cancellationToken, [NotNull] IProfiler profiler, [NotNull] JobActivatorScope scope ) { if (connection == null) { throw new ArgumentNullException(nameof(connection)); } if (backgroundJob == null) { throw new ArgumentNullException(nameof(backgroundJob)); } if (cancellationToken == null) { throw new ArgumentNullException(nameof(cancellationToken)); } if (profiler == null) { throw new ArgumentNullException(nameof(profiler)); } if (scope == null) { throw new ArgumentNullException(nameof(scope)); } Connection = connection; BackgroundJob = backgroundJob; CancellationToken = cancellationToken; Profiler = profiler; _scope = scope; Items = new Dictionary <string, object>(); }
public static IEnumerable <Tuple <string, string> > CheckAllCancellationTokens( string serverId, IStorageConnection connection, CancellationToken cancellationToken) { if (WatchedServers.TryGetValue(serverId, out var watchedTokens)) { var result = new List <Tuple <string, string> >(); foreach (var token in watchedTokens) { cancellationToken.ThrowIfCancellationRequested(); if (token.Key.TryCheckJobIsAborted(connection)) { result.Add(Tuple.Create(token.Key._jobId, token.Key._workerId)); } } return(result); } return(Enumerable.Empty <Tuple <string, string> >()); }
public virtual Task <JobInfo> GetJobInfoAsync(string jobId, CancellationToken cancellationToken) { if (jobId == null) { throw new ArgumentNullException(nameof(jobId)); } using (IStorageConnection connection = JobStorage.Current.GetConnection()) { JobData jobData = connection.GetJobData(jobId); if (jobData == null) { throw new AppException(FoundationMetadataBuilder.JobNotFound); } return(Task.FromResult(new JobInfo { Id = jobId, CreatedAt = jobData.CreatedAt, State = jobData.State })); } }
private void ApplyAuditLog(IStorageConnection auditContext, DbContext workingContext, DbEntityEntry entry) { LogOperation operation; switch (entry.State) { case EntityState.Added: operation = LogOperation.Create; break; case EntityState.Deleted: operation = LogOperation.Delete; break; case EntityState.Modified: operation = LogOperation.Update; break; default: throw new ArgumentOutOfRangeException(); } ApplyAuditLog(auditContext, workingContext, entry, operation); }
public ServerJobCancellationToken( [NotNull] string jobId, [NotNull] IStorageConnection connection, [NotNull] WorkerContext workerContext, CancellationToken shutdownToken) { if (jobId == null) { throw new ArgumentNullException("jobId"); } if (connection == null) { throw new ArgumentNullException("connection"); } if (workerContext == null) { throw new ArgumentNullException("workerContext"); } _jobId = jobId; _shutdownToken = shutdownToken; _connection = connection; _workerContext = workerContext; }
public PerformContext( [NotNull] IStorageConnection connection, [NotNull] BackgroundJob backgroundJob, [NotNull] IJobCancellationToken cancellationToken) { if (connection == null) { throw new ArgumentNullException(nameof(connection)); } if (backgroundJob == null) { throw new ArgumentNullException(nameof(backgroundJob)); } if (cancellationToken == null) { throw new ArgumentNullException(nameof(cancellationToken)); } Connection = connection; BackgroundJob = backgroundJob; CancellationToken = cancellationToken; Items = new Dictionary <string, object>(); }
private void TryScheduleJob( JobStorage storage, IStorageConnection connection, string recurringJobId, IReadOnlyDictionary<string, string> recurringJob) { var serializedJob = JobHelper.FromJson<InvocationData>(recurringJob["Job"]); var job = serializedJob.Deserialize(); var cron = recurringJob["Cron"]; var cronSchedule = CrontabSchedule.Parse(cron); try { var timeZone = recurringJob.ContainsKey("TimeZoneId") ? TimeZoneInfo.FindSystemTimeZoneById(recurringJob["TimeZoneId"]) : TimeZoneInfo.Utc; var instant = _instantFactory(cronSchedule, timeZone); var lastExecutionTime = recurringJob.ContainsKey("LastExecution") ? JobHelper.DeserializeDateTime(recurringJob["LastExecution"]) : (DateTime?)null; var changedFields = new Dictionary<string, string>(); const string lastJobId = "LastJobId"; if (recurringJob.ContainsKey(lastJobId) && !string.IsNullOrWhiteSpace(recurringJob[lastJobId])) { var jobDetails = storage.GetMonitoringApi().JobDetails(recurringJob[lastJobId]); var finalStates = new[] {"Succeeded", "Deleted"}; if (!jobDetails.History.Select(x => x.StateName).Any(finalStates.Contains)) { Logger.Info("The recurring task " + recurringJobId + " is still running, do not schedule it more. "); return; } } if (instant.GetNextInstants(lastExecutionTime).Any()) { var state = new EnqueuedState { Reason = "Triggered by recurring job scheduler" }; if (recurringJob.ContainsKey("Queue") && !String.IsNullOrEmpty(recurringJob["Queue"])) { state.Queue = recurringJob["Queue"]; } var backgroundJob = _factory.Create(new CreateContext(storage, connection, job, state)); var jobId = backgroundJob != null ? backgroundJob.Id : null; if (String.IsNullOrEmpty(jobId)) { Logger.DebugFormat( "Recurring job '{0}' execution at '{1}' has been canceled.", recurringJobId, instant.NowInstant); } changedFields.Add("LastExecution", JobHelper.SerializeDateTime(instant.NowInstant)); changedFields.Add(lastJobId, jobId ?? String.Empty); } changedFields.Add("NextExecution", JobHelper.SerializeDateTime(instant.NextInstant)); connection.SetRangeInHash( String.Format("recurring-job:{0}", recurringJobId), changedFields); } catch (TimeZoneNotFoundException ex) { Logger.ErrorException( String.Format("Recurring job '{0}' was not triggered: {1}.", recurringJobId, ex.Message), ex); } }
private static List <RecurringJobDto> GetRecurringJobDtos(IStorageConnection connection, IEnumerable <string> ids) { var result = new List <RecurringJobDto>(); foreach (var id in ids) { var hash = connection.GetAllEntriesFromHash(String.Format("recurring-job:{0}", id)); if (hash == null) { result.Add(new RecurringJobDto { Id = id, Removed = true }); continue; } var dto = new RecurringJobDto { Id = id }; dto.Cron = hash["Cron"]; try { var invocationData = JobHelper.FromJson <InvocationData>(hash["Job"]); dto.Job = invocationData.Deserialize(); } catch (JobLoadException ex) { dto.LoadException = ex; } if (hash.ContainsKey("NextExecution")) { dto.NextExecution = JobHelper.DeserializeDateTime(hash["NextExecution"]); } if (hash.ContainsKey("LastJobId") && !string.IsNullOrWhiteSpace(hash["LastJobId"])) { dto.LastJobId = hash["LastJobId"]; var stateData = connection.GetStateData(dto.LastJobId); if (stateData != null) { dto.LastJobState = stateData.Name; } } if (hash.ContainsKey("LastExecution")) { dto.LastExecution = JobHelper.DeserializeDateTime(hash["LastExecution"]); } if (hash.ContainsKey("TimeZoneId")) { dto.TimeZoneId = hash["TimeZoneId"]; } if (hash.ContainsKey("CreatedAt")) { dto.CreatedAt = JobHelper.DeserializeDateTime(hash["CreatedAt"]); } result.Add(dto); } return(result); }
private void TryScheduleJob(IStorageConnection connection, string recurringJobId, Dictionary<string, string> recurringJob) { var serializedJob = JobHelper.FromJson<InvocationData>(recurringJob["Job"]); var job = serializedJob.Deserialize(); var cron = recurringJob["Cron"]; var parts = cron.Split(" ".ToCharArray(), StringSplitOptions.RemoveEmptyEntries); var cronSchedule = CrontabSchedule.Parse(cron, new CrontabSchedule.ParseOptions { IncludingSeconds = (parts.Length >= 6) }); try { var timeZone = recurringJob.ContainsKey("TimeZoneId") ? TimeZoneInfo.FindSystemTimeZoneById(recurringJob["TimeZoneId"]) : TimeZoneInfo.Utc; var instant = _instantFactory.GetInstant(cronSchedule, timeZone); var lastExecutionTime = recurringJob.ContainsKey("LastExecution") ? JobHelper.DeserializeDateTime(recurringJob["LastExecution"]) : (DateTime?)null; var changedFields = new Dictionary<string, string>(); if (instant.GetNextInstants(lastExecutionTime).Any()) { var state = new EnqueuedState { Reason = "Triggered by recurring job scheduler" }; var jobId = _client.Create(job, state); if (String.IsNullOrEmpty(jobId)) { Logger.DebugFormat( "Recurring job '{0}' execution at '{1}' has been canceled.", recurringJobId, instant.NowInstant); } changedFields.Add("LastExecution", JobHelper.SerializeDateTime(instant.NowInstant)); changedFields.Add("LastJobId", jobId ?? String.Empty); } changedFields.Add("NextExecution", JobHelper.SerializeDateTime(instant.NextInstant)); connection.SetRangeInHash( String.Format("recurring-job:{0}", recurringJobId), changedFields); } catch (TimeZoneNotFoundException ex) { Logger.ErrorException( String.Format("Recurring job '{0}' was not triggered: {1}.", recurringJobId, ex.Message), ex); } }
private static void SetContinuations( IStorageConnection connection, string jobId, List<Continuation> continuations) { connection.SetJobParameter(jobId, "Continuations", JobHelper.ToJson(continuations)); }
private void TryScheduleJob( JobStorage storage, IStorageConnection connection, string recurringJobId, IReadOnlyDictionary<string, string> recurringJob) { var serializedJob = JobHelper.FromJson<InvocationData>(recurringJob["Job"]); var job = serializedJob.Deserialize(); var cron = recurringJob["Cron"]; var cronSchedule = CrontabSchedule.Parse(cron); try { var timeZone = recurringJob.ContainsKey("TimeZoneId") ? TimeZoneInfo.FindSystemTimeZoneById(recurringJob["TimeZoneId"]) : TimeZoneInfo.Utc; var nowInstant = _instantFactory(cronSchedule, timeZone); var changedFields = new Dictionary<string, string>(); var lastInstant = GetLastInstant(recurringJob, nowInstant); if (nowInstant.GetNextInstants(lastInstant).Any()) { var state = new EnqueuedState { Reason = "Triggered by recurring job scheduler" }; if (recurringJob.ContainsKey("Queue") && !String.IsNullOrEmpty(recurringJob["Queue"])) { state.Queue = recurringJob["Queue"]; } var backgroundJob = _factory.Create(new CreateContext(storage, connection, job, state)); var jobId = backgroundJob != null ? backgroundJob.Id : null; if (String.IsNullOrEmpty(jobId)) { Logger.DebugFormat( "Recurring job '{0}' execution at '{1}' has been canceled.", recurringJobId, nowInstant.NowInstant); } changedFields.Add("LastExecution", JobHelper.SerializeDateTime(nowInstant.NowInstant)); changedFields.Add("LastJobId", jobId ?? String.Empty); } // Fixing old recurring jobs that doesn't have the CreatedAt field if (!recurringJob.ContainsKey("CreatedAt")) { changedFields.Add("CreatedAt", JobHelper.SerializeDateTime(nowInstant.NowInstant)); } changedFields.Add("NextExecution", nowInstant.NextInstant.HasValue ? JobHelper.SerializeDateTime(nowInstant.NextInstant.Value) : null); connection.SetRangeInHash( String.Format("recurring-job:{0}", recurringJobId), changedFields); } catch (TimeZoneNotFoundException ex) { Logger.ErrorException( String.Format("Recurring job '{0}' was not triggered: {1}.", recurringJobId, ex.Message), ex); } }
public JobContext(string jobId, IStorageConnection connection) { this.jobId = jobId; }
private IState PerformJob(string jobId, IStorageConnection connection, IJobCancellationToken token) { try { var jobData = connection.GetJobData(jobId); if (jobData == null) { // Job expired just after moving to a processing state. This is an // unreal scenario, but shit happens. Returning null instead of throwing // an exception and rescuing from en-queueing a poisoned jobId back // to a queue. return null; } jobData.EnsureLoaded(); var performContext = new PerformContext( _context, connection, jobId, jobData.Job, jobData.CreatedAt, token); var latency = (DateTime.UtcNow - jobData.CreatedAt).TotalMilliseconds; var duration = Stopwatch.StartNew(); var result = _process.Run(performContext, jobData.Job); duration.Stop(); return new SucceededState(result, (long) latency, duration.ElapsedMilliseconds); } catch (OperationCanceledException) { throw; } catch (JobPerformanceException ex) { return new FailedState(ex.InnerException) { Reason = ex.Message }; } catch (Exception ex) { return new FailedState(ex) { Reason = "Internal Hangfire Server exception occurred. Please, report it to Hangfire developers." }; } }
private static List <Continuation> GetContinuations(IStorageConnection connection, string jobId) { return(JobHelper.FromJson <List <Continuation> >(connection.GetJobParameter( jobId, "Continuations")) ?? new List <Continuation>()); }
private IState PerformJob(BackgroundProcessContext context, IStorageConnection connection, string jobId) { try { var jobData = connection.GetJobData(jobId); if (jobData == null) { // Job expired just after moving to a processing state. This is an // unreal scenario, but shit happens. Returning null instead of throwing // an exception and rescuing from en-queueing a poisoned jobId back // to a queue. return null; } jobData.EnsureLoaded(); var backgroundJob = new BackgroundJob(jobId, jobData.Job, jobData.CreatedAt); var jobToken = new ServerJobCancellationToken(connection, jobId, context.ServerId, _workerId, context.CancellationToken); var performContext = new PerformContext(connection, backgroundJob, jobToken); var latency = (DateTime.UtcNow - jobData.CreatedAt).TotalMilliseconds; var duration = Stopwatch.StartNew(); var result = _performer.Perform(performContext); duration.Stop(); return new SucceededState(result, (long) latency, duration.ElapsedMilliseconds); } catch (OperationCanceledException) { throw; } catch (JobPerformanceException ex) { return new FailedState(ex.InnerException) { Reason = ex.Message }; } catch (Exception ex) { return new FailedState(ex) { Reason = "An exception occurred during processing of a background job." }; } }
public StateMachine(IStorageConnection connection) { if (connection == null) throw new ArgumentNullException("connection"); _connection = connection; }
private IDisposable AcquireDistributedSetLock(IStorageConnection connection, IEnumerable <object> args) { return(connection.AcquireDistributedLock(GetDistributedLockKey(args), DistributedLockTimeout)); }
private void PerformJob(IStorageConnection connection, JobPayload payload) { if (payload == null) { return; } var stateMachine = new StateMachine(connection); var processingState = new ProcessingState(_context.ServerName); if (!stateMachine.TryToChangeState( payload.Id, processingState, new [] { EnqueuedState.StateName, ProcessingState.StateName })) { return; } // Checkpoint #3. Job is in the Processing state. However, there are // no guarantees that it was performed. We need to re-queue it even // it was performed to guarantee that it was performed AT LEAST once. // It will be re-queued after the JobTimeout was expired. State state; try { IJobPerformStrategy performStrategy; var methodData = MethodData.Deserialize(payload.InvocationData); if (methodData.OldFormat) { // For compatibility with the Old Client API. // TODO: remove it in version 1.0 var arguments = JobHelper.FromJson<Dictionary<string, string>>( payload.Args); performStrategy = new JobAsClassPerformStrategy( methodData, arguments); } else { var arguments = JobHelper.FromJson<string[]>(payload.Arguments); performStrategy = new JobAsMethodPerformStrategy( methodData, arguments); } var performContext = new PerformContext(_context, connection, payload.Id, methodData); _context.PerformancePipeline.Run(performContext, performStrategy); state = new SucceededState(); } catch (JobPerformanceException ex) { state = new FailedState(ex.InnerException) { Reason = ex.Message }; } catch (Exception ex) { state = new FailedState(ex) { Reason = "Internal HangFire Server exception occurred. Please, report it to HangFire developers." }; } // TODO: check return value stateMachine.TryToChangeState(payload.Id, state, new [] { ProcessingState.StateName }); }
private static void SetContinuations( IStorageConnection connection, string jobId, List <Continuation> continuations) { connection.SetJobParameter(jobId, "Continuations", JobHelper.ToJson(continuations)); }
public SqLiteUnitOfWork(IStorageConnection connection) { _connection = connection; }
public static List <RecurringJobDto> GetRecurringJobs([NotNull] this IStorageConnection connection) { if (connection == null) { throw new ArgumentNullException("connection"); } var result = new List <RecurringJobDto>(); var ids = connection.GetAllItemsFromSet("recurring-jobs"); foreach (var id in ids) { var hash = connection.GetAllEntriesFromHash(String.Format("recurring-job:{0}", id)); if (hash == null) { result.Add(new RecurringJobDto { Id = id, Removed = true }); continue; } var dto = new RecurringJobDto { Id = id }; dto.Cron = hash["Cron"]; try { var invocationData = JobHelper.FromJson <InvocationData>(hash["Job"]); dto.Job = invocationData.Deserialize(); } catch (JobLoadException ex) { dto.LoadException = ex; } if (hash.ContainsKey("NextExecution")) { dto.NextExecution = JobHelper.DeserializeDateTime(hash["NextExecution"]); } if (hash.ContainsKey("LastJobId") && !string.IsNullOrWhiteSpace(hash["LastJobId"])) { dto.LastJobId = hash["LastJobId"]; var stateData = connection.GetStateData(dto.LastJobId); if (stateData != null) { dto.LastJobState = stateData.Name; } } if (hash.ContainsKey("LastExecution")) { dto.LastExecution = JobHelper.DeserializeDateTime(hash["LastExecution"]); } result.Add(dto); } return(result); }
private void TryScheduleJob(IStorageConnection connection, string recurringJobId, Dictionary<string, string> recurringJob) { var serializedJob = JobHelper.FromJson<InvocationData>(recurringJob["Job"]); var job = serializedJob.Deserialize(); var cron = recurringJob["Cron"]; var cronSchedule = CrontabSchedule.Parse(cron); var currentTime = _dateTimeProvider.CurrentDateTime; if (recurringJob.ContainsKey("NextExecution")) { var nextExecution = JobHelper.DeserializeDateTime(recurringJob["NextExecution"]); if (nextExecution <= currentTime) { var state = new EnqueuedState { Reason = "Triggered by recurring job scheduler" }; var jobId = _client.Create(job, state); connection.SetRangeInHash( String.Format("recurring-job:{0}", recurringJobId), new Dictionary<string, string> { { "LastExecution", JobHelper.SerializeDateTime(currentTime) }, { "LastJobId", jobId }, { "NextExecution", JobHelper.SerializeDateTime(_dateTimeProvider.GetNextOccurrence(cronSchedule)) } }); } } else { var nextExecution = _dateTimeProvider.GetNextOccurrence(cronSchedule); connection.SetRangeInHash( String.Format("recurring-job:{0}", recurringJobId), new Dictionary<string, string> { { "NextExecution", JobHelper.SerializeDateTime(nextExecution) } }); } }
private async Task ProcessPublishedAsync(IStorageConnection connection, ProcessingContext context) { var messages = await connection.GetPublishedMessagesOfNeedRetry(); var hasException = false; foreach (var message in messages) { if (message.Retries > _options.FailedRetryCount) { continue; } using (var transaction = connection.CreateTransaction()) { var result = await _publishExecutor.PublishAsync(message.Name, message.Content); if (result.Succeeded) { _stateChanger.ChangeState(message, new SucceededState(), transaction); _logger.LogInformation("The message was sent successfully during the retry. MessageId:" + message.Id); } else { message.Content = Helper.AddExceptionProperty(message.Content, result.Exception); message.Retries++; if (message.StatusName == StatusName.Scheduled) { message.ExpiresAt = GetDueTime(message.Added, message.Retries); message.StatusName = StatusName.Failed; } transaction.UpdateMessage(message); if (message.Retries >= _options.FailedRetryCount) { _logger.LogError($"The message still sent failed after {_options.FailedRetryCount} retries. We will stop retrying the message. " + "MessageId:" + message.Id); if (message.Retries == _options.FailedRetryCount) { if (!hasException) { try { _options.FailedThresholdCallback?.Invoke(MessageType.Publish, message.Name, message.Content); } catch (Exception ex) { hasException = true; _logger.LogWarning("Failed call-back method raised an exception:" + ex.Message); } } } } } await transaction.CommitAsync(); } context.ThrowIfStopping(); await context.WaitAsync(_delay); } }
private void TryScheduleJob(IStorageConnection connection, string recurringJobId, Dictionary<string, string> recurringJob) { var serializedJob = JobHelper.FromJson<InvocationData>(recurringJob["Job"]); var job = serializedJob.Deserialize(); var cron = recurringJob["Cron"]; var cronSchedule = CrontabSchedule.Parse(cron); var instant = _instantFactory.GetInstant(cronSchedule); var lastExecutionTime = recurringJob.ContainsKey("LastExecution") ? JobHelper.DeserializeDateTime(recurringJob["LastExecution"]) : (DateTime?)null; if (instant.GetMatches(lastExecutionTime).Any()) { var state = new EnqueuedState { Reason = "Triggered by recurring job scheduler" }; var jobId = _client.Create(job, state); connection.SetRangeInHash( String.Format("recurring-job:{0}", recurringJobId), new Dictionary<string, string> { { "LastExecution", JobHelper.SerializeDateTime(instant.UtcTime) }, { "LastJobId", jobId }, }); } connection.SetRangeInHash( String.Format("recurring-job:{0}", recurringJobId), new Dictionary<string, string> { { "NextExecution", JobHelper.SerializeDateTime(instant.NextOccurrence) } }); }
public GetJobDispatcher() { _connection = JobStorage.Current.GetConnection(); }
private bool EnqueueBackgroundJob( BackgroundProcessContext context, IStorageConnection connection, string recurringJobId, DateTime now) { using (connection.AcquireDistributedRecurringJobLock(recurringJobId, LockTimeout)) { try { var recurringJob = connection.GetRecurringJob(recurringJobId, _timeZoneResolver, now); if (recurringJob == null) { using (var transaction = connection.CreateWriteTransaction()) { transaction.RemoveFromSet("recurring-jobs", recurringJobId); transaction.Commit(); } return(false); } // If a recurring job has the "V" field, then it was created by a newer // version. Despite we can handle 1.7.0-based recurring jobs just fine, // future versions may introduce new features anyway, so it's safer to // let other servers to handle this recurring job. if (recurringJob.Version.HasValue && recurringJob.Version > 2) { return(false); } BackgroundJob backgroundJob = null; IReadOnlyDictionary <string, string> changedFields; if (recurringJob.TrySchedule(out var nextExecution, out var error)) { if (nextExecution.HasValue && nextExecution <= now) { backgroundJob = _factory.TriggerRecurringJob(context.Storage, connection, _profiler, recurringJob, now); if (String.IsNullOrEmpty(backgroundJob?.Id)) { _logger.Debug($"Recurring job '{recurringJobId}' execution at '{nextExecution}' has been canceled."); } } recurringJob.IsChanged(out changedFields, out nextExecution); } else if (recurringJob.RetryAttempt < MaxRetryAttemptCount) { var delay = _pollingDelay > TimeSpan.Zero ? _pollingDelay : TimeSpan.FromMinutes(1); _logger.WarnException($"Recurring job '{recurringJobId}' can't be scheduled due to an error and will be retried in {delay}.", error); recurringJob.ScheduleRetry(delay, out changedFields, out nextExecution); } else { _logger.ErrorException($"Recurring job '{recurringJobId}' can't be scheduled due to an error and will be disabled.", error); recurringJob.Disable(error, out changedFields, out nextExecution); } // We always start a transaction, regardless our recurring job was updated or not, // to prevent from infinite loop, when there's an old processing server (pre-1.7.0) // in our environment that doesn't know it should modify the score for entries in // the recurring jobs set. using (var transaction = connection.CreateWriteTransaction()) { if (backgroundJob != null) { _factory.StateMachine.EnqueueBackgroundJob( context.Storage, connection, transaction, recurringJob, backgroundJob, "Triggered by recurring job scheduler", _profiler); } transaction.UpdateRecurringJob(recurringJob, changedFields, nextExecution, _logger); transaction.Commit(); return(true); } } #if !NETSTANDARD1_3 catch (TimeZoneNotFoundException ex) { #else catch (Exception ex) { // https://github.com/dotnet/corefx/issues/7552 if (!ex.GetType().Name.Equals("TimeZoneNotFoundException")) { throw; } #endif _logger.ErrorException( $"Recurring job '{recurringJobId}' was not triggered: {ex.Message}.", ex); } return(false); } }
private async Task <bool> UpdateMessageForRetryAsync(CapPublishedMessage message, IStorageConnection connection) { var retryBehavior = RetryBehavior.DefaultRetry; var now = DateTime.Now; var retries = ++message.Retries; if (retries >= retryBehavior.RetryCount) { return(false); } var due = message.Added.AddSeconds(retryBehavior.RetryIn(retries)); message.ExpiresAt = due; using (var transaction = connection.CreateTransaction()) { transaction.UpdateMessage(message); await transaction.CommitAsync(); } return(true); }
private void EnqueueBackgroundJob(BackgroundProcessContext context, IStorageConnection connection, string jobId) { Exception exception = null; for (var retryAttempt = 0; retryAttempt < MaxStateChangeAttempts; retryAttempt++) { try { var appliedState = _stateChanger.ChangeState(new StateChangeContext( context.Storage, connection, jobId, new EnqueuedState { Reason = $"Triggered by {ToString()}" }, new [] { ScheduledState.StateName }, disableFilters: false, context.StoppingToken, _profiler)); if (appliedState == null && connection.GetJobData(jobId) == null) { // When a background job with the given id does not exist, we should // remove its id from a schedule manually. This may happen when someone // modifies a storage bypassing Hangfire API. using (var transaction = connection.CreateWriteTransaction()) { transaction.RemoveFromSet("schedule", jobId); transaction.Commit(); } } return; } catch (Exception ex) { _logger.DebugException( $"State change attempt {retryAttempt + 1} of {MaxStateChangeAttempts} failed due to an error, see inner exception for details", ex); exception = ex; } context.StoppingToken.Wait(TimeSpan.FromSeconds(retryAttempt)); context.StoppingToken.ThrowIfCancellationRequested(); } _logger.ErrorException( $"{MaxStateChangeAttempts} state change attempt(s) failed due to an exception, moving job to the FailedState", exception); // When exception occurs, it's essential to remove a background job identifier from the schedule, // because otherwise delayed job scheduler will fetch such a failing job identifier again and again // and will be unable to make any progress. Any successful state change will cause that identifier // to be removed from the schedule. _stateChanger.ChangeState(new StateChangeContext( context.Storage, connection, jobId, new FailedState(exception) { Reason = $"Failed to change state to the '{EnqueuedState.StateName}' one due to an exception after {MaxStateChangeAttempts} retry attempts" }, new[] { ScheduledState.StateName }, disableFilters: true, context.StoppingToken, _profiler)); }
private static List<Continuation> GetContinuations(IStorageConnection connection, string jobId) { return JobHelper.FromJson<List<Continuation>>(connection.GetJobParameter( jobId, "Continuations")) ?? new List<Continuation>(); }
public PetiodicStopDispatcher() { _connection = JobStorage.Current.GetConnection(); _periodicJobRepository = new PeriodicJobRepository(); }
private void ScheduleRecurringJob(BackgroundProcessContext context, IStorageConnection connection, string recurringJobId, RecurringJobEntity recurringJob, DateTime now) { // We always start a transaction, regardless our recurring job was updated or not, // to prevent from infinite loop, when there's an old processing server (pre-1.7.0) // in our environment that doesn't know it should modify the score for entries in // the recurring jobs set. using (var transaction = connection.CreateWriteTransaction()) { try { // We can't handle recurring job with unsupported versions - there may be additional // features. we don't know about. We also shouldn't stop the whole scheduler as // there may be jobs with lower versions. Instead, we'll re-schedule such a job and // emit a warning message to the log. if (recurringJob.Version.HasValue && recurringJob.Version > MaxSupportedVersion) { throw new NotSupportedException($"Server '{context.ServerId}' can't process recurring job '{recurringJobId}' of version '{recurringJob.Version ?? 1}'. Max supported version of this server is '{MaxSupportedVersion}'."); } BackgroundJob backgroundJob = null; IReadOnlyDictionary <string, string> changedFields; if (recurringJob.TrySchedule(out var nextExecution, out var error)) { if (nextExecution.HasValue && nextExecution <= now) { backgroundJob = _factory.TriggerRecurringJob(context.Storage, connection, _profiler, recurringJob, now); if (String.IsNullOrEmpty(backgroundJob?.Id)) { _logger.Debug($"Recurring job '{recurringJobId}' execution at '{nextExecution}' has been canceled."); } } recurringJob.IsChanged(out changedFields, out nextExecution); } else { throw new InvalidOperationException("Recurring job can't be scheduled, see inner exception for details.", error); } if (backgroundJob != null) { _factory.StateMachine.EnqueueBackgroundJob( context.Storage, connection, transaction, recurringJob, backgroundJob, "Triggered by recurring job scheduler", _profiler); } transaction.UpdateRecurringJob(recurringJob, changedFields, nextExecution, _logger); } catch (Exception ex) { throw new BackgroundJobClientException(ex.Message, ex); } // We should commit transaction outside of the internal try/catch block, because these // exceptions are always due to network issues, and in case of a timeout exception we // can't determine whether it was actually succeeded or not, and can't perform an // idempotent retry in this case. transaction.Commit(); } }
public IStateMachine Create(IStorageConnection connection) { if (connection == null) throw new ArgumentNullException("connection"); return new StateMachine(connection, _stateChangeProcess); }
private ActivatingContext( [NotNull] IStorageConnection connection, [NotNull] BackgroundJob backgroundJob) : base(connection, backgroundJob) { }
private void TryScheduleJob( JobStorage storage, IStorageConnection connection, string recurringJobId, IReadOnlyDictionary<string, string> recurringJob) { var serializedJob = JobHelper.FromJson<InvocationData>(recurringJob["Job"]); var job = serializedJob.Deserialize(); var cron = recurringJob["Cron"]; var cronSchedule = CrontabSchedule.Parse(cron); try { var timeZone = recurringJob.ContainsKey("TimeZoneId") ? TimeZoneInfo.FindSystemTimeZoneById(recurringJob["TimeZoneId"]) : TimeZoneInfo.Utc; var nowInstant = _instantFactory(cronSchedule, timeZone); var changedFields = new Dictionary<string, string>(); var lastInstant = GetLastInstant(recurringJob, nowInstant); if (nowInstant.GetNextInstants(lastInstant).Any()) { var state = new EnqueuedState { Reason = "Triggered by recurring job scheduler" }; if (recurringJob.ContainsKey("Queue") && !String.IsNullOrEmpty(recurringJob["Queue"])) { state.Queue = recurringJob["Queue"]; } var context = new CreateContext(storage, connection, job, state); context.Parameters["RecurringJobId"] = recurringJobId; var backgroundJob = _factory.Create(context); var jobId = backgroundJob?.Id; if (String.IsNullOrEmpty(jobId)) { Logger.Debug($"Recurring job '{recurringJobId}' execution at '{nowInstant.NowInstant}' has been canceled."); } changedFields.Add("LastExecution", JobHelper.SerializeDateTime(nowInstant.NowInstant)); changedFields.Add("LastJobId", jobId ?? String.Empty); } // Fixing old recurring jobs that doesn't have the CreatedAt field if (!recurringJob.ContainsKey("CreatedAt")) { changedFields.Add("CreatedAt", JobHelper.SerializeDateTime(nowInstant.NowInstant)); } changedFields.Add("NextExecution", nowInstant.NextInstant.HasValue ? JobHelper.SerializeDateTime(nowInstant.NextInstant.Value) : null); connection.SetRangeInHash( $"recurring-job:{recurringJobId}", changedFields); } #if NETFULL catch (TimeZoneNotFoundException ex) { #else catch (Exception ex) { // https://github.com/dotnet/corefx/issues/7552 if (!ex.GetType().Name.Equals("TimeZoneNotFoundException")) throw; #endif Logger.ErrorException( $"Recurring job '{recurringJobId}' was not triggered: {ex.Message}.", ex); } }
private void ProcessJob( string jobId, IStorageConnection connection, IJobPerformanceProcess process, CancellationToken shutdownToken) { var stateMachine = _context.StateMachineFactory.Create(connection); var processingState = new ProcessingState(_context.ServerId, _context.WorkerNumber); if (!stateMachine.TryToChangeState( jobId, processingState, new[] { EnqueuedState.StateName, ProcessingState.StateName })) { return; } // Checkpoint #3. Job is in the Processing state. However, there are // no guarantees that it was performed. We need to re-queue it even // it was performed to guarantee that it was performed AT LEAST once. // It will be re-queued after the JobTimeout was expired. IState state; try { var jobData = connection.GetJobData(jobId); jobData.EnsureLoaded(); var cancellationToken = new ServerJobCancellationToken( jobId, connection, _context, shutdownToken); var performContext = new PerformContext( _context, connection, jobId, jobData.Job, jobData.CreatedAt, cancellationToken); var latency = (DateTime.UtcNow - jobData.CreatedAt).TotalMilliseconds; var duration = Stopwatch.StartNew(); process.Run(performContext, jobData.Job); duration.Stop(); state = new SucceededState((long) latency, duration.ElapsedMilliseconds); } catch (OperationCanceledException) { throw; } catch (JobPerformanceException ex) { state = new FailedState(ex.InnerException) { Reason = ex.Message }; } catch (Exception ex) { state = new FailedState(ex) { Reason = "Internal HangFire Server exception occurred. Please, report it to HangFire developers." }; } // Ignore return value, because we should not do // anything when current state is not Processing. stateMachine.TryToChangeState(jobId, state, new[] { ProcessingState.StateName }); }
private static IEnumerable<StateHandler> GetStateHandlers( IStorageConnection connection) { var handlers = new List<StateHandler>(); handlers.AddRange(GlobalStateHandlers.Handlers); handlers.AddRange(connection.Storage.GetStateHandlers()); return handlers; }