public void Execute(BackgroundProcessContext context) { _logger.Trace("Checking for aborted jobs..."); using (var connection = context.Storage.GetConnection()) { var abortedJobIds = ServerJobCancellationToken.CheckAllCancellationTokens( context.ServerId, connection, context.StoppedToken); var aborted = false; foreach (var abortedJobId in abortedJobIds) { _logger.Debug($"Job {abortedJobId.Item1} was aborted on worker {abortedJobId.Item2}."); aborted = true; } if (!aborted) { _logger.Trace("No newly aborted jobs found."); } } context.Wait(_checkInterval); }
private IState PerformJob(BackgroundProcessContext context, IStorageConnection connection, string jobId) { try { var jobData = connection.GetJobData(jobId); if (jobData == null) { // Job expired just after moving to a processing state. This is an // unreal scenario, but shit happens. Returning null instead of throwing // an exception and rescuing from en-queueing a poisoned jobId back // to a queue. return(null); } jobData.EnsureLoaded(); var backgroundJob = new BackgroundJob(jobId, jobData.Job, jobData.CreatedAt); using (var jobToken = new ServerJobCancellationToken(connection, jobId, context.ServerId, context.ExecutionId.ToString(), context.StoppedToken)) { var performContext = new PerformContext(context.Storage, connection, backgroundJob, jobToken, _profiler); var latency = (DateTime.UtcNow - jobData.CreatedAt).TotalMilliseconds; var duration = Stopwatch.StartNew(); var result = _performer.Perform(performContext); duration.Stop(); return(new SucceededState(result, (long)latency, duration.ElapsedMilliseconds)); } } catch (JobAbortedException) { // Background job performance was aborted due to a // state change, so its identifier should be removed // from a queue. return(null); } catch (JobPerformanceException ex) { return(new FailedState(ex.InnerException) { Reason = ex.Message }); } catch (Exception ex) { if (ex is OperationCanceledException && context.IsStopped) { throw; } return(new FailedState(ex) { Reason = "An exception occurred during processing of a background job." }); } }
private void CreateServer(BackgroundServerContext context) { _logger.Trace($"{GetServerTemplate(context.ServerId)} is announcing itself..."); var stopwatch = Stopwatch.StartNew(); using (var connection = _storage.GetConnection()) { connection.AnnounceServer(context.ServerId, GetServerContext(_properties)); } stopwatch.Stop(); ServerJobCancellationToken.AddServer(context.ServerId); _logger.Info($"{GetServerTemplate(context.ServerId)} successfully announced in {stopwatch.Elapsed.TotalMilliseconds} ms"); }
private void ServerDelete(BackgroundServerContext context, Stopwatch stoppedAt) { try { _logger.Trace($"{GetServerTemplate(context.ServerId)} is reporting itself as stopped..."); ServerJobCancellationToken.RemoveServer(context.ServerId); var stopwatch = Stopwatch.StartNew(); using (var connection = _storage.GetConnection()) { connection.RemoveServer(context.ServerId); } stopwatch.Stop(); _logger.Info($"{GetServerTemplate(context.ServerId)} successfully reported itself as stopped in {stopwatch.Elapsed.TotalMilliseconds} ms"); _logger.Info($"{GetServerTemplate(context.ServerId)} has been stopped in total {stoppedAt?.Elapsed.TotalMilliseconds ?? 0} ms"); } catch (Exception ex) { _logger.WarnException($"{GetServerTemplate(context.ServerId)} there was an exception, server may not be removed", ex); } }
private IState PerformJob(BackgroundProcessContext context, IStorageConnection connection, string jobId) { try { var jobData = connection.GetJobData(jobId); if (jobData == null) { // Job expired just after moving to a processing state. This is an // unreal scenario, but shit happens. Returning null instead of throwing // an exception and rescuing from en-queueing a poisoned jobId back // to a queue. return null; } jobData.EnsureLoaded(); var backgroundJob = new BackgroundJob(jobId, jobData.Job, jobData.CreatedAt); var jobToken = new ServerJobCancellationToken(connection, jobId, context.ServerId, _workerId, context.CancellationToken); var performContext = new PerformContext(connection, backgroundJob, jobToken); var latency = (DateTime.UtcNow - jobData.CreatedAt).TotalMilliseconds; var duration = Stopwatch.StartNew(); var result = _performer.Perform(performContext); duration.Stop(); return new SucceededState(result, (long) latency, duration.ElapsedMilliseconds); } catch (OperationCanceledException) { throw; } catch (JobPerformanceException ex) { return new FailedState(ex.InnerException) { Reason = ex.Message }; } catch (Exception ex) { return new FailedState(ex) { Reason = "An exception occurred during processing of a background job." }; } }
private void ProcessJob( string jobId, IStorageConnection connection, IJobPerformanceProcess process, CancellationToken shutdownToken) { var stateMachine = _context.StateMachineFactory.Create(connection); var processingState = new ProcessingState(_context.ServerId, _context.WorkerNumber); if (!stateMachine.TryToChangeState( jobId, processingState, new[] { EnqueuedState.StateName, ProcessingState.StateName })) { return; } // Checkpoint #3. Job is in the Processing state. However, there are // no guarantees that it was performed. We need to re-queue it even // it was performed to guarantee that it was performed AT LEAST once. // It will be re-queued after the JobTimeout was expired. IState state; try { var jobData = connection.GetJobData(jobId); jobData.EnsureLoaded(); var cancellationToken = new ServerJobCancellationToken( jobId, connection, _context, shutdownToken); var performContext = new PerformContext( _context, connection, jobId, jobData.Job, jobData.CreatedAt, cancellationToken); var latency = (DateTime.UtcNow - jobData.CreatedAt).TotalMilliseconds; var duration = Stopwatch.StartNew(); process.Run(performContext, jobData.Job); duration.Stop(); state = new SucceededState((long) latency, duration.ElapsedMilliseconds); } catch (OperationCanceledException) { throw; } catch (JobPerformanceException ex) { state = new FailedState(ex.InnerException) { Reason = ex.Message }; } catch (Exception ex) { state = new FailedState(ex) { Reason = "Internal Hangfire Server exception occurred. Please, report it to Hangfire developers." }; } // Ignore return value, because we should not do // anything when current state is not Processing. stateMachine.TryToChangeState(jobId, state, new[] { ProcessingState.StateName }); }
public void Execute(CancellationToken cancellationToken) { using (var connection = _storage.GetConnection()) using (var fetchedJob = connection.FetchNextJob(_context.Queues, cancellationToken)) { try { var stateMachine = _stateMachineFactory.Create(connection); using (var timeoutCts = new CancellationTokenSource(JobInitializationWaitTimeout)) using (var linkedCts = CancellationTokenSource.CreateLinkedTokenSource( cancellationToken, timeoutCts.Token)) { var processingState = new ProcessingState(_context.ServerId, _context.WorkerNumber); if (!stateMachine.ChangeState( fetchedJob.JobId, processingState, new[] { EnqueuedState.StateName, ProcessingState.StateName }, linkedCts.Token)) { // We should re-queue a job identifier only when graceful shutdown // initiated. cancellationToken.ThrowIfCancellationRequested(); // We should forget a job in a wrong state, or when timeout exceeded. fetchedJob.RemoveFromQueue(); return; } } // Checkpoint #3. Job is in the Processing state. However, there are // no guarantees that it was performed. We need to re-queue it even // it was performed to guarantee that it was performed AT LEAST once. // It will be re-queued after the JobTimeout was expired. var jobCancellationToken = new ServerJobCancellationToken( fetchedJob.JobId, connection, _context, cancellationToken); var state = PerformJob(fetchedJob.JobId, connection, jobCancellationToken); if (state != null) { // Ignore return value, because we should not do anything when current state is not Processing. stateMachine.ChangeState(fetchedJob.JobId, state, new[] { ProcessingState.StateName }); } // Checkpoint #4. The job was performed, and it is in the one // of the explicit states (Succeeded, Scheduled and so on). // It should not be re-queued, but we still need to remove its // processing information. fetchedJob.RemoveFromQueue(); // Success point. No things must be done after previous command // was succeeded. } catch (JobAbortedException) { fetchedJob.RemoveFromQueue(); } catch (Exception ex) { Logger.DebugException("An exception occurred while processing a job. It will be re-queued.", ex); fetchedJob.Requeue(); throw; } } }
private void ProcessJob( string jobId, IStorageConnection connection, IJobPerformanceProcess process, CancellationToken shutdownToken) { var stateMachine = _context.StateMachineFactory.Create(connection); var processingState = new ProcessingState(_context.ServerId, _context.WorkerNumber); if (!stateMachine.TryToChangeState( jobId, processingState, new[] { EnqueuedState.StateName, ProcessingState.StateName })) { return; } // Checkpoint #3. Job is in the Processing state. However, there are // no guarantees that it was performed. We need to re-queue it even // it was performed to guarantee that it was performed AT LEAST once. // It will be re-queued after the JobTimeout was expired. IState state; try { var jobData = connection.GetJobData(jobId); jobData.EnsureLoaded(); var cancellationToken = new ServerJobCancellationToken( jobId, connection, _context, shutdownToken); var performContext = new PerformContext( _context, connection, jobId, jobData.Job, jobData.CreatedAt, cancellationToken); var latency = (DateTime.UtcNow - jobData.CreatedAt).TotalMilliseconds; var duration = Stopwatch.StartNew(); process.Run(performContext, jobData.Job); duration.Stop(); state = new SucceededState((long)latency, duration.ElapsedMilliseconds); } catch (OperationCanceledException) { throw; } catch (JobPerformanceException ex) { state = new FailedState(ex.InnerException) { Reason = ex.Message }; } catch (Exception ex) { state = new FailedState(ex) { Reason = "Internal Hangfire Server exception occurred. Please, report it to Hangfire developers." }; } // Ignore return value, because we should not do // anything when current state is not Processing. stateMachine.TryToChangeState(jobId, state, new[] { ProcessingState.StateName }); }
private IState PerformJob(BackgroundProcessContext context, IStorageConnection connection, string jobId) { try { var jobData = connection.GetJobData(jobId); if (jobData == null) { // Job expired just after moving to a processing state. This is an // unreal scenario, but shit happens. Returning null instead of throwing // an exception and rescuing from en-queueing a poisoned jobId back // to a queue. return(null); } jobData.EnsureLoaded(); var backgroundJob = new BackgroundJob(jobId, jobData.Job, jobData.CreatedAt); var jobToken = new ServerJobCancellationToken(connection, jobId, context.ServerId, _workerId, context.CancellationToken); var jobActivatorContext = new JobActivatorContext(connection, backgroundJob, jobToken); var scope = _activator.BeginScope(jobActivatorContext); if (scope != null) { using (scope) { var performContext = new PerformContext(connection, backgroundJob, jobToken, _profiler, scope); var latency = (DateTime.UtcNow - jobData.CreatedAt).TotalMilliseconds; var duration = Stopwatch.StartNew(); var result = _performer.Perform(performContext); duration.Stop(); return(new SucceededState(result, (long)latency, duration.ElapsedMilliseconds)); } } else { if (jobActivatorContext.ReEnqueueAt.HasValue) { return(new ScheduledState(jobActivatorContext.ReEnqueueAt.Value)); } else { return(new FailedState(new InvalidOperationException("Scope could not be started"))); } } } catch (JobAbortedException) { // Background job performance was aborted due to a // state change, so it's idenfifier should be removed // from a queue. return(null); } catch (JobPerformanceException ex) { return(new FailedState(ex.InnerException) { Reason = ex.Message }); } catch (Exception ex) { if (ex is OperationCanceledException && context.IsShutdownRequested) { throw; } return(new FailedState(ex) { Reason = "An exception occurred during processing of a background job." }); } }
public void Execute(CancellationToken cancellationToken) { using (var connection = _storage.GetConnection()) using (var fetchedJob = connection.FetchNextJob(_context.Queues, cancellationToken)) { try { var stateMachine = _stateMachineFactory.Create(connection); using (var timeoutCts = new CancellationTokenSource(JobInitializationWaitTimeout)) using (var linkedCts = CancellationTokenSource.CreateLinkedTokenSource( cancellationToken, timeoutCts.Token)) { var processingState = new ProcessingState(_context.ServerId, _context.WorkerNumber); if (!stateMachine.ChangeState( fetchedJob.JobId, processingState, new[] { EnqueuedState.StateName, ProcessingState.StateName }, linkedCts.Token)) { // We should re-queue a job identifier only when graceful shutdown // initiated. cancellationToken.ThrowIfCancellationRequested(); // We should forget a job in a wrong state, or when timeout exceeded. fetchedJob.RemoveFromQueue(); return; } } // Checkpoint #3. Job is in the Processing state. However, there are // no guarantees that it was performed. We need to re-queue it even // it was performed to guarantee that it was performed AT LEAST once. // It will be re-queued after the JobTimeout was expired. var jobCancellationToken = new ServerJobCancellationToken( fetchedJob.JobId, connection, _context, cancellationToken); var state = PerformJob(fetchedJob.JobId, connection, jobCancellationToken); if (state != null) { // Ignore return value, because we should not do anything when current state is not Processing. stateMachine.ChangeState(fetchedJob.JobId, state, new[] { ProcessingState.StateName }); } // Checkpoint #4. The job was performed, and it is in the one // of the explicit states (Succeeded, Scheduled and so on). // It should not be re-queued, but we still need to remove its // processing information. fetchedJob.RemoveFromQueue(); // Success point. No things must be done after previous command // was succeeded. } catch (JobAbortedException) { fetchedJob.RemoveFromQueue(); } catch (Exception ex) { Logger.DebugException("An exception occurred while processing a job. It will be re-queued.", ex); fetchedJob.Requeue(); throw; } } }