public void Execute(BackgroundProcessContext context)
        {
            _logger.Trace("Checking for aborted jobs...");

            using (var connection = context.Storage.GetConnection())
            {
                var abortedJobIds = ServerJobCancellationToken.CheckAllCancellationTokens(
                    context.ServerId,
                    connection,
                    context.StoppedToken);

                var aborted = false;

                foreach (var abortedJobId in abortedJobIds)
                {
                    _logger.Debug($"Job {abortedJobId.Item1} was aborted on worker {abortedJobId.Item2}.");
                    aborted = true;
                }

                if (!aborted)
                {
                    _logger.Trace("No newly aborted jobs found.");
                }
            }

            context.Wait(_checkInterval);
        }
Ejemplo n.º 2
0
        private IState PerformJob(BackgroundProcessContext context, IStorageConnection connection, string jobId)
        {
            try
            {
                var jobData = connection.GetJobData(jobId);
                if (jobData == null)
                {
                    // Job expired just after moving to a processing state. This is an
                    // unreal scenario, but shit happens. Returning null instead of throwing
                    // an exception and rescuing from en-queueing a poisoned jobId back
                    // to a queue.
                    return(null);
                }

                jobData.EnsureLoaded();

                var backgroundJob = new BackgroundJob(jobId, jobData.Job, jobData.CreatedAt);

                using (var jobToken = new ServerJobCancellationToken(connection, jobId, context.ServerId, context.ExecutionId.ToString(), context.StoppedToken))
                {
                    var performContext = new PerformContext(context.Storage, connection, backgroundJob, jobToken, _profiler);

                    var latency  = (DateTime.UtcNow - jobData.CreatedAt).TotalMilliseconds;
                    var duration = Stopwatch.StartNew();

                    var result = _performer.Perform(performContext);
                    duration.Stop();

                    return(new SucceededState(result, (long)latency, duration.ElapsedMilliseconds));
                }
            }
            catch (JobAbortedException)
            {
                // Background job performance was aborted due to a
                // state change, so its identifier should be removed
                // from a queue.
                return(null);
            }
            catch (JobPerformanceException ex)
            {
                return(new FailedState(ex.InnerException)
                {
                    Reason = ex.Message
                });
            }
            catch (Exception ex)
            {
                if (ex is OperationCanceledException && context.IsStopped)
                {
                    throw;
                }

                return(new FailedState(ex)
                {
                    Reason = "An exception occurred during processing of a background job."
                });
            }
        }
        private void CreateServer(BackgroundServerContext context)
        {
            _logger.Trace($"{GetServerTemplate(context.ServerId)} is announcing itself...");

            var stopwatch = Stopwatch.StartNew();

            using (var connection = _storage.GetConnection())
            {
                connection.AnnounceServer(context.ServerId, GetServerContext(_properties));
            }

            stopwatch.Stop();

            ServerJobCancellationToken.AddServer(context.ServerId);
            _logger.Info($"{GetServerTemplate(context.ServerId)} successfully announced in {stopwatch.Elapsed.TotalMilliseconds} ms");
        }
        private void ServerDelete(BackgroundServerContext context, Stopwatch stoppedAt)
        {
            try
            {
                _logger.Trace($"{GetServerTemplate(context.ServerId)} is reporting itself as stopped...");
                ServerJobCancellationToken.RemoveServer(context.ServerId);

                var stopwatch = Stopwatch.StartNew();

                using (var connection = _storage.GetConnection())
                {
                    connection.RemoveServer(context.ServerId);
                }

                stopwatch.Stop();

                _logger.Info($"{GetServerTemplate(context.ServerId)} successfully reported itself as stopped in {stopwatch.Elapsed.TotalMilliseconds} ms");
                _logger.Info($"{GetServerTemplate(context.ServerId)} has been stopped in total {stoppedAt?.Elapsed.TotalMilliseconds ?? 0} ms");
            }
            catch (Exception ex)
            {
                _logger.WarnException($"{GetServerTemplate(context.ServerId)} there was an exception, server may not be removed", ex);
            }
        }
Ejemplo n.º 5
0
        private IState PerformJob(BackgroundProcessContext context, IStorageConnection connection, string jobId)
        {
            try
            {
                var jobData = connection.GetJobData(jobId);
                if (jobData == null)
                {
                    // Job expired just after moving to a processing state. This is an
                    // unreal scenario, but shit happens. Returning null instead of throwing
                    // an exception and rescuing from en-queueing a poisoned jobId back
                    // to a queue.
                    return null;
                }

                jobData.EnsureLoaded();

                var backgroundJob = new BackgroundJob(jobId, jobData.Job, jobData.CreatedAt);

                var jobToken = new ServerJobCancellationToken(connection, jobId, context.ServerId, _workerId, context.CancellationToken);
                var performContext = new PerformContext(connection, backgroundJob, jobToken);

                var latency = (DateTime.UtcNow - jobData.CreatedAt).TotalMilliseconds;
                var duration = Stopwatch.StartNew();

                var result = _performer.Perform(performContext);
                duration.Stop();

                return new SucceededState(result, (long) latency, duration.ElapsedMilliseconds);
            }
            catch (OperationCanceledException)
            {
                throw;
            }
            catch (JobPerformanceException ex)
            {
                return new FailedState(ex.InnerException)
                {
                    Reason = ex.Message
                };
            }
            catch (Exception ex)
            {
                return new FailedState(ex)
                {
                    Reason = "An exception occurred during processing of a background job."
                };
            }
        }
Ejemplo n.º 6
0
        private void ProcessJob(
            string jobId,
            IStorageConnection connection, 
            IJobPerformanceProcess process,
            CancellationToken shutdownToken)
        {
            var stateMachine = _context.StateMachineFactory.Create(connection);
            var processingState = new ProcessingState(_context.ServerId, _context.WorkerNumber);

            if (!stateMachine.TryToChangeState(
                jobId,
                processingState,
                new[] { EnqueuedState.StateName, ProcessingState.StateName }))
            {
                return;
            }

            // Checkpoint #3. Job is in the Processing state. However, there are
            // no guarantees that it was performed. We need to re-queue it even
            // it was performed to guarantee that it was performed AT LEAST once.
            // It will be re-queued after the JobTimeout was expired.

            IState state;

            try
            {
                var jobData = connection.GetJobData(jobId);
                jobData.EnsureLoaded();

                var cancellationToken = new ServerJobCancellationToken(
                    jobId, connection, _context, shutdownToken);

                var performContext = new PerformContext(
                    _context, connection, jobId, jobData.Job, jobData.CreatedAt, cancellationToken);

                var latency = (DateTime.UtcNow - jobData.CreatedAt).TotalMilliseconds;
                var duration = Stopwatch.StartNew();

                process.Run(performContext, jobData.Job);
                duration.Stop();

                state = new SucceededState((long) latency, duration.ElapsedMilliseconds);
            }
            catch (OperationCanceledException)
            {
                throw;
            }
            catch (JobPerformanceException ex)
            {
                state = new FailedState(ex.InnerException)
                {
                    Reason = ex.Message
                };
            }
            catch (Exception ex)
            {
                state = new FailedState(ex)
                {
                    Reason = "Internal Hangfire Server exception occurred. Please, report it to Hangfire developers."
                };
            }

            // Ignore return value, because we should not do
            // anything when current state is not Processing.
            stateMachine.TryToChangeState(jobId, state, new[] { ProcessingState.StateName });
        }
Ejemplo n.º 7
0
        public void Execute(CancellationToken cancellationToken)
        {
            using (var connection = _storage.GetConnection())
            using (var fetchedJob = connection.FetchNextJob(_context.Queues, cancellationToken))
            {
                try
                {
                    var stateMachine = _stateMachineFactory.Create(connection);

                    using (var timeoutCts = new CancellationTokenSource(JobInitializationWaitTimeout))
                    using (var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(
                        cancellationToken,
                        timeoutCts.Token))
                    {
                        var processingState = new ProcessingState(_context.ServerId, _context.WorkerNumber);

                        if (!stateMachine.ChangeState(
                            fetchedJob.JobId,
                            processingState,
                            new[] { EnqueuedState.StateName, ProcessingState.StateName },
                            linkedCts.Token))
                        {
                            // We should re-queue a job identifier only when graceful shutdown
                            // initiated.
                            cancellationToken.ThrowIfCancellationRequested();

                            // We should forget a job in a wrong state, or when timeout exceeded.
                            fetchedJob.RemoveFromQueue();
                            return;
                        }
                    }

                    // Checkpoint #3. Job is in the Processing state. However, there are
                    // no guarantees that it was performed. We need to re-queue it even
                    // it was performed to guarantee that it was performed AT LEAST once.
                    // It will be re-queued after the JobTimeout was expired.

                    var jobCancellationToken = new ServerJobCancellationToken(
                        fetchedJob.JobId, connection, _context, cancellationToken);

                    var state = PerformJob(fetchedJob.JobId, connection, jobCancellationToken);

                    if (state != null)
                    {
                        // Ignore return value, because we should not do anything when current state is not Processing.
                        stateMachine.ChangeState(fetchedJob.JobId, state, new[] { ProcessingState.StateName });
                    }

                    // Checkpoint #4. The job was performed, and it is in the one
                    // of the explicit states (Succeeded, Scheduled and so on).
                    // It should not be re-queued, but we still need to remove its
                    // processing information.

                    fetchedJob.RemoveFromQueue();

                    // Success point. No things must be done after previous command
                    // was succeeded.
                }
                catch (JobAbortedException)
                {
                    fetchedJob.RemoveFromQueue();
                }
                catch (Exception ex)
                {
                    Logger.DebugException("An exception occurred while processing a job. It will be re-queued.", ex);

                    fetchedJob.Requeue();
                    throw;
                }
            }
        }
Ejemplo n.º 8
0
        private void ProcessJob(
            string jobId,
            IStorageConnection connection,
            IJobPerformanceProcess process,
            CancellationToken shutdownToken)
        {
            var stateMachine    = _context.StateMachineFactory.Create(connection);
            var processingState = new ProcessingState(_context.ServerId, _context.WorkerNumber);

            if (!stateMachine.TryToChangeState(
                    jobId,
                    processingState,
                    new[] { EnqueuedState.StateName, ProcessingState.StateName }))
            {
                return;
            }

            // Checkpoint #3. Job is in the Processing state. However, there are
            // no guarantees that it was performed. We need to re-queue it even
            // it was performed to guarantee that it was performed AT LEAST once.
            // It will be re-queued after the JobTimeout was expired.

            IState state;

            try
            {
                var jobData = connection.GetJobData(jobId);
                jobData.EnsureLoaded();

                var cancellationToken = new ServerJobCancellationToken(
                    jobId, connection, _context, shutdownToken);

                var performContext = new PerformContext(
                    _context, connection, jobId, jobData.Job, jobData.CreatedAt, cancellationToken);

                var latency  = (DateTime.UtcNow - jobData.CreatedAt).TotalMilliseconds;
                var duration = Stopwatch.StartNew();

                process.Run(performContext, jobData.Job);
                duration.Stop();

                state = new SucceededState((long)latency, duration.ElapsedMilliseconds);
            }
            catch (OperationCanceledException)
            {
                throw;
            }
            catch (JobPerformanceException ex)
            {
                state = new FailedState(ex.InnerException)
                {
                    Reason = ex.Message
                };
            }
            catch (Exception ex)
            {
                state = new FailedState(ex)
                {
                    Reason = "Internal Hangfire Server exception occurred. Please, report it to Hangfire developers."
                };
            }

            // Ignore return value, because we should not do
            // anything when current state is not Processing.
            stateMachine.TryToChangeState(jobId, state, new[] { ProcessingState.StateName });
        }
Ejemplo n.º 9
0
        private IState PerformJob(BackgroundProcessContext context, IStorageConnection connection, string jobId)
        {
            try
            {
                var jobData = connection.GetJobData(jobId);
                if (jobData == null)
                {
                    // Job expired just after moving to a processing state. This is an
                    // unreal scenario, but shit happens. Returning null instead of throwing
                    // an exception and rescuing from en-queueing a poisoned jobId back
                    // to a queue.
                    return(null);
                }

                jobData.EnsureLoaded();

                var backgroundJob = new BackgroundJob(jobId, jobData.Job, jobData.CreatedAt);

                var jobToken            = new ServerJobCancellationToken(connection, jobId, context.ServerId, _workerId, context.CancellationToken);
                var jobActivatorContext = new JobActivatorContext(connection, backgroundJob, jobToken);
                var scope = _activator.BeginScope(jobActivatorContext);
                if (scope != null)
                {
                    using (scope)
                    {
                        var performContext = new PerformContext(connection, backgroundJob, jobToken, _profiler, scope);

                        var latency  = (DateTime.UtcNow - jobData.CreatedAt).TotalMilliseconds;
                        var duration = Stopwatch.StartNew();

                        var result = _performer.Perform(performContext);
                        duration.Stop();

                        return(new SucceededState(result, (long)latency, duration.ElapsedMilliseconds));
                    }
                }
                else
                {
                    if (jobActivatorContext.ReEnqueueAt.HasValue)
                    {
                        return(new ScheduledState(jobActivatorContext.ReEnqueueAt.Value));
                    }
                    else
                    {
                        return(new FailedState(new InvalidOperationException("Scope could not be started")));
                    }
                }
            }
            catch (JobAbortedException)
            {
                // Background job performance was aborted due to a
                // state change, so it's idenfifier should be removed
                // from a queue.
                return(null);
            }
            catch (JobPerformanceException ex)
            {
                return(new FailedState(ex.InnerException)
                {
                    Reason = ex.Message
                });
            }
            catch (Exception ex)
            {
                if (ex is OperationCanceledException && context.IsShutdownRequested)
                {
                    throw;
                }

                return(new FailedState(ex)
                {
                    Reason = "An exception occurred during processing of a background job."
                });
            }
        }
Ejemplo n.º 10
0
        public void Execute(CancellationToken cancellationToken)
        {
            using (var connection = _storage.GetConnection())
                using (var fetchedJob = connection.FetchNextJob(_context.Queues, cancellationToken))
                {
                    try
                    {
                        var stateMachine = _stateMachineFactory.Create(connection);

                        using (var timeoutCts = new CancellationTokenSource(JobInitializationWaitTimeout))
                            using (var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(
                                       cancellationToken,
                                       timeoutCts.Token))
                            {
                                var processingState = new ProcessingState(_context.ServerId, _context.WorkerNumber);

                                if (!stateMachine.ChangeState(
                                        fetchedJob.JobId,
                                        processingState,
                                        new[] { EnqueuedState.StateName, ProcessingState.StateName },
                                        linkedCts.Token))
                                {
                                    // We should re-queue a job identifier only when graceful shutdown
                                    // initiated.
                                    cancellationToken.ThrowIfCancellationRequested();

                                    // We should forget a job in a wrong state, or when timeout exceeded.
                                    fetchedJob.RemoveFromQueue();
                                    return;
                                }
                            }

                        // Checkpoint #3. Job is in the Processing state. However, there are
                        // no guarantees that it was performed. We need to re-queue it even
                        // it was performed to guarantee that it was performed AT LEAST once.
                        // It will be re-queued after the JobTimeout was expired.

                        var jobCancellationToken = new ServerJobCancellationToken(
                            fetchedJob.JobId, connection, _context, cancellationToken);

                        var state = PerformJob(fetchedJob.JobId, connection, jobCancellationToken);

                        if (state != null)
                        {
                            // Ignore return value, because we should not do anything when current state is not Processing.
                            stateMachine.ChangeState(fetchedJob.JobId, state, new[] { ProcessingState.StateName });
                        }

                        // Checkpoint #4. The job was performed, and it is in the one
                        // of the explicit states (Succeeded, Scheduled and so on).
                        // It should not be re-queued, but we still need to remove its
                        // processing information.

                        fetchedJob.RemoveFromQueue();

                        // Success point. No things must be done after previous command
                        // was succeeded.
                    }
                    catch (JobAbortedException)
                    {
                        fetchedJob.RemoveFromQueue();
                    }
                    catch (Exception ex)
                    {
                        Logger.DebugException("An exception occurred while processing a job. It will be re-queued.", ex);

                        fetchedJob.Requeue();
                        throw;
                    }
                }
        }