If a job throws this exception the Worker will retry.
Inheritance: System.Exception
Esempio n. 1
0
        /// <summary>
        /// Sequentially pick and run jobs from the <see cref="Queue"/> until stop is requested.
        /// </summary>
        public void Work()
        {
            try
            {
                IsStopRequested = false;

                if (State == WorkerState.Waiting)
                {
                    throw new Exception("This worker is already waiting for work");
                }
                if (State == WorkerState.Working)
                {
                    throw new Exception("This worker is already working");
                }
                State = WorkerState.Waiting;

                RoqueTrace.Source.Trace(TraceEventType.Information, "Worker {0} started. Roque v{1}. AppDomain: {2}",
                                        Name, new Func <object>(() => Assembly.GetAssembly(typeof(Worker)).GetName().Version), AppDomain.CurrentDomain.FriendlyName);

                while (!_SubscribersRegistered && !IsStopRequested)
                {
                    try
                    {
                        Executor.Default.RegisterSubscribersForWorker(this);
                        _SubscribersRegistered = true;
                    }
                    catch
                    {
                        // error registering subscriber, log is already done
                        RoqueTrace.Source.Trace(TraceEventType.Information, "Error registering subscribers, retrying in 10 seconds...");
                        Thread.Sleep(10000);
                    }
                }

                bool attemptWipResume = true;

                int  consecutiveErrors = 0;
                int  retries           = 0;
                bool firstTime         = true;
                bool shouldRetry       = false;

                Stopwatch stopwatchBatchWork   = null;
                Stopwatch stopwatchLastDequeue = null;

                while (!IsStopRequested)
                {
                    Job job = null;
                    try
                    {
                        State = WorkerState.Waiting;
                        stopwatchLastDequeue = new Stopwatch();
                        stopwatchLastDequeue.Start();
                        if (attemptWipResume)
                        {
                            job = Queue.GetInProgressJob(this);
                            attemptWipResume = false;
                            if (job == null)
                            {
                                RoqueTrace.Source.Trace(TraceEventType.Information, "No pending jobs to resume for worker: {0}", Name);
                            }
                            else
                            {
                                retries = 1;
                                RoqueTrace.Source.Trace(TraceEventType.Information, "Resuming pending job for worker: {0}", Name);
                            }
                        }

                        if (job == null)
                        {
                            if (shouldRetry)
                            {
                                job         = Queue.GetInProgressJob(this);
                                shouldRetry = false;
                                RoqueTrace.Source.Trace(TraceEventType.Information, "Retry #'{0}'. Worker {1}, Method: {2}", retries, Name, job.Method);
                            }
                            else
                            {
                                job     = Queue.Dequeue(this, firstTime ? 1 : 10);
                                retries = 0;
                            }
                        }

                        if (job != null)
                        {
                            if (stopwatchBatchWork == null)
                            {
                                stopwatchBatchWork = new Stopwatch();
                                stopwatchBatchWork.Start();
                            }
                            State = WorkerState.Working;
                            job.Execute();
                            consecutiveErrors = 0;
                            try
                            {
                                Queue.Completed(this, job);
                            }
                            catch (Exception ex)
                            {
                                RoqueTrace.Source.Trace(TraceEventType.Error, "Error marking job as completed for worker {0}: {1}", Name, ex.Message, ex);
                            }
                        }
                        else
                        {
                            if (stopwatchBatchWork != null)
                            {
                                RoqueTrace.Source.Trace(TraceEventType.Information, "Queue is empty. Worker {0} worked for: {1}", Name, stopwatchBatchWork.Elapsed.Subtract(stopwatchLastDequeue.Elapsed));
                                stopwatchBatchWork = null;
                            }
                        }
                    }
                    catch (Exception jobEx)
                    {
                        State = WorkerState.Waiting;
                        consecutiveErrors++;

                        if (job != null)
                        {
                            ShouldRetryException retryEx = jobEx as ShouldRetryException;
                            if (retryEx != null && (retryEx.MaxTimes > 0 && retries >= retryEx.MaxTimes))
                            {
                                retryEx = null;
                            }

                            if (retryEx == null)
                            {
                                // job completed with errors, but we can move on
                                try
                                {
                                    Queue.Completed(this, job, true);
                                }
                                catch (Exception ex)
                                {
                                    RoqueTrace.Source.Trace(TraceEventType.Error, "Error marking failed job as completed for worker {0}: {1}", Name, ex.Message, ex);
                                }
                                if (consecutiveErrors >= TooManyErrors)
                                {
                                    RoqueTrace.Source.Trace(TraceEventType.Warning, "Too many errors on worker '{0}', picking next job in {1} seconds", Name, TooManyErrorsRetrySeconds);
                                    // too many errors, wait some time before picking next job
                                    Thread.Sleep(TimeSpan.FromSeconds(TooManyErrorsRetrySeconds));
                                }
                            }
                            else
                            {
                                retries++;
                                shouldRetry = true;
                                RoqueTrace.Source.Trace(TraceEventType.Information, "Retrying failed job on worker '{0}' in {1}", Name, retryEx.Delay);
                                // wait some time before retrying the failed job
                                if (retryEx.Delay.Ticks > 0)
                                {
                                    Thread.Sleep(retryEx.Delay);
                                }
                            }
                        }
                    }
                    firstTime = false;
                }
                IsStopRequested = false;
            }
            catch (Exception ex)
            {
                RoqueTrace.Source.Trace(TraceEventType.Error, "Error running worker {0}: {1}", Name, ex.Message, ex);
            }
            finally
            {
                State = WorkerState.Stopped;
                RoqueTrace.Source.Trace(TraceEventType.Information, "Worker stopped: {0}", Name);
                var handler = Stopped;
                if (handler != null)
                {
                    Stopped(this, EventArgs.Empty);
                }
            }
        }
Esempio n. 2
0
        /// <summary>
        /// Sequentially pick and run jobs from the <see cref="Queue"/> until stop is requested.
        /// </summary>
        public void Work()
        {
            try
            {
                if (!_SubscribersRegistered)
                {
                    Executor.Default.RegisterSubscribersForWorker(this);
                    _SubscribersRegistered = true;
                }

                if (State == WorkerState.Waiting)
                {
                    throw new Exception("This worker is already waiting for work");
                }
                if (State == WorkerState.Working)
                {
                    throw new Exception("This worker is already working");
                }
                State = WorkerState.Waiting;
                if (RoqueTrace.Switch.TraceInfo)
                {
                    Trace.TraceInformation("Worker started: " + Name);
                }

                bool attemptWipResume = true;

                int  consecutiveErrors = 0;
                int  retries           = 0;
                bool firstTime         = true;
                bool shouldRetry       = false;

                Stopwatch stopwatchBatchWork   = null;
                Stopwatch stopwatchLastDequeue = null;

                while (!IsStopRequested)
                {
                    Job job = null;
                    try
                    {
                        State = WorkerState.Waiting;
                        stopwatchLastDequeue = new Stopwatch();
                        stopwatchLastDequeue.Start();
                        if (attemptWipResume)
                        {
                            job = Queue.GetInProgressJob(this);
                            attemptWipResume = false;
                            if (job == null)
                            {
                                if (RoqueTrace.Switch.TraceInfo)
                                {
                                    Trace.TraceInformation("No pending jobs to resume for worker: " + Name);
                                }
                            }
                            else
                            {
                                retries = 1;
                                if (RoqueTrace.Switch.TraceInfo)
                                {
                                    Trace.TraceInformation("Resuming pending job for worker: " + Name);
                                }
                            }
                        }

                        if (job == null)
                        {
                            if (shouldRetry)
                            {
                                job         = Queue.GetInProgressJob(this);
                                shouldRetry = false;
                                if (RoqueTrace.Switch.TraceInfo)
                                {
                                    Trace.TraceInformation(string.Format("Retry #'{0}'. Worker {1}, Method: {2}", retries, Name, job.Method));
                                }
                            }
                            else
                            {
                                job     = Queue.Dequeue(this, firstTime ? 1 : 10);
                                retries = 0;
                            }
                        }

                        if (job != null)
                        {
                            if (stopwatchBatchWork == null)
                            {
                                stopwatchBatchWork = new Stopwatch();
                                stopwatchBatchWork.Start();
                            }
                            State = WorkerState.Working;
                            job.Execute();
                            consecutiveErrors = 0;
                            try
                            {
                                Queue.Completed(this, job);
                            }
                            catch (Exception ex)
                            {
                                if (RoqueTrace.Switch.TraceError)
                                {
                                    Trace.TraceError(
                                        "Error marking job as completed for worker " + Name + ": " + ex.Message, ex);
                                }
                            }
                        }
                        else
                        {
                            if (stopwatchBatchWork != null)
                            {
                                if (RoqueTrace.Switch.TraceInfo)
                                {
                                    Trace.TraceInformation("Queue is empty. Worker {0} worked for: {1}", Name,
                                                           stopwatchBatchWork.Elapsed.Subtract(stopwatchLastDequeue.Elapsed));
                                }
                                stopwatchBatchWork = null;
                            }
                        }
                    }
                    catch (Exception jobEx)
                    {
                        State = WorkerState.Waiting;
                        consecutiveErrors++;

                        if (job != null)
                        {
                            ShouldRetryException retryEx = jobEx as ShouldRetryException;
                            if (retryEx != null && (retryEx.MaxTimes > 0 && retries >= retryEx.MaxTimes))
                            {
                                retryEx = null;
                            }

                            if (retryEx == null)
                            {
                                // job completed with errors, but we can move on
                                try
                                {
                                    Queue.Completed(this, job, true);
                                }
                                catch (Exception ex)
                                {
                                    if (RoqueTrace.Switch.TraceError)
                                    {
                                        Trace.TraceError(
                                            "Error marking job as completed for worker " + Name + ": " + ex.Message, ex);
                                    }
                                }
                                if (consecutiveErrors >= TooManyErrors)
                                {
                                    if (RoqueTrace.Switch.TraceInfo)
                                    {
                                        Trace.TraceInformation(string.Format("Too many errors on worker '{0}', picking next job in {1} seconds", Name, TooManyErrorsRetrySeconds));
                                    }
                                    // too many errors, wait some time before picking next job
                                    Thread.Sleep(TimeSpan.FromSeconds(TooManyErrorsRetrySeconds));
                                }
                            }
                            else
                            {
                                retries++;
                                shouldRetry = true;
                                if (RoqueTrace.Switch.TraceInfo)
                                {
                                    Trace.TraceInformation(string.Format("Retrying failed job on worker '{0}' in {1}", Name, retryEx.Delay));
                                }
                                // wait some time before retrying the failed job
                                if (retryEx.Delay.Ticks > 0)
                                {
                                    Thread.Sleep(retryEx.Delay);
                                }
                            }
                        }
                    }
                    firstTime = false;
                }
            }
            catch (Exception ex)
            {
                if (RoqueTrace.Switch.TraceError)
                {
                    Trace.TraceError("Error running worker " + Name + ": " + ex.Message, ex);
                }
            }
            finally
            {
                State = WorkerState.Stopped;
                if (RoqueTrace.Switch.TraceInfo)
                {
                    Trace.TraceInformation("Worker stopped: " + Name);
                }
                var handler = Stopped;
                if (handler != null)
                {
                    Stopped(this, EventArgs.Empty);
                }
            }
        }