/// <summary> /// Sequentially pick and run jobs from the <see cref="Queue"/> until stop is requested. /// </summary> public void Work() { try { IsStopRequested = false; if (State == WorkerState.Waiting) { throw new Exception("This worker is already waiting for work"); } if (State == WorkerState.Working) { throw new Exception("This worker is already working"); } State = WorkerState.Waiting; RoqueTrace.Source.Trace(TraceEventType.Information, "Worker {0} started. Roque v{1}. AppDomain: {2}", Name, new Func <object>(() => Assembly.GetAssembly(typeof(Worker)).GetName().Version), AppDomain.CurrentDomain.FriendlyName); while (!_SubscribersRegistered && !IsStopRequested) { try { Executor.Default.RegisterSubscribersForWorker(this); _SubscribersRegistered = true; } catch { // error registering subscriber, log is already done RoqueTrace.Source.Trace(TraceEventType.Information, "Error registering subscribers, retrying in 10 seconds..."); Thread.Sleep(10000); } } bool attemptWipResume = true; int consecutiveErrors = 0; int retries = 0; bool firstTime = true; bool shouldRetry = false; Stopwatch stopwatchBatchWork = null; Stopwatch stopwatchLastDequeue = null; while (!IsStopRequested) { Job job = null; try { State = WorkerState.Waiting; stopwatchLastDequeue = new Stopwatch(); stopwatchLastDequeue.Start(); if (attemptWipResume) { job = Queue.GetInProgressJob(this); attemptWipResume = false; if (job == null) { RoqueTrace.Source.Trace(TraceEventType.Information, "No pending jobs to resume for worker: {0}", Name); } else { retries = 1; RoqueTrace.Source.Trace(TraceEventType.Information, "Resuming pending job for worker: {0}", Name); } } if (job == null) { if (shouldRetry) { job = Queue.GetInProgressJob(this); shouldRetry = false; RoqueTrace.Source.Trace(TraceEventType.Information, "Retry #'{0}'. Worker {1}, Method: {2}", retries, Name, job.Method); } else { job = Queue.Dequeue(this, firstTime ? 1 : 10); retries = 0; } } if (job != null) { if (stopwatchBatchWork == null) { stopwatchBatchWork = new Stopwatch(); stopwatchBatchWork.Start(); } State = WorkerState.Working; job.Execute(); consecutiveErrors = 0; try { Queue.Completed(this, job); } catch (Exception ex) { RoqueTrace.Source.Trace(TraceEventType.Error, "Error marking job as completed for worker {0}: {1}", Name, ex.Message, ex); } } else { if (stopwatchBatchWork != null) { RoqueTrace.Source.Trace(TraceEventType.Information, "Queue is empty. Worker {0} worked for: {1}", Name, stopwatchBatchWork.Elapsed.Subtract(stopwatchLastDequeue.Elapsed)); stopwatchBatchWork = null; } } } catch (Exception jobEx) { State = WorkerState.Waiting; consecutiveErrors++; if (job != null) { ShouldRetryException retryEx = jobEx as ShouldRetryException; if (retryEx != null && (retryEx.MaxTimes > 0 && retries >= retryEx.MaxTimes)) { retryEx = null; } if (retryEx == null) { // job completed with errors, but we can move on try { Queue.Completed(this, job, true); } catch (Exception ex) { RoqueTrace.Source.Trace(TraceEventType.Error, "Error marking failed job as completed for worker {0}: {1}", Name, ex.Message, ex); } if (consecutiveErrors >= TooManyErrors) { RoqueTrace.Source.Trace(TraceEventType.Warning, "Too many errors on worker '{0}', picking next job in {1} seconds", Name, TooManyErrorsRetrySeconds); // too many errors, wait some time before picking next job Thread.Sleep(TimeSpan.FromSeconds(TooManyErrorsRetrySeconds)); } } else { retries++; shouldRetry = true; RoqueTrace.Source.Trace(TraceEventType.Information, "Retrying failed job on worker '{0}' in {1}", Name, retryEx.Delay); // wait some time before retrying the failed job if (retryEx.Delay.Ticks > 0) { Thread.Sleep(retryEx.Delay); } } } } firstTime = false; } IsStopRequested = false; } catch (Exception ex) { RoqueTrace.Source.Trace(TraceEventType.Error, "Error running worker {0}: {1}", Name, ex.Message, ex); } finally { State = WorkerState.Stopped; RoqueTrace.Source.Trace(TraceEventType.Information, "Worker stopped: {0}", Name); var handler = Stopped; if (handler != null) { Stopped(this, EventArgs.Empty); } } }
/// <summary> /// Sequentially pick and run jobs from the <see cref="Queue"/> until stop is requested. /// </summary> public void Work() { try { if (!_SubscribersRegistered) { Executor.Default.RegisterSubscribersForWorker(this); _SubscribersRegistered = true; } if (State == WorkerState.Waiting) { throw new Exception("This worker is already waiting for work"); } if (State == WorkerState.Working) { throw new Exception("This worker is already working"); } State = WorkerState.Waiting; if (RoqueTrace.Switch.TraceInfo) { Trace.TraceInformation("Worker started: " + Name); } bool attemptWipResume = true; int consecutiveErrors = 0; int retries = 0; bool firstTime = true; bool shouldRetry = false; Stopwatch stopwatchBatchWork = null; Stopwatch stopwatchLastDequeue = null; while (!IsStopRequested) { Job job = null; try { State = WorkerState.Waiting; stopwatchLastDequeue = new Stopwatch(); stopwatchLastDequeue.Start(); if (attemptWipResume) { job = Queue.GetInProgressJob(this); attemptWipResume = false; if (job == null) { if (RoqueTrace.Switch.TraceInfo) { Trace.TraceInformation("No pending jobs to resume for worker: " + Name); } } else { retries = 1; if (RoqueTrace.Switch.TraceInfo) { Trace.TraceInformation("Resuming pending job for worker: " + Name); } } } if (job == null) { if (shouldRetry) { job = Queue.GetInProgressJob(this); shouldRetry = false; if (RoqueTrace.Switch.TraceInfo) { Trace.TraceInformation(string.Format("Retry #'{0}'. Worker {1}, Method: {2}", retries, Name, job.Method)); } } else { job = Queue.Dequeue(this, firstTime ? 1 : 10); retries = 0; } } if (job != null) { if (stopwatchBatchWork == null) { stopwatchBatchWork = new Stopwatch(); stopwatchBatchWork.Start(); } State = WorkerState.Working; job.Execute(); consecutiveErrors = 0; try { Queue.Completed(this, job); } catch (Exception ex) { if (RoqueTrace.Switch.TraceError) { Trace.TraceError( "Error marking job as completed for worker " + Name + ": " + ex.Message, ex); } } } else { if (stopwatchBatchWork != null) { if (RoqueTrace.Switch.TraceInfo) { Trace.TraceInformation("Queue is empty. Worker {0} worked for: {1}", Name, stopwatchBatchWork.Elapsed.Subtract(stopwatchLastDequeue.Elapsed)); } stopwatchBatchWork = null; } } } catch (Exception jobEx) { State = WorkerState.Waiting; consecutiveErrors++; if (job != null) { ShouldRetryException retryEx = jobEx as ShouldRetryException; if (retryEx != null && (retryEx.MaxTimes > 0 && retries >= retryEx.MaxTimes)) { retryEx = null; } if (retryEx == null) { // job completed with errors, but we can move on try { Queue.Completed(this, job, true); } catch (Exception ex) { if (RoqueTrace.Switch.TraceError) { Trace.TraceError( "Error marking job as completed for worker " + Name + ": " + ex.Message, ex); } } if (consecutiveErrors >= TooManyErrors) { if (RoqueTrace.Switch.TraceInfo) { Trace.TraceInformation(string.Format("Too many errors on worker '{0}', picking next job in {1} seconds", Name, TooManyErrorsRetrySeconds)); } // too many errors, wait some time before picking next job Thread.Sleep(TimeSpan.FromSeconds(TooManyErrorsRetrySeconds)); } } else { retries++; shouldRetry = true; if (RoqueTrace.Switch.TraceInfo) { Trace.TraceInformation(string.Format("Retrying failed job on worker '{0}' in {1}", Name, retryEx.Delay)); } // wait some time before retrying the failed job if (retryEx.Delay.Ticks > 0) { Thread.Sleep(retryEx.Delay); } } } } firstTime = false; } } catch (Exception ex) { if (RoqueTrace.Switch.TraceError) { Trace.TraceError("Error running worker " + Name + ": " + ex.Message, ex); } } finally { State = WorkerState.Stopped; if (RoqueTrace.Switch.TraceInfo) { Trace.TraceInformation("Worker stopped: " + Name); } var handler = Stopped; if (handler != null) { Stopped(this, EventArgs.Empty); } } }