/// <summary>
/// Sequentially pick and run jobs from the <see cref="Queue"/> until stop is requested.
/// </summary>
public void Work()
{
try
{
IsStopRequested = false;
if (State == WorkerState.Waiting)
{
throw new Exception("This worker is already waiting for work");
}
if (State == WorkerState.Working)
{
throw new Exception("This worker is already working");
}
State = WorkerState.Waiting;
RoqueTrace.Source.Trace(TraceEventType.Information, "Worker {0} started. Roque v{1}. AppDomain: {2}",
Name, new Func <object>(() => Assembly.GetAssembly(typeof(Worker)).GetName().Version), AppDomain.CurrentDomain.FriendlyName);
while (!_SubscribersRegistered && !IsStopRequested)
{
try
{
Executor.Default.RegisterSubscribersForWorker(this);
_SubscribersRegistered = true;
}
catch
{
// error registering subscriber, log is already done
RoqueTrace.Source.Trace(TraceEventType.Information, "Error registering subscribers, retrying in 10 seconds...");
Thread.Sleep(10000);
}
}
bool attemptWipResume = true;
int consecutiveErrors = 0;
int retries = 0;
bool firstTime = true;
bool shouldRetry = false;
Stopwatch stopwatchBatchWork = null;
Stopwatch stopwatchLastDequeue = null;
while (!IsStopRequested)
{
Job job = null;
try
{
State = WorkerState.Waiting;
stopwatchLastDequeue = new Stopwatch();
stopwatchLastDequeue.Start();
if (attemptWipResume)
{
job = Queue.GetInProgressJob(this);
attemptWipResume = false;
if (job == null)
{
RoqueTrace.Source.Trace(TraceEventType.Information, "No pending jobs to resume for worker: {0}", Name);
}
else
{
retries = 1;
RoqueTrace.Source.Trace(TraceEventType.Information, "Resuming pending job for worker: {0}", Name);
}
}
if (job == null)
{
if (shouldRetry)
{
job = Queue.GetInProgressJob(this);
shouldRetry = false;
RoqueTrace.Source.Trace(TraceEventType.Information, "Retry #'{0}'. Worker {1}, Method: {2}", retries, Name, job.Method);
}
else
{
job = Queue.Dequeue(this, firstTime ? 1 : 10);
retries = 0;
}
}
if (job != null)
{
if (stopwatchBatchWork == null)
{
stopwatchBatchWork = new Stopwatch();
stopwatchBatchWork.Start();
}
State = WorkerState.Working;
job.Execute();
consecutiveErrors = 0;
try
{
Queue.Completed(this, job);
}
catch (Exception ex)
{
RoqueTrace.Source.Trace(TraceEventType.Error, "Error marking job as completed for worker {0}: {1}", Name, ex.Message, ex);
}
}
else
{
if (stopwatchBatchWork != null)
{
RoqueTrace.Source.Trace(TraceEventType.Information, "Queue is empty. Worker {0} worked for: {1}", Name, stopwatchBatchWork.Elapsed.Subtract(stopwatchLastDequeue.Elapsed));
stopwatchBatchWork = null;
}
}
}
catch (Exception jobEx)
{
State = WorkerState.Waiting;
consecutiveErrors++;
if (job != null)
{
ShouldRetryException retryEx = jobEx as ShouldRetryException;
if (retryEx != null && (retryEx.MaxTimes > 0 && retries >= retryEx.MaxTimes))
{
retryEx = null;
}
if (retryEx == null)
{
// job completed with errors, but we can move on
try
{
Queue.Completed(this, job, true);
}
catch (Exception ex)
{
RoqueTrace.Source.Trace(TraceEventType.Error, "Error marking failed job as completed for worker {0}: {1}", Name, ex.Message, ex);
}
if (consecutiveErrors >= TooManyErrors)
{
RoqueTrace.Source.Trace(TraceEventType.Warning, "Too many errors on worker '{0}', picking next job in {1} seconds", Name, TooManyErrorsRetrySeconds);
// too many errors, wait some time before picking next job
Thread.Sleep(TimeSpan.FromSeconds(TooManyErrorsRetrySeconds));
}
}
else
{
retries++;
shouldRetry = true;
RoqueTrace.Source.Trace(TraceEventType.Information, "Retrying failed job on worker '{0}' in {1}", Name, retryEx.Delay);
// wait some time before retrying the failed job
if (retryEx.Delay.Ticks > 0)
{
Thread.Sleep(retryEx.Delay);
}
}
}
}
firstTime = false;
}
IsStopRequested = false;
}
catch (Exception ex)
{
RoqueTrace.Source.Trace(TraceEventType.Error, "Error running worker {0}: {1}", Name, ex.Message, ex);
}
finally
{
State = WorkerState.Stopped;
RoqueTrace.Source.Trace(TraceEventType.Information, "Worker stopped: {0}", Name);
var handler = Stopped;
if (handler != null)
{
Stopped(this, EventArgs.Empty);
}
}
}
/// <summary>
/// Sequentially pick and run jobs from the <see cref="Queue"/> until stop is requested.
/// </summary>
public void Work()
{
try
{
if (!_SubscribersRegistered)
{
Executor.Default.RegisterSubscribersForWorker(this);
_SubscribersRegistered = true;
}
if (State == WorkerState.Waiting)
{
throw new Exception("This worker is already waiting for work");
}
if (State == WorkerState.Working)
{
throw new Exception("This worker is already working");
}
State = WorkerState.Waiting;
if (RoqueTrace.Switch.TraceInfo)
{
Trace.TraceInformation("Worker started: " + Name);
}
bool attemptWipResume = true;
int consecutiveErrors = 0;
int retries = 0;
bool firstTime = true;
bool shouldRetry = false;
Stopwatch stopwatchBatchWork = null;
Stopwatch stopwatchLastDequeue = null;
while (!IsStopRequested)
{
Job job = null;
try
{
State = WorkerState.Waiting;
stopwatchLastDequeue = new Stopwatch();
stopwatchLastDequeue.Start();
if (attemptWipResume)
{
job = Queue.GetInProgressJob(this);
attemptWipResume = false;
if (job == null)
{
if (RoqueTrace.Switch.TraceInfo)
{
Trace.TraceInformation("No pending jobs to resume for worker: " + Name);
}
}
else
{
retries = 1;
if (RoqueTrace.Switch.TraceInfo)
{
Trace.TraceInformation("Resuming pending job for worker: " + Name);
}
}
}
if (job == null)
{
if (shouldRetry)
{
job = Queue.GetInProgressJob(this);
shouldRetry = false;
if (RoqueTrace.Switch.TraceInfo)
{
Trace.TraceInformation(string.Format("Retry #'{0}'. Worker {1}, Method: {2}", retries, Name, job.Method));
}
}
else
{
job = Queue.Dequeue(this, firstTime ? 1 : 10);
retries = 0;
}
}
if (job != null)
{
if (stopwatchBatchWork == null)
{
stopwatchBatchWork = new Stopwatch();
stopwatchBatchWork.Start();
}
State = WorkerState.Working;
job.Execute();
consecutiveErrors = 0;
try
{
Queue.Completed(this, job);
}
catch (Exception ex)
{
if (RoqueTrace.Switch.TraceError)
{
Trace.TraceError(
"Error marking job as completed for worker " + Name + ": " + ex.Message, ex);
}
}
}
else
{
if (stopwatchBatchWork != null)
{
if (RoqueTrace.Switch.TraceInfo)
{
Trace.TraceInformation("Queue is empty. Worker {0} worked for: {1}", Name,
stopwatchBatchWork.Elapsed.Subtract(stopwatchLastDequeue.Elapsed));
}
stopwatchBatchWork = null;
}
}
}
catch (Exception jobEx)
{
State = WorkerState.Waiting;
consecutiveErrors++;
if (job != null)
{
ShouldRetryException retryEx = jobEx as ShouldRetryException;
if (retryEx != null && (retryEx.MaxTimes > 0 && retries >= retryEx.MaxTimes))
{
retryEx = null;
}
if (retryEx == null)
{
// job completed with errors, but we can move on
try
{
Queue.Completed(this, job, true);
}
catch (Exception ex)
{
if (RoqueTrace.Switch.TraceError)
{
Trace.TraceError(
"Error marking job as completed for worker " + Name + ": " + ex.Message, ex);
}
}
if (consecutiveErrors >= TooManyErrors)
{
if (RoqueTrace.Switch.TraceInfo)
{
Trace.TraceInformation(string.Format("Too many errors on worker '{0}', picking next job in {1} seconds", Name, TooManyErrorsRetrySeconds));
}
// too many errors, wait some time before picking next job
Thread.Sleep(TimeSpan.FromSeconds(TooManyErrorsRetrySeconds));
}
}
else
{
retries++;
shouldRetry = true;
if (RoqueTrace.Switch.TraceInfo)
{
Trace.TraceInformation(string.Format("Retrying failed job on worker '{0}' in {1}", Name, retryEx.Delay));
}
// wait some time before retrying the failed job
if (retryEx.Delay.Ticks > 0)
{
Thread.Sleep(retryEx.Delay);
}
}
}
}
firstTime = false;
}
}
catch (Exception ex)
{
if (RoqueTrace.Switch.TraceError)
{
Trace.TraceError("Error running worker " + Name + ": " + ex.Message, ex);
}
}
finally
{
State = WorkerState.Stopped;
if (RoqueTrace.Switch.TraceInfo)
{
Trace.TraceInformation("Worker stopped: " + Name);
}
var handler = Stopped;
if (handler != null)
{
Stopped(this, EventArgs.Empty);
}
}
}