internal void RunLoop()
{
while (true)
{
bool working = false, needsRest = true;
WorkingRecord record = null;
IJob job = null;
Exception ex = null;
try
{
// Find out if we're supposed to be doing work.
lock (this.statusLocker)
{
working = this.Status == WorkerStatus.Working;
}
if (working)
{
// Dequeue a new job to work on.
record = this.DequeueRecord();
// If a record exists, we have work to do.
if (record != null)
{
// Try to de-serialize a job.
try
{
job = JobSerializer.Deserialize(record.JobType, record.Data);
this.logger.Debug("Worker {0} ({1}) de-serialized a job instance for '{2}'.", this.name, this.id, record.JobType);
}
catch (Exception sx)
{
ex = sx;
this.logger.Warn("Worker {0} ({1}) failed to de-serialize a job instane for '{2}'.", this.name, this.id, record.JobType);
}
// If we failed to de-serialize, fail the job.
if (job == null)
{
HistoryRecord history = CreateHistory(record, HistoryStatus.Failed);
if (ex != null)
{
history.Exception = new ExceptionXElement(ex).ToString();
}
using (IRepository repository = this.repositoryFactory.Create())
{
using (IDbTransaction transaction = repository.BeginTransaction())
{
repository.DeleteWorking(record.Id.Value, transaction);
history = repository.CreateHistory(history, transaction);
transaction.Commit();
}
}
}
else
{
// Update this instance's current record so we can interrupt
// execution if necessary.
lock (this.runLocker)
{
this.currentRecord = record;
}
// Execute the job.
bool success = this.ExecuteJob(job, out ex);
// Acquire the run lock and move the job from the working
// state to the history state, including the execution results.
lock (this.runLocker)
{
HistoryStatus status = HistoryStatus.Succeeded;
string exceptionString = null;
if (success)
{
this.logger.Info("Worker {0} ({1}) executed '{2}' successfully.", this.name, this.id, this.currentRecord.JobName);
}
else
{
if (ex as TimeoutException != null)
{
status = HistoryStatus.TimedOut;
this.logger.Warn("Worker {0} ({1}) timed out '{2}'.", this.name, this.id, this.currentRecord.JobName);
}
else
{
status = HistoryStatus.Failed;
if (ex != null)
{
exceptionString = new ExceptionXElement(ex).ToString();
}
this.logger.Warn("Worker {0} ({1}) encountered an exception during execution of '{2}'.", this.name, this.id, this.currentRecord.JobName);
}
}
HistoryRecord history = CreateHistory(this.currentRecord, status);
history.Exception = exceptionString;
using (IRepository repository = this.repositoryFactory.Create())
{
using (IDbTransaction transaction = repository.BeginTransaction())
{
repository.DeleteWorking(this.currentRecord.Id.Value, transaction);
history = repository.CreateHistory(history, transaction);
// Re-try?
if ((status == HistoryStatus.Failed ||
status == HistoryStatus.Interrupted ||
status == HistoryStatus.TimedOut) &&
(job.Retries == 0 || job.Retries >= this.currentRecord.TryNumber))
{
repository.CreateQueued(CreateQueueRetry(this.currentRecord), transaction);
}
transaction.Commit();
}
}
this.currentRecord = null;
}
}
needsRest = false;
}
}
else
{
break;
}
}
catch (ThreadAbortException)
{
throw;
}
catch (Exception rx)
{
this.logger.Error(rx, "Exception thrown during the run loop for worker {0} ({1}).", this.name, this.id);
}
if (working)
{
// Take a breather real quick.
if (needsRest)
{
this.logger.Debug("Worker {0} ({1}) is resting before trying to de-queue another job.", this.name, this.id);
Thread.Sleep(this.heartbeat.Randomize());
}
else
{
this.logger.Debug("Worker {0} ({1}) will immediately try to de-queue another job.", this.name, this.id);
}
}
}
}
internal void RunLoop()
{
while (true)
{
bool working = false, needsRest = true;
WorkingRecord record = null;
IJob job = null;
Exception ex = null;
try
{
// Find out if we're supposed to be doing work.
lock (this.statusLocker)
{
working = this.Status == WorkerStatus.Working;
}
if (working)
{
// Dequeue a new job to work on.
record = this.DequeueRecord();
// If a record exists, we have work to do.
if (record != null)
{
// Try to de-serialize a job.
try
{
job = JobSerializer.Deserialize(record.JobType, record.Data);
this.logger.Debug("Worker {0} ({1}) de-serialized a job instance for '{2}'.", this.name, this.id, record.JobType);
}
catch (Exception sx)
{
ex = sx;
this.logger.Warn("Worker {0} ({1}) failed to de-serialize a job instane for '{2}'.", this.name, this.id, record.JobType);
}
// If we failed to de-serialize, fail the job.
if (job == null)
{
HistoryRecord history = CreateHistory(record, HistoryStatus.Failed);
if (ex != null)
{
history.Exception = new ExceptionXElement(ex).ToString();
}
using (IRepository repository = this.repositoryFactory.Create())
{
using (IDbTransaction transaction = repository.BeginTransaction())
{
repository.DeleteWorking(record.Id.Value, transaction);
history = repository.CreateHistory(history, transaction);
transaction.Commit();
}
}
}
else
{
// Update this instance's current record so we can interrupt
// execution if necessary.
lock (this.runLocker)
{
this.currentRecord = record;
}
// Execute the job.
bool success = this.ExecuteJob(job, out ex);
// Acquire the run lock and move the job from the working
// state to the history state, including the execution results.
lock (this.runLocker)
{
HistoryStatus status = HistoryStatus.Succeeded;
string exceptionString = null;
if (success)
{
this.logger.Info("Worker {0} ({1}) executed '{2}' successfully.", this.name, this.id, this.currentRecord.JobName);
}
else
{
if (ex as TimeoutException != null)
{
status = HistoryStatus.TimedOut;
this.logger.Warn("Worker {0} ({1}) timed out '{2}'.", this.name, this.id, this.currentRecord.JobName);
}
else
{
status = HistoryStatus.Failed;
if (ex != null)
{
exceptionString = new ExceptionXElement(ex).ToString();
}
this.logger.Warn("Worker {0} ({1}) encountered an exception during execution of '{2}'.", this.name, this.id, this.currentRecord.JobName);
}
}
HistoryRecord history = CreateHistory(this.currentRecord, status);
history.Exception = exceptionString;
using (IRepository repository = this.repositoryFactory.Create())
{
using (IDbTransaction transaction = repository.BeginTransaction())
{
repository.DeleteWorking(this.currentRecord.Id.Value, transaction);
history = repository.CreateHistory(history, transaction);
// Re-try?
if ((status == HistoryStatus.Failed
|| status == HistoryStatus.Interrupted
|| status == HistoryStatus.TimedOut)
&& (job.Retries == 0 || job.Retries >= this.currentRecord.TryNumber))
{
repository.CreateQueued(CreateQueueRetry(this.currentRecord), transaction);
}
transaction.Commit();
}
}
this.currentRecord = null;
}
}
needsRest = false;
}
}
else
{
break;
}
}
catch (ThreadAbortException)
{
throw;
}
catch (Exception rx)
{
this.logger.Error(rx, "Exception thrown during the run loop for worker {0} ({1}).", this.name, this.id);
}
if (working)
{
// Take a breather real quick.
if (needsRest)
{
this.logger.Debug("Worker {0} ({1}) is resting before trying to de-queue another job.", this.name, this.id);
Thread.Sleep(this.heartbeat.Randomize());
}
else
{
this.logger.Debug("Worker {0} ({1}) will immediately try to de-queue another job.", this.name, this.id);
}
}
}
}
