/// <summary>
/// Dequeues a job to do work on.
/// </summary>
/// <returns>A working record representing the dequeued job.</returns>
internal WorkingRecord DequeueRecord()
{
WorkingRecord working = null;
QueueNameFilters queues = null;
lock (this.runLocker)
{
queues = new QueueNameFilters(this.queueFilters.Include, this.queueFilters.Exclude);
}
using (IRepository repository = this.repositoryFactory.Create())
{
using (IDbTransaction transaction = repository.BeginTransaction(IsolationLevel.RepeatableRead))
{
QueueRecord queued = repository.GetQueued(this.applicationName, queues, DateTime.UtcNow, transaction);
if (queued != null)
{
working = CreateWorking(queued, this.id, queued.ScheduleId, DateTime.UtcNow);
repository.DeleteQueued(queued.Id.Value, transaction);
working = repository.CreateWorking(working, transaction);
this.logger.Info("Worker {0} ({1}) dequeued '{2}'.", this.name, this.id, queued.JobName);
}
transaction.Commit();
}
}
return(working);
}
/// <summary>
/// Cancels the current job.
/// </summary>
internal void CancelCurrent()
{
lock (this.runLocker)
{
this.KillRunThread();
if (this.currentRecord != null)
{
this.logger.Info("Worker {0} ({1}) canceled '{2}'.", this.name, this.id, this.currentRecord.JobName);
using (IRepository repository = this.repositoryFactory.Create())
{
using (IDbTransaction transaction = repository.BeginTransaction())
{
HistoryRecord history = CreateHistory(this.currentRecord, HistoryStatus.Canceled);
repository.DeleteWorking(this.currentRecord.Id.Value, transaction);
history = repository.CreateHistory(history, transaction);
transaction.Commit();
}
}
this.currentRecord = null;
}
this.runThread = new Thread(this.RunLoop);
this.runThread.Name = "BlueCollar Run Thread";
this.runThread.Start();
}
}
/// <summary>
/// Creates a queue retry record from the given working record.
/// </summary>
/// <param name="working">The working record to create the queue record from.</param>
/// <returns>A queue record.</returns>
internal static QueueRecord CreateQueueRetry(WorkingRecord working)
{
return(new QueueRecord()
{
ApplicationName = working.ApplicationName,
Data = working.Data,
JobName = working.JobName,
JobType = working.JobType,
QueuedOn = DateTime.UtcNow,
QueueName = working.QueueName,
ScheduleId = working.ScheduleId,
TryNumber = working.TryNumber + 1
});
}
/// <summary>
/// Creates a history record from the given working record and status.
/// </summary>
/// <param name="working">The working record to create the history record for.</param>
/// <param name="status">The status to create the history record with.</param>
/// <returns>A history record.</returns>
internal static HistoryRecord CreateHistory(WorkingRecord working, HistoryStatus status)
{
return(new HistoryRecord()
{
ApplicationName = working.ApplicationName,
Data = working.Data,
FinishedOn = DateTime.UtcNow,
JobName = working.JobName,
JobType = working.JobType,
QueuedOn = working.QueuedOn,
QueueName = working.QueueName,
ScheduleId = working.ScheduleId,
StartedOn = working.StartedOn,
Status = status,
TryNumber = working.TryNumber,
WorkerId = working.WorkerId
});
}
internal void RunLoop()
{
while (true)
{
bool working = false, needsRest = true;
WorkingRecord record = null;
IJob job = null;
Exception ex = null;
try
{
// Find out if we're supposed to be doing work.
lock (this.statusLocker)
{
working = this.Status == WorkerStatus.Working;
}
if (working)
{
// Dequeue a new job to work on.
record = this.DequeueRecord();
// If a record exists, we have work to do.
if (record != null)
{
// Try to de-serialize a job.
try
{
job = JobSerializer.Deserialize(record.JobType, record.Data);
this.logger.Debug("Worker {0} ({1}) de-serialized a job instance for '{2}'.", this.name, this.id, record.JobType);
}
catch (Exception sx)
{
ex = sx;
this.logger.Warn("Worker {0} ({1}) failed to de-serialize a job instane for '{2}'.", this.name, this.id, record.JobType);
}
// If we failed to de-serialize, fail the job.
if (job == null)
{
HistoryRecord history = CreateHistory(record, HistoryStatus.Failed);
if (ex != null)
{
history.Exception = new ExceptionXElement(ex).ToString();
}
using (IRepository repository = this.repositoryFactory.Create())
{
using (IDbTransaction transaction = repository.BeginTransaction())
{
repository.DeleteWorking(record.Id.Value, transaction);
history = repository.CreateHistory(history, transaction);
transaction.Commit();
}
}
}
else
{
// Update this instance's current record so we can interrupt
// execution if necessary.
lock (this.runLocker)
{
this.currentRecord = record;
}
// Execute the job.
bool success = this.ExecuteJob(job, out ex);
// Acquire the run lock and move the job from the working
// state to the history state, including the execution results.
lock (this.runLocker)
{
HistoryStatus status = HistoryStatus.Succeeded;
string exceptionString = null;
if (success)
{
this.logger.Info("Worker {0} ({1}) executed '{2}' successfully.", this.name, this.id, this.currentRecord.JobName);
}
else
{
if (ex as TimeoutException != null)
{
status = HistoryStatus.TimedOut;
this.logger.Warn("Worker {0} ({1}) timed out '{2}'.", this.name, this.id, this.currentRecord.JobName);
}
else
{
status = HistoryStatus.Failed;
if (ex != null)
{
exceptionString = new ExceptionXElement(ex).ToString();
}
this.logger.Warn("Worker {0} ({1}) encountered an exception during execution of '{2}'.", this.name, this.id, this.currentRecord.JobName);
}
}
HistoryRecord history = CreateHistory(this.currentRecord, status);
history.Exception = exceptionString;
using (IRepository repository = this.repositoryFactory.Create())
{
using (IDbTransaction transaction = repository.BeginTransaction())
{
repository.DeleteWorking(this.currentRecord.Id.Value, transaction);
history = repository.CreateHistory(history, transaction);
// Re-try?
if ((status == HistoryStatus.Failed ||
status == HistoryStatus.Interrupted ||
status == HistoryStatus.TimedOut) &&
(job.Retries == 0 || job.Retries >= this.currentRecord.TryNumber))
{
repository.CreateQueued(CreateQueueRetry(this.currentRecord), transaction);
}
transaction.Commit();
}
}
this.currentRecord = null;
}
}
needsRest = false;
}
}
else
{
break;
}
}
catch (ThreadAbortException)
{
throw;
}
catch (Exception rx)
{
this.logger.Error(rx, "Exception thrown during the run loop for worker {0} ({1}).", this.name, this.id);
}
if (working)
{
// Take a breather real quick.
if (needsRest)
{
this.logger.Debug("Worker {0} ({1}) is resting before trying to de-queue another job.", this.name, this.id);
Thread.Sleep(this.heartbeat.Randomize());
}
else
{
this.logger.Debug("Worker {0} ({1}) will immediately try to de-queue another job.", this.name, this.id);
}
}
}
}
/// <summary>
/// Stops the worker.
/// </summary>
/// <param name="force">A value indicating whether to force the worker to stop, even if work will be abandoned.</param>
public void Stop(bool force)
{
if (this.disposed)
{
throw new ObjectDisposedException("Worker");
}
bool isWorking = false, isAlive = false;
lock (this.statusLocker)
{
if (this.Status == WorkerStatus.Working)
{
isWorking = true;
this.SetStatus(WorkerStatus.Stopping);
}
}
lock (this.runLocker)
{
isAlive = this.runThread != null && this.runThread.IsAlive;
}
if (isWorking && isAlive)
{
if (!force)
{
try
{
this.runThread.Join();
this.runThread = null;
}
catch (NullReferenceException)
{
}
}
else
{
this.KillRunThread();
}
}
lock (this.statusLocker)
{
lock (this.runLocker)
{
WorkingRecord record = this.currentRecord;
using (IRepository repository = this.repositoryFactory.Create())
{
using (IDbTransaction transaction = repository.BeginTransaction())
{
if (record != null)
{
HistoryRecord history = CreateHistory(record, HistoryStatus.Interrupted);
repository.DeleteWorking(record.Id.Value, transaction);
history = repository.CreateHistory(history, transaction);
}
this.SetStatus(WorkerStatus.Stopped, repository, transaction);
transaction.Commit();
}
}
this.currentRecord = null;
}
}
if (isWorking)
{
this.logger.Info("Worker {0} ({1}) has stopped.", this.name, this.id);
}
}
/// <summary>
/// Updates the given working record.
/// </summary>
/// <param name="record">The working record to update.</param>
/// <param name="transaction">The transaction to use, if applicable.</param>
/// <returns>The updated working record.</returns>
public WorkingRecord UpdateWorking(WorkingRecord record, IDbTransaction transaction)
{
const string Sql =
@"UPDATE [BlueCollarWorking]
SET
[WorkerId] = @WorkerId,
[ScheduleId] = @ScheduleId,
[QueueName] = @QueueName,
[JobName] = @JobName,
[JobType] = @JobType,
[Data] = @Data,
[QueuedOn] = @QueuedOn,
[TryNumber] = @TryNumber,
[StartedOn] = @StartedOn,
[Signal] = @Signal
WHERE
[Id] = @Id;";
this.connection.Execute(
Sql,
record,
transaction,
null,
null);
return record;
}
public WorkingRecord CreateWorking(WorkingRecord record, IDbTransaction transaction)
{
const string Sql =
@"INSERT INTO [BlueCollarWorking]([ApplicationName],[WorkerId],[ScheduleId],[QueueName],[JobName],[JobType],[Data],[QueuedOn],[TryNumber],[StartedOn],[Signal])
VALUES(@ApplicationName,@WorkerId,@ScheduleId,@QueueName,@JobName,@JobType,@Data,@QueuedOn,@TryNumber,@StartedOn,@SignalString);
SELECT CAST(SCOPE_IDENTITY() AS bigint);";
record.Id = this.connection.Query<long>(
Sql,
record,
transaction,
true,
null,
null).First();
return record;
}
internal void RunLoop()
{
while (true)
{
bool working = false, needsRest = true;
WorkingRecord record = null;
IJob job = null;
Exception ex = null;
try
{
// Find out if we're supposed to be doing work.
lock (this.statusLocker)
{
working = this.Status == WorkerStatus.Working;
}
if (working)
{
// Dequeue a new job to work on.
record = this.DequeueRecord();
// If a record exists, we have work to do.
if (record != null)
{
// Try to de-serialize a job.
try
{
job = JobSerializer.Deserialize(record.JobType, record.Data);
this.logger.Debug("Worker {0} ({1}) de-serialized a job instance for '{2}'.", this.name, this.id, record.JobType);
}
catch (Exception sx)
{
ex = sx;
this.logger.Warn("Worker {0} ({1}) failed to de-serialize a job instane for '{2}'.", this.name, this.id, record.JobType);
}
// If we failed to de-serialize, fail the job.
if (job == null)
{
HistoryRecord history = CreateHistory(record, HistoryStatus.Failed);
if (ex != null)
{
history.Exception = new ExceptionXElement(ex).ToString();
}
using (IRepository repository = this.repositoryFactory.Create())
{
using (IDbTransaction transaction = repository.BeginTransaction())
{
repository.DeleteWorking(record.Id.Value, transaction);
history = repository.CreateHistory(history, transaction);
transaction.Commit();
}
}
}
else
{
// Update this instance's current record so we can interrupt
// execution if necessary.
lock (this.runLocker)
{
this.currentRecord = record;
}
// Execute the job.
bool success = this.ExecuteJob(job, out ex);
// Acquire the run lock and move the job from the working
// state to the history state, including the execution results.
lock (this.runLocker)
{
HistoryStatus status = HistoryStatus.Succeeded;
string exceptionString = null;
if (success)
{
this.logger.Info("Worker {0} ({1}) executed '{2}' successfully.", this.name, this.id, this.currentRecord.JobName);
}
else
{
if (ex as TimeoutException != null)
{
status = HistoryStatus.TimedOut;
this.logger.Warn("Worker {0} ({1}) timed out '{2}'.", this.name, this.id, this.currentRecord.JobName);
}
else
{
status = HistoryStatus.Failed;
if (ex != null)
{
exceptionString = new ExceptionXElement(ex).ToString();
}
this.logger.Warn("Worker {0} ({1}) encountered an exception during execution of '{2}'.", this.name, this.id, this.currentRecord.JobName);
}
}
HistoryRecord history = CreateHistory(this.currentRecord, status);
history.Exception = exceptionString;
using (IRepository repository = this.repositoryFactory.Create())
{
using (IDbTransaction transaction = repository.BeginTransaction())
{
repository.DeleteWorking(this.currentRecord.Id.Value, transaction);
history = repository.CreateHistory(history, transaction);
// Re-try?
if ((status == HistoryStatus.Failed
|| status == HistoryStatus.Interrupted
|| status == HistoryStatus.TimedOut)
&& (job.Retries == 0 || job.Retries >= this.currentRecord.TryNumber))
{
repository.CreateQueued(CreateQueueRetry(this.currentRecord), transaction);
}
transaction.Commit();
}
}
this.currentRecord = null;
}
}
needsRest = false;
}
}
else
{
break;
}
}
catch (ThreadAbortException)
{
throw;
}
catch (Exception rx)
{
this.logger.Error(rx, "Exception thrown during the run loop for worker {0} ({1}).", this.name, this.id);
}
if (working)
{
// Take a breather real quick.
if (needsRest)
{
this.logger.Debug("Worker {0} ({1}) is resting before trying to de-queue another job.", this.name, this.id);
Thread.Sleep(this.heartbeat.Randomize());
}
else
{
this.logger.Debug("Worker {0} ({1}) will immediately try to de-queue another job.", this.name, this.id);
}
}
}
}
/// <summary>
/// Cancels the current job.
/// </summary>
internal void CancelCurrent()
{
lock (this.runLocker)
{
this.KillRunThread();
if (this.currentRecord != null)
{
this.logger.Info("Worker {0} ({1}) canceled '{2}'.", this.name, this.id, this.currentRecord.JobName);
using (IRepository repository = this.repositoryFactory.Create())
{
using (IDbTransaction transaction = repository.BeginTransaction())
{
HistoryRecord history = CreateHistory(this.currentRecord, HistoryStatus.Canceled);
repository.DeleteWorking(this.currentRecord.Id.Value, transaction);
history = repository.CreateHistory(history, transaction);
transaction.Commit();
}
}
this.currentRecord = null;
}
this.runThread = new Thread(this.RunLoop);
this.runThread.Name = "BlueCollar Run Thread";
this.runThread.Start();
}
}
/// <summary>
/// Creates a queue retry record from the given working record.
/// </summary>
/// <param name="working">The working record to create the queue record from.</param>
/// <returns>A queue record.</returns>
internal static QueueRecord CreateQueueRetry(WorkingRecord working)
{
return new QueueRecord()
{
ApplicationName = working.ApplicationName,
Data = working.Data,
JobName = working.JobName,
JobType = working.JobType,
QueuedOn = DateTime.UtcNow,
QueueName = working.QueueName,
ScheduleId = working.ScheduleId,
TryNumber = working.TryNumber + 1
};
}
/// <summary>
/// Creates a history record from the given working record and status.
/// </summary>
/// <param name="working">The working record to create the history record for.</param>
/// <param name="status">The status to create the history record with.</param>
/// <returns>A history record.</returns>
internal static HistoryRecord CreateHistory(WorkingRecord working, HistoryStatus status)
{
return new HistoryRecord()
{
ApplicationName = working.ApplicationName,
Data = working.Data,
FinishedOn = DateTime.UtcNow,
JobName = working.JobName,
JobType = working.JobType,
QueuedOn = working.QueuedOn,
QueueName = working.QueueName,
ScheduleId = working.ScheduleId,
StartedOn = working.StartedOn,
Status = status,
TryNumber = working.TryNumber,
WorkerId = working.WorkerId
};
}
/// <summary>
/// Stops the worker.
/// </summary>
/// <param name="force">A value indicating whether to force the worker to stop, even if work will be abandoned.</param>
public void Stop(bool force)
{
if (this.disposed)
{
throw new ObjectDisposedException("Worker");
}
bool isWorking = false, isAlive = false;
lock (this.statusLocker)
{
if (this.Status == WorkerStatus.Working)
{
isWorking = true;
this.SetStatus(WorkerStatus.Stopping);
}
}
lock (this.runLocker)
{
isAlive = this.runThread != null && this.runThread.IsAlive;
}
if (isWorking && isAlive)
{
if (!force)
{
try
{
this.runThread.Join();
this.runThread = null;
}
catch (NullReferenceException)
{
}
}
else
{
this.KillRunThread();
}
}
lock (this.statusLocker)
{
lock (this.runLocker)
{
WorkingRecord record = this.currentRecord;
using (IRepository repository = this.repositoryFactory.Create())
{
using (IDbTransaction transaction = repository.BeginTransaction())
{
if (record != null)
{
HistoryRecord history = CreateHistory(record, HistoryStatus.Interrupted);
repository.DeleteWorking(record.Id.Value, transaction);
history = repository.CreateHistory(history, transaction);
}
this.SetStatus(WorkerStatus.Stopped, repository, transaction);
transaction.Commit();
}
}
this.currentRecord = null;
}
}
if (isWorking)
{
this.logger.Info("Worker {0} ({1}) has stopped.", this.name, this.id);
}
}
