/// <summary>
/// Prunes orphaned jobs assigned to this worker that for one reason or another didn't get marked as interrupted.
/// </summary>
internal void PruneOrphans()
{
long?currentId = null;
lock (this.runLocker)
{
currentId = this.currentRecord != null ? this.currentRecord.Id : null;
}
using (IRepository repository = this.repositoryFactory.Create())
{
using (IDbTransaction transaction = repository.BeginTransaction())
{
foreach (WorkingRecord working in repository.GetWorkingForWorker(this.Id, currentId, transaction))
{
if (working.Id != currentId)
{
HistoryRecord history = CreateHistory(working, HistoryStatus.Interrupted);
repository.DeleteWorking(working.Id.Value, transaction);
repository.CreateHistory(history, transaction);
}
}
transaction.Commit();
}
}
}
/// <summary>
/// Cancels the current job.
/// </summary>
internal void CancelCurrent()
{
lock (this.runLocker)
{
this.KillRunThread();
if (this.currentRecord != null)
{
this.logger.Info("Worker {0} ({1}) canceled '{2}'.", this.name, this.id, this.currentRecord.JobName);
using (IRepository repository = this.repositoryFactory.Create())
{
using (IDbTransaction transaction = repository.BeginTransaction())
{
HistoryRecord history = CreateHistory(this.currentRecord, HistoryStatus.Canceled);
repository.DeleteWorking(this.currentRecord.Id.Value, transaction);
history = repository.CreateHistory(history, transaction);
transaction.Commit();
}
}
this.currentRecord = null;
}
this.runThread = new Thread(this.RunLoop);
this.runThread.Name = "BlueCollar Run Thread";
this.runThread.Start();
}
}
internal void RunLoop()
{
while (true)
{
bool working = false, needsRest = true;
WorkingRecord record = null;
IJob job = null;
Exception ex = null;
try
{
// Find out if we're supposed to be doing work.
lock (this.statusLocker)
{
working = this.Status == WorkerStatus.Working;
}
if (working)
{
// Dequeue a new job to work on.
record = this.DequeueRecord();
// If a record exists, we have work to do.
if (record != null)
{
// Try to de-serialize a job.
try
{
job = JobSerializer.Deserialize(record.JobType, record.Data);
this.logger.Debug("Worker {0} ({1}) de-serialized a job instance for '{2}'.", this.name, this.id, record.JobType);
}
catch (Exception sx)
{
ex = sx;
this.logger.Warn("Worker {0} ({1}) failed to de-serialize a job instane for '{2}'.", this.name, this.id, record.JobType);
}
// If we failed to de-serialize, fail the job.
if (job == null)
{
HistoryRecord history = CreateHistory(record, HistoryStatus.Failed);
if (ex != null)
{
history.Exception = new ExceptionXElement(ex).ToString();
}
using (IRepository repository = this.repositoryFactory.Create())
{
using (IDbTransaction transaction = repository.BeginTransaction())
{
repository.DeleteWorking(record.Id.Value, transaction);
history = repository.CreateHistory(history, transaction);
transaction.Commit();
}
}
}
else
{
// Update this instance's current record so we can interrupt
// execution if necessary.
lock (this.runLocker)
{
this.currentRecord = record;
}
// Execute the job.
bool success = this.ExecuteJob(job, out ex);
// Acquire the run lock and move the job from the working
// state to the history state, including the execution results.
lock (this.runLocker)
{
HistoryStatus status = HistoryStatus.Succeeded;
string exceptionString = null;
if (success)
{
this.logger.Info("Worker {0} ({1}) executed '{2}' successfully.", this.name, this.id, this.currentRecord.JobName);
}
else
{
if (ex as TimeoutException != null)
{
status = HistoryStatus.TimedOut;
this.logger.Warn("Worker {0} ({1}) timed out '{2}'.", this.name, this.id, this.currentRecord.JobName);
}
else
{
status = HistoryStatus.Failed;
if (ex != null)
{
exceptionString = new ExceptionXElement(ex).ToString();
}
this.logger.Warn("Worker {0} ({1}) encountered an exception during execution of '{2}'.", this.name, this.id, this.currentRecord.JobName);
}
}
HistoryRecord history = CreateHistory(this.currentRecord, status);
history.Exception = exceptionString;
using (IRepository repository = this.repositoryFactory.Create())
{
using (IDbTransaction transaction = repository.BeginTransaction())
{
repository.DeleteWorking(this.currentRecord.Id.Value, transaction);
history = repository.CreateHistory(history, transaction);
// Re-try?
if ((status == HistoryStatus.Failed ||
status == HistoryStatus.Interrupted ||
status == HistoryStatus.TimedOut) &&
(job.Retries == 0 || job.Retries >= this.currentRecord.TryNumber))
{
repository.CreateQueued(CreateQueueRetry(this.currentRecord), transaction);
}
transaction.Commit();
}
}
this.currentRecord = null;
}
}
needsRest = false;
}
}
else
{
break;
}
}
catch (ThreadAbortException)
{
throw;
}
catch (Exception rx)
{
this.logger.Error(rx, "Exception thrown during the run loop for worker {0} ({1}).", this.name, this.id);
}
if (working)
{
// Take a breather real quick.
if (needsRest)
{
this.logger.Debug("Worker {0} ({1}) is resting before trying to de-queue another job.", this.name, this.id);
Thread.Sleep(this.heartbeat.Randomize());
}
else
{
this.logger.Debug("Worker {0} ({1}) will immediately try to de-queue another job.", this.name, this.id);
}
}
}
}
/// <summary>
/// Stops the worker.
/// </summary>
/// <param name="force">A value indicating whether to force the worker to stop, even if work will be abandoned.</param>
public void Stop(bool force)
{
if (this.disposed)
{
throw new ObjectDisposedException("Worker");
}
bool isWorking = false, isAlive = false;
lock (this.statusLocker)
{
if (this.Status == WorkerStatus.Working)
{
isWorking = true;
this.SetStatus(WorkerStatus.Stopping);
}
}
lock (this.runLocker)
{
isAlive = this.runThread != null && this.runThread.IsAlive;
}
if (isWorking && isAlive)
{
if (!force)
{
try
{
this.runThread.Join();
this.runThread = null;
}
catch (NullReferenceException)
{
}
}
else
{
this.KillRunThread();
}
}
lock (this.statusLocker)
{
lock (this.runLocker)
{
WorkingRecord record = this.currentRecord;
using (IRepository repository = this.repositoryFactory.Create())
{
using (IDbTransaction transaction = repository.BeginTransaction())
{
if (record != null)
{
HistoryRecord history = CreateHistory(record, HistoryStatus.Interrupted);
repository.DeleteWorking(record.Id.Value, transaction);
history = repository.CreateHistory(history, transaction);
}
this.SetStatus(WorkerStatus.Stopped, repository, transaction);
transaction.Commit();
}
}
this.currentRecord = null;
}
}
if (isWorking)
{
this.logger.Info("Worker {0} ({1}) has stopped.", this.name, this.id);
}
}
public HistoryRecord CreateHistory(HistoryRecord record, IDbTransaction transaction)
{
const string Sql =
@"INSERT INTO [BlueCollarHistory]([ApplicationName],[WorkerId],[ScheduleId],[QueueName],[JobName],[JobType],[Data],[QueuedOn],[TryNumber],[StartedOn],[Status],[Exception],[FinishedOn])
VALUES(@ApplicationName,@WorkerId,@ScheduleId,@QueueName,@JobName,@JobType,@Data,@QueuedOn,@TryNumber,@StartedOn,@StatusString,@Exception,@FinishedOn);
SELECT CAST(SCOPE_IDENTITY() AS bigint);";
record.Id = this.connection.Query<long>(
Sql,
record,
transaction,
true,
null,
null).First();
return record;
}
