/// <summary>
/// Enqueues a job for the given application name and queue name.
/// </summary>
/// <param name="applicationName">The name of the application to enqueue the job for.</param>
/// <param name="queueName">The name of the queue to enqueue the job on, or null for the default queue.</param>
/// <param name="repository">The repository to use when enqueueing the job record.</param>
public virtual void Enqueue(string applicationName, string queueName, IRepository repository)
{
if (string.IsNullOrEmpty(applicationName))
{
throw new ArgumentNullException("applicationName", "applicationName must contain a value.");
}
if (string.IsNullOrEmpty(queueName))
{
queueName = "*";
}
if (repository == null)
{
throw new ArgumentNullException("repository", "repository cannot be null.");
}
QueueRecord record = new QueueRecord()
{
ApplicationName = applicationName,
Data = JobSerializer.Serialize(this),
JobName = this.Name,
JobType = JobSerializer.GetTypeName(this),
QueuedOn = DateTime.UtcNow,
QueueName = queueName,
TryNumber = 1
};
repository.CreateQueued(record, null);
}
internal void RunLoop()
{
while (true)
{
bool working = false, needsRest = true;
WorkingRecord record = null;
IJob job = null;
Exception ex = null;
try
{
// Find out if we're supposed to be doing work.
lock (this.statusLocker)
{
working = this.Status == WorkerStatus.Working;
}
if (working)
{
// Dequeue a new job to work on.
record = this.DequeueRecord();
// If a record exists, we have work to do.
if (record != null)
{
// Try to de-serialize a job.
try
{
job = JobSerializer.Deserialize(record.JobType, record.Data);
this.logger.Debug("Worker {0} ({1}) de-serialized a job instance for '{2}'.", this.name, this.id, record.JobType);
}
catch (Exception sx)
{
ex = sx;
this.logger.Warn("Worker {0} ({1}) failed to de-serialize a job instane for '{2}'.", this.name, this.id, record.JobType);
}
// If we failed to de-serialize, fail the job.
if (job == null)
{
HistoryRecord history = CreateHistory(record, HistoryStatus.Failed);
if (ex != null)
{
history.Exception = new ExceptionXElement(ex).ToString();
}
using (IRepository repository = this.repositoryFactory.Create())
{
using (IDbTransaction transaction = repository.BeginTransaction())
{
repository.DeleteWorking(record.Id.Value, transaction);
history = repository.CreateHistory(history, transaction);
transaction.Commit();
}
}
}
else
{
// Update this instance's current record so we can interrupt
// execution if necessary.
lock (this.runLocker)
{
this.currentRecord = record;
}
// Execute the job.
bool success = this.ExecuteJob(job, out ex);
// Acquire the run lock and move the job from the working
// state to the history state, including the execution results.
lock (this.runLocker)
{
HistoryStatus status = HistoryStatus.Succeeded;
string exceptionString = null;
if (success)
{
this.logger.Info("Worker {0} ({1}) executed '{2}' successfully.", this.name, this.id, this.currentRecord.JobName);
}
else
{
if (ex as TimeoutException != null)
{
status = HistoryStatus.TimedOut;
this.logger.Warn("Worker {0} ({1}) timed out '{2}'.", this.name, this.id, this.currentRecord.JobName);
}
else
{
status = HistoryStatus.Failed;
if (ex != null)
{
exceptionString = new ExceptionXElement(ex).ToString();
}
this.logger.Warn("Worker {0} ({1}) encountered an exception during execution of '{2}'.", this.name, this.id, this.currentRecord.JobName);
}
}
HistoryRecord history = CreateHistory(this.currentRecord, status);
history.Exception = exceptionString;
using (IRepository repository = this.repositoryFactory.Create())
{
using (IDbTransaction transaction = repository.BeginTransaction())
{
repository.DeleteWorking(this.currentRecord.Id.Value, transaction);
history = repository.CreateHistory(history, transaction);
// Re-try?
if ((status == HistoryStatus.Failed ||
status == HistoryStatus.Interrupted ||
status == HistoryStatus.TimedOut) &&
(job.Retries == 0 || job.Retries >= this.currentRecord.TryNumber))
{
repository.CreateQueued(CreateQueueRetry(this.currentRecord), transaction);
}
transaction.Commit();
}
}
this.currentRecord = null;
}
}
needsRest = false;
}
}
else
{
break;
}
}
catch (ThreadAbortException)
{
throw;
}
catch (Exception rx)
{
this.logger.Error(rx, "Exception thrown during the run loop for worker {0} ({1}).", this.name, this.id);
}
if (working)
{
// Take a breather real quick.
if (needsRest)
{
this.logger.Debug("Worker {0} ({1}) is resting before trying to de-queue another job.", this.name, this.id);
Thread.Sleep(this.heartbeat.Randomize());
}
else
{
this.logger.Debug("Worker {0} ({1}) will immediately try to de-queue another job.", this.name, this.id);
}
}
}
}
public void EnqueueScheduledJobs()
{
// Create the schedule window.
DateTime end = DateTime.UtcNow.FloorWithSeconds();
DateTime begin = this.LastEnqueuedOn != null ? this.LastEnqueuedOn.Value : end.AddSeconds(-1 * this.heartbeat);
this.LastEnqueuedOn = end;
using (IRepository repository = this.repositoryFactory.Create())
{
// Ensure the schedules have been loaded.
if (this.lastRefreshOn == null)
{
this.RefreshSchedules(repository, null);
}
}
foreach (ScheduleRecord schedule in this.Schedules)
{
bool hasEnqueueingLock = false;
using (IRepository repository = this.repositoryFactory.Create())
{
using (IDbTransaction transaction = repository.BeginTransaction(IsolationLevel.RepeatableRead))
{
try
{
hasEnqueueingLock = repository.GetScheduleEnqueueingLock(schedule.Id.Value, DateTime.UtcNow.AddMinutes(-1), transaction);
transaction.Commit();
}
catch
{
transaction.Rollback();
throw;
}
}
}
if (hasEnqueueingLock)
{
DateTime?scheduleDate;
try
{
if (this.CanScheduleBeEnqueued(schedule, begin, end, out scheduleDate))
{
List <QueueRecord> queues = new List <QueueRecord>();
List <HistoryRecord> histories = new List <HistoryRecord>();
foreach (ScheduledJobRecord scheduledJob in schedule.ScheduledJobs)
{
IJob job = null;
Exception ex = null;
// Try do de-serialize the job. We do this because we need the job name,
// and also just to save the queue bandwidth in case in can't be de-serialized.
try
{
job = JobSerializer.Deserialize(scheduledJob.JobType, scheduledJob.Data);
this.logger.Debug("Scheduler de-serialized scheduled job instance for '{0}' for schedule '{1}'.", scheduledJob.JobType, schedule.Name);
}
catch (Exception sx)
{
ex = sx;
this.logger.Warn("Scheduler failed to de-serialize scheduled job instance for '{0}' for schedule '{1}'.", scheduledJob.JobType, schedule.Name);
}
if (job != null)
{
queues.Add(
new QueueRecord()
{
ApplicationName = this.applicationName,
Data = scheduledJob.Data,
JobName = job.Name,
JobType = scheduledJob.JobType,
QueuedOn = scheduleDate.Value,
QueueName = schedule.QueueName,
ScheduleId = schedule.Id,
TryNumber = 1
});
}
else
{
histories.Add(
new HistoryRecord()
{
ApplicationName = this.applicationName,
Data = scheduledJob.Data,
Exception = ex != null ? new ExceptionXElement(ex).ToString() : null,
FinishedOn = scheduleDate.Value,
JobName = null,
JobType = scheduledJob.JobType,
QueuedOn = scheduleDate.Value,
QueueName = schedule.QueueName,
ScheduleId = schedule.Id,
StartedOn = scheduleDate.Value,
Status = HistoryStatus.Failed,
TryNumber = 1,
WorkerId = this.workerId
});
}
}
if (queues.Count > 0 || histories.Count > 0)
{
using (IRepository repository = this.repositoryFactory.Create())
{
repository.CreateQueuedAndHistoryForSchedule(schedule.Id.Value, scheduleDate.Value, queues, histories, null);
}
this.logger.Debug("Scheduler created {0} queued jobs and {1} failed history jobs for schedule '{2}'.", queues.Count, histories.Count, schedule.Name);
}
}
}
finally
{
using (IRepository repository = this.repositoryFactory.Create())
{
repository.ReleaseScheduleEnqueueingLock(schedule.Id.Value, null);
}
}
}
}
}