internal void RunLoop() { while (true) { bool working = false, needsRest = true; WorkingRecord record = null; IJob job = null; Exception ex = null; try { // Find out if we're supposed to be doing work. lock (this.statusLocker) { working = this.Status == WorkerStatus.Working; } if (working) { // Dequeue a new job to work on. record = this.DequeueRecord(); // If a record exists, we have work to do. if (record != null) { // Try to de-serialize a job. try { job = JobSerializer.Deserialize(record.JobType, record.Data); this.logger.Debug("Worker {0} ({1}) de-serialized a job instance for '{2}'.", this.name, this.id, record.JobType); } catch (Exception sx) { ex = sx; this.logger.Warn("Worker {0} ({1}) failed to de-serialize a job instane for '{2}'.", this.name, this.id, record.JobType); } // If we failed to de-serialize, fail the job. if (job == null) { HistoryRecord history = CreateHistory(record, HistoryStatus.Failed); if (ex != null) { history.Exception = new ExceptionXElement(ex).ToString(); } using (IRepository repository = this.repositoryFactory.Create()) { using (IDbTransaction transaction = repository.BeginTransaction()) { repository.DeleteWorking(record.Id.Value, transaction); history = repository.CreateHistory(history, transaction); transaction.Commit(); } } } else { // Update this instance's current record so we can interrupt // execution if necessary. lock (this.runLocker) { this.currentRecord = record; } // Execute the job. bool success = this.ExecuteJob(job, out ex); // Acquire the run lock and move the job from the working // state to the history state, including the execution results. lock (this.runLocker) { HistoryStatus status = HistoryStatus.Succeeded; string exceptionString = null; if (success) { this.logger.Info("Worker {0} ({1}) executed '{2}' successfully.", this.name, this.id, this.currentRecord.JobName); } else { if (ex as TimeoutException != null) { status = HistoryStatus.TimedOut; this.logger.Warn("Worker {0} ({1}) timed out '{2}'.", this.name, this.id, this.currentRecord.JobName); } else { status = HistoryStatus.Failed; if (ex != null) { exceptionString = new ExceptionXElement(ex).ToString(); } this.logger.Warn("Worker {0} ({1}) encountered an exception during execution of '{2}'.", this.name, this.id, this.currentRecord.JobName); } } HistoryRecord history = CreateHistory(this.currentRecord, status); history.Exception = exceptionString; using (IRepository repository = this.repositoryFactory.Create()) { using (IDbTransaction transaction = repository.BeginTransaction()) { repository.DeleteWorking(this.currentRecord.Id.Value, transaction); history = repository.CreateHistory(history, transaction); // Re-try? if ((status == HistoryStatus.Failed || status == HistoryStatus.Interrupted || status == HistoryStatus.TimedOut) && (job.Retries == 0 || job.Retries >= this.currentRecord.TryNumber)) { repository.CreateQueued(CreateQueueRetry(this.currentRecord), transaction); } transaction.Commit(); } } this.currentRecord = null; } } needsRest = false; } } else { break; } } catch (ThreadAbortException) { throw; } catch (Exception rx) { this.logger.Error(rx, "Exception thrown during the run loop for worker {0} ({1}).", this.name, this.id); } if (working) { // Take a breather real quick. if (needsRest) { this.logger.Debug("Worker {0} ({1}) is resting before trying to de-queue another job.", this.name, this.id); Thread.Sleep(this.heartbeat.Randomize()); } else { this.logger.Debug("Worker {0} ({1}) will immediately try to de-queue another job.", this.name, this.id); } } } }