/// <summary> /// Prunes orphaned jobs assigned to this worker that for one reason or another didn't get marked as interrupted. /// </summary> internal void PruneOrphans() { long?currentId = null; lock (this.runLocker) { currentId = this.currentRecord != null ? this.currentRecord.Id : null; } using (IRepository repository = this.repositoryFactory.Create()) { using (IDbTransaction transaction = repository.BeginTransaction()) { foreach (WorkingRecord working in repository.GetWorkingForWorker(this.Id, currentId, transaction)) { if (working.Id != currentId) { HistoryRecord history = CreateHistory(working, HistoryStatus.Interrupted); repository.DeleteWorking(working.Id.Value, transaction); repository.CreateHistory(history, transaction); } } transaction.Commit(); } } }
/// <summary> /// Cancels the current job. /// </summary> internal void CancelCurrent() { lock (this.runLocker) { this.KillRunThread(); if (this.currentRecord != null) { this.logger.Info("Worker {0} ({1}) canceled '{2}'.", this.name, this.id, this.currentRecord.JobName); using (IRepository repository = this.repositoryFactory.Create()) { using (IDbTransaction transaction = repository.BeginTransaction()) { HistoryRecord history = CreateHistory(this.currentRecord, HistoryStatus.Canceled); repository.DeleteWorking(this.currentRecord.Id.Value, transaction); history = repository.CreateHistory(history, transaction); transaction.Commit(); } } this.currentRecord = null; } this.runThread = new Thread(this.RunLoop); this.runThread.Name = "BlueCollar Run Thread"; this.runThread.Start(); } }
internal void RunLoop() { while (true) { bool working = false, needsRest = true; WorkingRecord record = null; IJob job = null; Exception ex = null; try { // Find out if we're supposed to be doing work. lock (this.statusLocker) { working = this.Status == WorkerStatus.Working; } if (working) { // Dequeue a new job to work on. record = this.DequeueRecord(); // If a record exists, we have work to do. if (record != null) { // Try to de-serialize a job. try { job = JobSerializer.Deserialize(record.JobType, record.Data); this.logger.Debug("Worker {0} ({1}) de-serialized a job instance for '{2}'.", this.name, this.id, record.JobType); } catch (Exception sx) { ex = sx; this.logger.Warn("Worker {0} ({1}) failed to de-serialize a job instane for '{2}'.", this.name, this.id, record.JobType); } // If we failed to de-serialize, fail the job. if (job == null) { HistoryRecord history = CreateHistory(record, HistoryStatus.Failed); if (ex != null) { history.Exception = new ExceptionXElement(ex).ToString(); } using (IRepository repository = this.repositoryFactory.Create()) { using (IDbTransaction transaction = repository.BeginTransaction()) { repository.DeleteWorking(record.Id.Value, transaction); history = repository.CreateHistory(history, transaction); transaction.Commit(); } } } else { // Update this instance's current record so we can interrupt // execution if necessary. lock (this.runLocker) { this.currentRecord = record; } // Execute the job. bool success = this.ExecuteJob(job, out ex); // Acquire the run lock and move the job from the working // state to the history state, including the execution results. lock (this.runLocker) { HistoryStatus status = HistoryStatus.Succeeded; string exceptionString = null; if (success) { this.logger.Info("Worker {0} ({1}) executed '{2}' successfully.", this.name, this.id, this.currentRecord.JobName); } else { if (ex as TimeoutException != null) { status = HistoryStatus.TimedOut; this.logger.Warn("Worker {0} ({1}) timed out '{2}'.", this.name, this.id, this.currentRecord.JobName); } else { status = HistoryStatus.Failed; if (ex != null) { exceptionString = new ExceptionXElement(ex).ToString(); } this.logger.Warn("Worker {0} ({1}) encountered an exception during execution of '{2}'.", this.name, this.id, this.currentRecord.JobName); } } HistoryRecord history = CreateHistory(this.currentRecord, status); history.Exception = exceptionString; using (IRepository repository = this.repositoryFactory.Create()) { using (IDbTransaction transaction = repository.BeginTransaction()) { repository.DeleteWorking(this.currentRecord.Id.Value, transaction); history = repository.CreateHistory(history, transaction); // Re-try? if ((status == HistoryStatus.Failed || status == HistoryStatus.Interrupted || status == HistoryStatus.TimedOut) && (job.Retries == 0 || job.Retries >= this.currentRecord.TryNumber)) { repository.CreateQueued(CreateQueueRetry(this.currentRecord), transaction); } transaction.Commit(); } } this.currentRecord = null; } } needsRest = false; } } else { break; } } catch (ThreadAbortException) { throw; } catch (Exception rx) { this.logger.Error(rx, "Exception thrown during the run loop for worker {0} ({1}).", this.name, this.id); } if (working) { // Take a breather real quick. if (needsRest) { this.logger.Debug("Worker {0} ({1}) is resting before trying to de-queue another job.", this.name, this.id); Thread.Sleep(this.heartbeat.Randomize()); } else { this.logger.Debug("Worker {0} ({1}) will immediately try to de-queue another job.", this.name, this.id); } } } }
/// <summary> /// Stops the worker. /// </summary> /// <param name="force">A value indicating whether to force the worker to stop, even if work will be abandoned.</param> public void Stop(bool force) { if (this.disposed) { throw new ObjectDisposedException("Worker"); } bool isWorking = false, isAlive = false; lock (this.statusLocker) { if (this.Status == WorkerStatus.Working) { isWorking = true; this.SetStatus(WorkerStatus.Stopping); } } lock (this.runLocker) { isAlive = this.runThread != null && this.runThread.IsAlive; } if (isWorking && isAlive) { if (!force) { try { this.runThread.Join(); this.runThread = null; } catch (NullReferenceException) { } } else { this.KillRunThread(); } } lock (this.statusLocker) { lock (this.runLocker) { WorkingRecord record = this.currentRecord; using (IRepository repository = this.repositoryFactory.Create()) { using (IDbTransaction transaction = repository.BeginTransaction()) { if (record != null) { HistoryRecord history = CreateHistory(record, HistoryStatus.Interrupted); repository.DeleteWorking(record.Id.Value, transaction); history = repository.CreateHistory(history, transaction); } this.SetStatus(WorkerStatus.Stopped, repository, transaction); transaction.Commit(); } } this.currentRecord = null; } } if (isWorking) { this.logger.Info("Worker {0} ({1}) has stopped.", this.name, this.id); } }
public HistoryRecord CreateHistory(HistoryRecord record, IDbTransaction transaction) { const string Sql = @"INSERT INTO [BlueCollarHistory]([ApplicationName],[WorkerId],[ScheduleId],[QueueName],[JobName],[JobType],[Data],[QueuedOn],[TryNumber],[StartedOn],[Status],[Exception],[FinishedOn]) VALUES(@ApplicationName,@WorkerId,@ScheduleId,@QueueName,@JobName,@JobType,@Data,@QueuedOn,@TryNumber,@StartedOn,@StatusString,@Exception,@FinishedOn); SELECT CAST(SCOPE_IDENTITY() AS bigint);"; record.Id = this.connection.Query<long>( Sql, record, transaction, true, null, null).First(); return record; }