/// <summary> /// Simple routine for failing a work queue item. /// </summary> /// <param name="item">The item to fail.</param> /// <param name="failureDescription">The reason for the failure.</param> protected override void FailQueueItem(Model.WorkQueue item, string failureDescription) { DBUpdateTime.Add( delegate { using (IUpdateContext updateContext = PersistentStoreRegistry.GetDefaultStore().OpenUpdateContext(UpdateContextSyncMode.Flush)) { IUpdateWorkQueue update = updateContext.GetBroker <IUpdateWorkQueue>(); UpdateWorkQueueParameters parms = new UpdateWorkQueueParameters { ProcessorID = ServerPlatform.ProcessorId, WorkQueueKey = item.GetKey(), StudyStorageKey = item.StudyStorageKey, FailureCount = item.FailureCount + 1, FailureDescription = failureDescription }; Platform.Log(LogLevel.Error, "Failing {0} WorkQueue entry ({1}): {2}", item.WorkQueueTypeEnum, item.GetKey(), failureDescription); parms.WorkQueueStatusEnum = WorkQueueStatusEnum.Failed; parms.ScheduledTime = Platform.Time; parms.ExpirationTime = Platform.Time.AddDays(1); if (false == update.Execute(parms)) { Platform.Log(LogLevel.Error, "Unable to update {0} WorkQueue GUID: {1}", item.WorkQueueTypeEnum, item.GetKey().ToString()); } else { updateContext.Commit(); } } } ); }
/// <summary> /// Simple routine for failing a work queue item. /// </summary> /// <param name="item">The item to fail.</param> /// <param name="failureDescription">The reason for the failure.</param> private void FailQueueItem(Model.WorkQueue item, string failureDescription) { // Must retry to reset the status of the entry in case of db error // Failure to do so will create stale work queue entry (stuck in "In Progress" state) // which can only be recovered by restarting the service. while (true) { try { WorkQueueTypeProperties prop = _propertiesDictionary[item.WorkQueueTypeEnum]; using (IUpdateContext updateContext = _store.OpenUpdateContext(UpdateContextSyncMode.Flush)) { IUpdateWorkQueue update = updateContext.GetBroker <IUpdateWorkQueue>(); UpdateWorkQueueParameters parms = new UpdateWorkQueueParameters { ProcessorID = ServerPlatform.ProcessorId, WorkQueueKey = item.GetKey(), StudyStorageKey = item.StudyStorageKey, FailureCount = item.FailureCount + 1, FailureDescription = failureDescription }; var settings = WorkQueueSettings.Instance; if ((item.FailureCount + 1) > prop.MaxFailureCount) { Platform.Log(LogLevel.Error, "Failing {0} WorkQueue entry ({1}), reached max retry count of {2}. Failure Reason: {3}", item.WorkQueueTypeEnum, item.GetKey(), item.FailureCount + 1, failureDescription); parms.WorkQueueStatusEnum = WorkQueueStatusEnum.Failed; parms.ScheduledTime = Platform.Time; parms.ExpirationTime = Platform.Time.AddDays(1); OnWorkQueueEntryFailed(item, failureDescription); } else { Platform.Log(LogLevel.Error, "Resetting {0} WorkQueue entry ({1}) to Pending, current retry count {2}. Failure Reason: {3}", item.WorkQueueTypeEnum, item.GetKey(), item.FailureCount + 1, failureDescription); parms.WorkQueueStatusEnum = WorkQueueStatusEnum.Pending; parms.ScheduledTime = Platform.Time.AddMilliseconds(settings.WorkQueueQueryDelay); parms.ExpirationTime = Platform.Time.AddSeconds((prop.MaxFailureCount - item.FailureCount) * prop.FailureDelaySeconds); } if (false == update.Execute(parms)) { Platform.Log(LogLevel.Error, "Unable to update {0} WorkQueue GUID: {1}", item.WorkQueueTypeEnum, item.GetKey().ToString()); } else { updateContext.Commit(); break; // done } } } catch (Exception ex) { Platform.Log(LogLevel.Error, "Error occurred when calling FailQueueItem. Retry later. {0}", ex.Message); _terminateEvent.WaitOne(2000, false); if (_stop) { Platform.Log(LogLevel.Warn, "Service is stopping. Retry to fail the entry is terminated."); break; } } } }