public void RemoveSerializedObject(Object instance) { var key = Model.GetKey(instance); var searchKey = String.Format("{0}{1}*", key, Model.KeySeparator); var keys = this.GetKeys(searchKey).ToArray(); if (keys != null && keys.Any()) { this.Remove(keys); } }
private static WorkQueueDetails CreateGeneralWorkQueueItemDetails(Model.WorkQueue item) { var detail = new WorkQueueDetails(); detail.Key = item.Key; detail.ScheduledDateTime = item.ScheduledTime; detail.ExpirationTime = item.ExpirationTime; detail.InsertTime = item.InsertTime; detail.FailureCount = item.FailureCount; detail.Type = item.WorkQueueTypeEnum; detail.Status = item.WorkQueueStatusEnum; detail.Priority = item.WorkQueuePriorityEnum; detail.FailureDescription = item.FailureDescription; detail.ServerDescription = item.ProcessorID; StudyStorageLocation storage = WorkQueueController.GetLoadStorageLocation(item); detail.StorageLocationPath = storage.GetStudyPath(); // Fetch UIDs var wqUidsAdaptor = new WorkQueueUidAdaptor(); var uidCriteria = new WorkQueueUidSelectCriteria(); uidCriteria.WorkQueueKey.EqualTo(item.GetKey()); IList<WorkQueueUid> uids = wqUidsAdaptor.Get(uidCriteria); var mapSeries = new Hashtable(); foreach (WorkQueueUid uid in uids) { if (mapSeries.ContainsKey(uid.SeriesInstanceUid) == false) mapSeries.Add(uid.SeriesInstanceUid, uid.SopInstanceUid); } detail.NumInstancesPending = uids.Count; detail.NumSeriesPending = mapSeries.Count; // Fetch the study and patient info var ssAdaptor = new StudyStorageAdaptor(); StudyStorage storages = ssAdaptor.Get(item.StudyStorageKey); var studyAdaptor = new StudyAdaptor(); var studycriteria = new StudySelectCriteria(); studycriteria.StudyInstanceUid.EqualTo(storages.StudyInstanceUid); studycriteria.ServerPartitionKey.EqualTo(item.ServerPartitionKey); Study study = studyAdaptor.GetFirst(studycriteria); // Study may not be available until the images are processed. if (study != null) { var studyAssembler = new StudyDetailsAssembler(); detail.Study = studyAssembler.CreateStudyDetail(study); } return detail; }
public void UpdateSerializedObjectValue <T, F>(T instance, Expression <Func <T, F> > expression, F value) { if (value == null) { throw new ArgumentException("Cannot be null.", "value"); } var prefix = Model.GetKey(instance); var memberKey = Model.GetKeyForMemberExpression(expression.Body as MemberExpression); var key = Model.CreateKey(prefix, memberKey); Set(key, RedisDataFormatUtil.FormatForStorage(value)); }
public void SerializeObject(Object instance, String keyPrefix = null, TimeSpan?slidingExpiration = null) { var type = instance.GetType(); var instanceKey = Model.GetKey(instance); var fullPrefix = instanceKey; if (!String.IsNullOrEmpty(keyPrefix)) { fullPrefix = Model.CreateKey(keyPrefix, instanceKey); } var members = new Dictionary <Member, Object>(); ReflectionUtil.MapToDictionary(instance, members); foreach (var member in members) { var fullKey = Model.CreateKey(fullPrefix, member.Key.FullyQualifiedName); var value = member.Value; if (member.Key.IsBinarySerialized) { BinarySerializeObject(fullKey, value); ExpireByTimespan(fullKey, slidingExpiration); } else { if (value != null) { Set(fullKey, RedisDataFormatUtil.FormatForStorage(value)); ExpireByTimespan(fullKey, slidingExpiration); } } } }
public void RedisSessionStateItem_Tests() { RedisContext.AddHost("Default", "127.0.0.1"); var sessionState = new RedisSessionStateItem(); sessionState.SessionId = System.Guid.NewGuid().ToString("N").ToLower(); sessionState.ApplicationName = "TestApplication"; sessionState.Created = DateTime.UtcNow; sessionState.SesssionItems = new MockObject() { Id = 2, EmailAddress = "*****@*****.**", Specialty = Speciality.Foos, Name = "Will", Tags = new List <String>() { "Stuff", "More Stuff" } }; using (var rc = RedisContext.GetClient()) { rc.SerializeObject(sessionState); var deserialized = rc.DeserializeObject <RedisSessionStateItem>(Model.GetKey(sessionState)); Assert.Equal(sessionState.SessionId, deserialized.SessionId); Assert.Equal((sessionState.SesssionItems as MockObject).EmailAddress, (deserialized.SesssionItems as MockObject).EmailAddress); rc.UpdateSerializedObjectValue(sessionState, s => s.Locked, true); rc.UpdateSerializedObjectValue(sessionState, s => s.LockDate, DateTime.UtcNow); deserialized = rc.DeserializeObject <RedisSessionStateItem>(Model.GetKey(sessionState)); Assert.True(deserialized.Locked); Assert.True(deserialized.LockDate != default(DateTime)); rc.RemoveSerializedObject(sessionState); } }
public ReconcileStudyWorkQueue(Model.WorkQueue workqueue) { Platform.CheckTrue(workqueue.WorkQueueTypeEnum.Equals(WorkQueueTypeEnum.ReconcileStudy), String.Format("Cannot copy data from Work Queue record of type {0}", workqueue.WorkQueueTypeEnum)); this.SetKey(workqueue.GetKey()); this.Data= workqueue.Data; this.InsertTime = workqueue.InsertTime; this.DeviceKey = workqueue.DeviceKey; this.ExpirationTime = workqueue.ExpirationTime; this.FailureCount = workqueue.FailureCount; this.FailureDescription = workqueue.FailureDescription; this.GroupID = workqueue.GroupID; this.InsertTime = workqueue.InsertTime; this.ProcessorID = workqueue.ProcessorID; this.ScheduledTime = workqueue.ScheduledTime; this.ServerPartitionKey = workqueue.ServerPartitionKey; this.StudyHistoryKey = workqueue.StudyHistoryKey; this.StudyStorageKey = workqueue.StudyStorageKey; this.WorkQueuePriorityEnum = workqueue.WorkQueuePriorityEnum; this.WorkQueueStatusEnum = workqueue.WorkQueueStatusEnum; this.WorkQueueTypeEnum = this.WorkQueueTypeEnum; }
/// <summary> /// Load the specific SOP Instance Uids in the database for the WorkQueue item. /// </summary> /// <param name="item">The WorkQueue item.</param> protected void LoadUids(Model.WorkQueue item) { if (_uidList==null) { UidsLoadTime.Add(delegate { using (ServerExecutionContext context = new ServerExecutionContext()) { IWorkQueueUidEntityBroker select = context.ReadContext.GetBroker<IWorkQueueUidEntityBroker>(); WorkQueueUidSelectCriteria parms = new WorkQueueUidSelectCriteria(); parms.WorkQueueKey.EqualTo(item.GetKey()); _uidList = select.Find(parms); _uidList = TruncateList(item, _uidList); } } ); } }
public void RaiseAlert(Model.WorkQueue queueItem, AlertLevel level, string message) { if (WorkQueueProperties.AlertFailedWorkQueue || level == AlertLevel.Critical) { ServerPlatform.Alert(AlertCategory.Application, level, queueItem.WorkQueueTypeEnum.ToString(), AlertTypeCodes.UnableToProcess, GetWorkQueueContextData(queueItem), TimeSpan.Zero, "Work Queue item failed: Type={0}, GUID={1}: {2}", queueItem.WorkQueueTypeEnum, queueItem.GetKey(), message); } }
private void InternalPostponeWorkQueue(Model.WorkQueue item, DateTime newScheduledTime, DateTime expireTime, string reasonText, bool updateWorkQueueEntry, WorkQueueProcessorFailureType? errorType) { if (errorType!=null) { Platform.Log(LogLevel.Info, "Postpone {0} entry until {1}: {2}. [GUID={3}.] (This transaction is treated as a failure)", item.WorkQueueTypeEnum, newScheduledTime, reasonText, item.GetKey()); item.FailureDescription = reasonText; PostProcessingFailure(item, WorkQueueProcessorFailureType.NonFatal); return; } Platform.Log(LogLevel.Info, "Postpone {0} entry until {1}: {2}. [GUID={3}]", item.WorkQueueTypeEnum, newScheduledTime, reasonText, item.GetKey()); using (IUpdateContext updateContext = PersistentStoreRegistry.GetDefaultStore().OpenUpdateContext(UpdateContextSyncMode.Flush)) { IPostponeWorkQueue broker = updateContext.GetBroker<IPostponeWorkQueue>(); PostponeWorkQueueParameters parameters = new PostponeWorkQueueParameters { WorkQueueKey = item.Key, Reason = reasonText, ScheduledTime = newScheduledTime, ExpirationTime = expireTime, UpdateWorkQueue = updateWorkQueueEntry }; if (broker.Execute(parameters) == false) { Platform.Log(LogLevel.Error, "Unable to reschedule {0} WorkQueue GUID: {1}", item.WorkQueueTypeEnum, item.GetKey().ToString()); } else { updateContext.Commit(); } } }
protected override void ProcessItem(Model.WorkQueue item) { Platform.CheckMemberIsSet(StorageLocation, "StorageLocation"); Platform.CheckForNullReference(Study, "Study doesn't exist"); if (WorkQueueUidList.Count == 0) { // we are done. Just need to cleanup the duplicate folder Platform.Log(LogLevel.Info, "{0} is completed. Cleaning up duplicate storage folder. (GUID={1}, action={2})", item.WorkQueueTypeEnum, item.GetKey().Key, _processDuplicateEntry.QueueData.Action); CleanUpReconcileFolders(); PostProcessing(item, WorkQueueProcessorStatus.Complete, WorkQueueProcessorDatabaseUpdate.ResetQueueState); } else { Platform.Log(LogLevel.Info, "Processing {0} entry (GUID={1}, action={2})", item.WorkQueueTypeEnum, item.GetKey().Key, _processDuplicateEntry.QueueData.Action); Platform.CheckTrue(Directory.Exists(DuplicateFolder), String.Format("Duplicate Folder {0} doesn't exist.", DuplicateFolder)); LogWorkQueueInfo(); EnsureStorageLocationIsWritable(StorageLocation); _currentStudyInfo = StudyInformation.CreateFrom(Study); ImageSetDetails duplicateSopDetails = null; // If deleting duplicates then don't log the history if (_processDuplicateEntry.QueueData.Action != ProcessDuplicateAction.Delete && !HistoryLogged) { duplicateSopDetails = LoadDuplicateDetails(); } try { UpdateStudyOrDuplicates(); int count = ProcessUidList(); // If deleting duplicates then don't log the history if (_processDuplicateEntry.QueueData.Action != ProcessDuplicateAction.Delete && !HistoryLogged && duplicateSopDetails != null && count > 0) { LogHistory(duplicateSopDetails); } PostProcessing(item, WorkQueueProcessorStatus.Pending, WorkQueueProcessorDatabaseUpdate.None); } finally { UpdateQueueData(); } } }
/// <summary> /// Simple routine for failing a work queue item. /// </summary> /// <param name="item">The item to fail.</param> /// <param name="failureDescription">The reason for the failure.</param> protected override void FailQueueItem(Model.WorkQueue item, string failureDescription) { DBUpdateTime.Add( delegate { using (IUpdateContext updateContext = PersistentStoreRegistry.GetDefaultStore().OpenUpdateContext(UpdateContextSyncMode.Flush)) { IUpdateWorkQueue update = updateContext.GetBroker<IUpdateWorkQueue>(); UpdateWorkQueueParameters parms = new UpdateWorkQueueParameters { ProcessorID = ServerPlatform.ProcessorId, WorkQueueKey = item.GetKey(), StudyStorageKey = item.StudyStorageKey, FailureCount = item.FailureCount + 1, FailureDescription = failureDescription }; Platform.Log(LogLevel.Error, "Failing {0} WorkQueue entry ({1}): {2}", item.WorkQueueTypeEnum, item.GetKey(), failureDescription); parms.WorkQueueStatusEnum = WorkQueueStatusEnum.Failed; parms.ScheduledTime = Platform.Time; parms.ExpirationTime = Platform.Time.AddDays(1); if (false == update.Execute(parms)) { Platform.Log(LogLevel.Error, "Unable to update {0} WorkQueue GUID: {1}", item.WorkQueueTypeEnum, item.GetKey().ToString()); } else updateContext.Commit(); } } ); }
public WorkQueueProcessorContext(Model.WorkQueue item) :base(item.GetKey().Key.ToString()) { _item = item; }
private static WorkQueueDetails CreateEditWorkQueueItemDetails(Model.WorkQueue item) { string studyPath; try { StudyStorageLocation storage = WorkQueueController.GetLoadStorageLocation(item); studyPath = storage.GetStudyPath(); } catch(Exception) { studyPath = string.Empty; } var detail = new WorkQueueDetails { Key = item.Key, ScheduledDateTime = item.ScheduledTime, ExpirationTime = item.ExpirationTime, InsertTime = item.InsertTime, FailureCount = item.FailureCount, Type = item.WorkQueueTypeEnum, Status = item.WorkQueueStatusEnum, Priority = item.WorkQueuePriorityEnum, FailureDescription = item.FailureDescription, ServerDescription = item.ProcessorID, StorageLocationPath = studyPath }; // Fetch UIDs var wqUidsAdaptor = new WorkQueueUidAdaptor(); var uidCriteria = new WorkQueueUidSelectCriteria(); uidCriteria.WorkQueueKey.EqualTo(item.GetKey()); IList<WorkQueueUid> uids = wqUidsAdaptor.Get(uidCriteria); var mapSeries = new Hashtable(); foreach (WorkQueueUid uid in uids) { if (mapSeries.ContainsKey(uid.SeriesInstanceUid) == false) mapSeries.Add(uid.SeriesInstanceUid, uid.SopInstanceUid); } detail.NumInstancesPending = uids.Count; detail.NumSeriesPending = mapSeries.Count; // Fetch the study and patient info var ssAdaptor = new StudyStorageAdaptor(); StudyStorage storages = ssAdaptor.Get(item.StudyStorageKey); var studyAdaptor = new StudyAdaptor(); var studycriteria = new StudySelectCriteria(); studycriteria.StudyInstanceUid.EqualTo(storages.StudyInstanceUid); studycriteria.ServerPartitionKey.EqualTo(item.ServerPartitionKey); Study study = studyAdaptor.GetFirst(studycriteria); // Study may not be available until the images are processed. if (study != null) { var studyAssembler = new StudyDetailsAssembler(); detail.Study = studyAssembler.CreateStudyDetail(study); } var parser = new EditStudyWorkQueueDataParser(); EditStudyWorkQueueData data = parser.Parse(item.Data.DocumentElement); detail.EditUpdateItems = data.EditRequest.UpdateEntries.ToArray(); return detail; }
/// <summary> /// Set a status of <see cref="WorkQueue"/> item after batch processing has been completed. /// </summary> /// <remarks> /// <para> /// This routine will set the status of the <paramref name="item"/> to one of the followings /// <list type="bullet"> /// <item>Failed: if the current process failed and number of retries has been reached.</item> /// <item>Pending: if the current batch has been processed successfully</item> /// <item>Idle : if current batch size = 0.</item> /// <item>Completed: if batch size =0 (idle) and the item has expired.</item> /// </list> /// </para> /// </remarks> /// <param name="item">The <see cref="WorkQueue"/> item to set.</param> /// <param name="status">Indicates if complete.</param> /// <param name="resetQueueStudyState">Reset the queue study state back to Idle</param> protected virtual void PostProcessing(Model.WorkQueue item, WorkQueueProcessorStatus status, WorkQueueProcessorDatabaseUpdate resetQueueStudyState) { Completed = status == WorkQueueProcessorStatus.Complete || (status == WorkQueueProcessorStatus.Idle && item.ExpirationTime < Platform.Time); if (Completed) { if (WorkQueueSettings.Instance.EnableStudyIntegrityValidation) { Platform.Log(LogLevel.Info, "{0} has completed (GUID={1})", item.WorkQueueTypeEnum, item.GetKey().Key); VerifyStudy(StorageLocation); } } DBUpdateTime.Add( delegate { using ( IUpdateContext updateContext = PersistentStoreRegistry.GetDefaultStore().OpenUpdateContext(UpdateContextSyncMode.Flush)) { IUpdateWorkQueue update = updateContext.GetBroker<IUpdateWorkQueue>(); UpdateWorkQueueParameters parms = new UpdateWorkQueueParameters { WorkQueueKey = item.GetKey(), StudyStorageKey = item.StudyStorageKey, ProcessorID = item.ProcessorID }; DateTime now = Platform.Time; if (item.FailureDescription != null) parms.FailureDescription = item.FailureDescription; DateTime scheduledTime = now.AddSeconds(WorkQueueProperties.ProcessDelaySeconds); if (scheduledTime > item.ExpirationTime) scheduledTime = item.ExpirationTime; if (status == WorkQueueProcessorStatus.CompleteDelayDelete) { parms.WorkQueueStatusEnum = WorkQueueStatusEnum.Idle; parms.FailureCount = item.FailureCount; parms.FailureDescription = ""; parms.ScheduledTime = parms.ExpirationTime = Platform.Time.AddSeconds(WorkQueueProperties.DeleteDelaySeconds); if (resetQueueStudyState == WorkQueueProcessorDatabaseUpdate.ResetQueueState) parms.QueueStudyStateEnum = QueueStudyStateEnum.Idle; } else if (status == WorkQueueProcessorStatus.Complete || (status == WorkQueueProcessorStatus.Idle && item.ExpirationTime < Platform.Time)) { parms.WorkQueueStatusEnum = WorkQueueStatusEnum.Completed; parms.FailureCount = item.FailureCount; parms.ScheduledTime = scheduledTime; if (resetQueueStudyState == WorkQueueProcessorDatabaseUpdate.ResetQueueState) parms.QueueStudyStateEnum = QueueStudyStateEnum.Idle; parms.ExpirationTime = item.ExpirationTime; // Keep the same Completed = true; } else if (status == WorkQueueProcessorStatus.Idle || status == WorkQueueProcessorStatus.IdleNoDelete) { scheduledTime = now.AddSeconds(WorkQueueProperties.DeleteDelaySeconds); if (scheduledTime > item.ExpirationTime) scheduledTime = item.ExpirationTime; parms.WorkQueueStatusEnum = WorkQueueStatusEnum.Idle; parms.ScheduledTime = scheduledTime; parms.ExpirationTime = item.ExpirationTime; // keep the same parms.FailureCount = item.FailureCount; } else { parms.WorkQueueStatusEnum = WorkQueueStatusEnum.Pending; parms.ExpirationTime = scheduledTime.AddSeconds(WorkQueueProperties.ExpireDelaySeconds); parms.ScheduledTime = scheduledTime; parms.FailureCount = item.FailureCount; } if (false == update.Execute(parms)) { Platform.Log(LogLevel.Error, "Unable to update {0} WorkQueue Key: {1}", item.WorkQueueTypeEnum, item.Key.ToString()); } else updateContext.Commit(); } } ); }
/// <summary> /// Simple routine for abort (fail) a work queue item immediately. /// </summary> /// <param name="item">The item to fail.</param> /// <param name="failureDescription">The reason for the failure.</param> /// <param name="generateAlert"></param> protected virtual void AbortQueueItem(Model.WorkQueue item, string failureDescription, bool generateAlert) { int retryCount = 0; while (true) { try { int count = retryCount; DBUpdateTime.Add( delegate { #region Fail the WorkQueue entry using (IUpdateContext updateContext = PersistentStoreRegistry.GetDefaultStore().OpenUpdateContext(UpdateContextSyncMode.Flush)) { if (count>0) Platform.Log(LogLevel.Error, "Abort {0} WorkQueue entry ({1}). Retry # {2}. Reason: {3}", item.WorkQueueTypeEnum, item.GetKey(), count, failureDescription); else Platform.Log(LogLevel.Error, "Abort {0} WorkQueue entry ({1}). Reason: {2}", item.WorkQueueTypeEnum, item.GetKey(), failureDescription); IUpdateWorkQueue broker = updateContext.GetBroker<IUpdateWorkQueue>(); UpdateWorkQueueParameters parms = new UpdateWorkQueueParameters { ProcessorID = ServerPlatform.ProcessorId, WorkQueueKey = item.GetKey(), StudyStorageKey = item.StudyStorageKey, FailureCount = item.FailureCount + 1, FailureDescription = failureDescription, WorkQueueStatusEnum = WorkQueueStatusEnum.Failed, ScheduledTime = Platform.Time, ExpirationTime = Platform.Time.AddDays(1) }; if (false == broker.Execute(parms)) { Platform.Log(LogLevel.Error, "Unable to update {0} WorkQueue GUID: {1}", item.WorkQueueTypeEnum, item.GetKey().ToString()); } else { updateContext.Commit(); } } #endregion }); break; // done } catch (Exception ex) { if (ex is PersistenceException || ex is SqlException) { if (retryCount > MAX_DB_RETRY) { Platform.Log(LogLevel.Error, ex, "Error occurred when calling AbortQueueItem. Max db retry count has been reached."); throw; } Platform.Log(LogLevel.Error, ex, "Error occurred when calling AbortQueueItem. Retry later. GUID={0}", item.Key); SleepForRetry(); // Service is stoping if (CancelPending) { Platform.Log(LogLevel.Warn, "Stop is requested. Attempt to abort WorkQueue entry is now terminated."); break; } retryCount++; } else throw; } } }
/// <summary> /// The actual delegate /// </summary> /// <param name="processor"></param> /// <param name="queueItem"></param> private void ExecuteProcessor(IWorkQueueItemProcessor processor, Model.WorkQueue queueItem) { try { processor.Process(queueItem); } catch (Exception e) { Platform.Log(LogLevel.Error, e, "Unexpected exception when processing WorkQueue item of type {0}. Failing Queue item. (GUID: {1})", queueItem.WorkQueueTypeEnum, queueItem.GetKey()); String error = e.InnerException != null ? e.InnerException.Message : e.Message; FailQueueItem(queueItem, error); } finally { // Signal the parent thread, so it can query again _threadStop.Set(); // Cleanup the processor processor.Dispose(); } }
/// <summary> /// Simple routine for failing a work queue item. /// </summary> /// <param name="item">The item to fail.</param> /// <param name="failureDescription">The reason for the failure.</param> private void FailQueueItem(Model.WorkQueue item, string failureDescription) { // Must retry to reset the status of the entry in case of db error // Failure to do so will create stale work queue entry (stuck in "In Progress" state) // which can only be recovered by restarting the service. while(true) { try { WorkQueueTypeProperties prop = _propertiesDictionary[item.WorkQueueTypeEnum]; using (IUpdateContext updateContext = _store.OpenUpdateContext(UpdateContextSyncMode.Flush)) { IUpdateWorkQueue update = updateContext.GetBroker<IUpdateWorkQueue>(); UpdateWorkQueueParameters parms = new UpdateWorkQueueParameters { ProcessorID = ServerPlatform.ProcessorId, WorkQueueKey = item.GetKey(), StudyStorageKey = item.StudyStorageKey, FailureCount = item.FailureCount + 1, FailureDescription = failureDescription }; var settings = WorkQueueSettings.Instance; if ((item.FailureCount + 1) > prop.MaxFailureCount) { Platform.Log(LogLevel.Error, "Failing {0} WorkQueue entry ({1}), reached max retry count of {2}. Failure Reason: {3}", item.WorkQueueTypeEnum, item.GetKey(), item.FailureCount + 1, failureDescription); parms.WorkQueueStatusEnum = WorkQueueStatusEnum.Failed; parms.ScheduledTime = Platform.Time; parms.ExpirationTime = Platform.Time.AddDays(1); OnWorkQueueEntryFailed(item, failureDescription); } else { Platform.Log(LogLevel.Error, "Resetting {0} WorkQueue entry ({1}) to Pending, current retry count {2}. Failure Reason: {3}", item.WorkQueueTypeEnum, item.GetKey(), item.FailureCount + 1, failureDescription); parms.WorkQueueStatusEnum = WorkQueueStatusEnum.Pending; parms.ScheduledTime = Platform.Time.AddMilliseconds(settings.WorkQueueQueryDelay); parms.ExpirationTime = Platform.Time.AddSeconds((prop.MaxFailureCount - item.FailureCount) * prop.FailureDelaySeconds); } if (false == update.Execute(parms)) { Platform.Log(LogLevel.Error, "Unable to update {0} WorkQueue GUID: {1}", item.WorkQueueTypeEnum, item.GetKey().ToString()); } else { updateContext.Commit(); break; // done } } } catch(Exception ex) { Platform.Log(LogLevel.Error, "Error occurred when calling FailQueueItem. Retry later. {0}", ex.Message); _terminateEvent.WaitOne(2000, false); if (_stop) { Platform.Log(LogLevel.Warn, "Service is stopping. Retry to fail the entry is terminated."); break; } } } }
/// <summary> /// Set a <see cref="ServiceLock"/> entry to pending. /// </summary> /// <param name="item">The <see cref="ServiceLock"/> entry to set.</param> /// <param name="scheduledTime"></param> /// <param name="enabled">Bool telling if the ServiceLock entry should be enabled after unlock.</param> protected static void UnlockServiceLock(Model.ServiceLock item, bool enabled, DateTime scheduledTime) { using (IUpdateContext updateContext = PersistentStoreRegistry.GetDefaultStore().OpenUpdateContext(UpdateContextSyncMode.Flush)) { // Update the WorkQueue item status and times. IUpdateServiceLock update = updateContext.GetBroker<IUpdateServiceLock>(); ServiceLockUpdateParameters parms = new ServiceLockUpdateParameters { ServiceLockKey = item.GetKey(), Lock = false, ScheduledTime = scheduledTime, ProcessorId = item.ProcessorId, Enabled = enabled }; if (false == update.Execute(parms)) { Platform.Log(LogLevel.Error, "Unable to update StudyLock GUID Status: {0}", item.GetKey().ToString()); } updateContext.Commit(); } }
protected override void ProcessItem(Model.WorkQueue item) { LoadUids(item); if (WorkQueueUidList.Count == 0) { // No UIDs associated with the WorkQueue item. Set the status back to idle PostProcessing(item, WorkQueueProcessorStatus.Idle, WorkQueueProcessorDatabaseUpdate.ResetQueueState); return; } XmlElement element = item.Data.DocumentElement; string syntax = element.Attributes["syntax"].Value; TransferSyntax compressSyntax = TransferSyntax.GetTransferSyntax(syntax); if (compressSyntax == null) { item.FailureDescription = String.Format("Invalid transfer syntax in compression WorkQueue item: {0}", element.Attributes["syntax"].Value); Platform.Log(LogLevel.Error, "Error with work queue item {0}: {1}", item.GetKey(), item.FailureDescription); base.PostProcessingFailure(item, WorkQueueProcessorFailureType.Fatal); return; } if (Study == null) { item.FailureDescription = String.Format("Compression item does not have a linked Study record"); Platform.Log(LogLevel.Error, "Error with work queue item {0}: {1}", item.GetKey(), item.FailureDescription); base.PostProcessingFailure(item, WorkQueueProcessorFailureType.Fatal); return; } Platform.Log(LogLevel.Info, "Compressing study {0} for Patient {1} (PatientId:{2} A#:{3}) on partition {4} to {5}", Study.StudyInstanceUid, Study.PatientsName, Study.PatientId, Study.AccessionNumber, ServerPartition.Description, compressSyntax.Name); IDicomCodecFactory[] codecs = DicomCodecRegistry.GetCodecFactories(); IDicomCodecFactory theCodecFactory = null; foreach (IDicomCodecFactory codec in codecs) if (codec.CodecTransferSyntax.Equals(compressSyntax)) { theCodecFactory = codec; break; } if (theCodecFactory == null) { item.FailureDescription = String.Format("Unable to find codec for compression: {0}", compressSyntax.Name); Platform.Log(LogLevel.Error, "Error with work queue item {0}: {1}", item.GetKey(), item.FailureDescription); base.PostProcessingFailure(item, WorkQueueProcessorFailureType.Fatal); return; } if (!ProcessUidList(item, theCodecFactory)) PostProcessingFailure(item, WorkQueueProcessorFailureType.NonFatal); else { Platform.Log(LogLevel.Info, "Completed Compressing study {0} for Patient {1} (PatientId:{2} A#:{3}) on partition {4} to {5}", Study.StudyInstanceUid, Study.PatientsName, Study.PatientId, Study.AccessionNumber, ServerPartition.Description, compressSyntax.Name); if (compressSyntax.LossyCompressed) UpdateStudyStatus(StorageLocation, StudyStatusEnum.OnlineLossy, compressSyntax); else UpdateStudyStatus(StorageLocation, StudyStatusEnum.OnlineLossless, compressSyntax); PostProcessing(item, WorkQueueProcessorStatus.Pending, WorkQueueProcessorDatabaseUpdate.None); // batch processed, not complete } }
/// <summary> /// Creates an instance of <see cref="ServiceLockProcessorContext"/> /// </summary> /// <param name="item"></param> public ServiceLockProcessorContext(Model.ServiceLock item) :base(item.GetKey().Key.ToString()) { Platform.CheckForNullReference(item, "item"); _item = item; }
/// <summary> /// Set a status of <see cref="WorkQueue"/> item after batch processing has been completed. /// </summary> /// <remarks> /// <para> /// This routine will set the status of the <paramref name="item"/> to one of the following /// <list type="bullet"> /// <item>Failed: if the current process failed and number of retries has been reached or a fatal error.</item> /// <item>Pending: if the number of retries has not been reached</item> /// </list> /// </para> /// </remarks> /// <param name="item">The <see cref="WorkQueue"/> item to set.</param> /// <param name="processorFailureType">The failure is unrecoverable</param> protected virtual void PostProcessingFailure(Model.WorkQueue item, WorkQueueProcessorFailureType processorFailureType) { int retryCount = 0; while (true) { try { #region Fail the entry DBUpdateTime.Add( delegate { using ( IUpdateContext updateContext = PersistentStoreRegistry.GetDefaultStore().OpenUpdateContext(UpdateContextSyncMode.Flush)) { IUpdateWorkQueue update = updateContext.GetBroker<IUpdateWorkQueue>(); UpdateWorkQueueParameters parms = new UpdateWorkQueueParameters { WorkQueueKey = item.GetKey(), StudyStorageKey = item.StudyStorageKey, ProcessorID = item.ProcessorID }; if (item.FailureDescription != null) parms.FailureDescription = item.FailureDescription; parms.FailureCount = item.FailureCount + 1; if (processorFailureType == WorkQueueProcessorFailureType.Fatal) { Platform.Log(LogLevel.Error, "Failing {0} WorkQueue entry ({1}), fatal error: {2}", item.WorkQueueTypeEnum, item.GetKey(), item.FailureDescription); parms.WorkQueueStatusEnum = WorkQueueStatusEnum.Failed; parms.ScheduledTime = Platform.Time; parms.ExpirationTime = Platform.Time; // expire now RaiseAlert(item, AlertLevel.Error, String.Format("Failing {0} WorkQueue entry ({1}), fatal error: {2}", item.WorkQueueTypeEnum, item.GetKey(), item.FailureDescription)); } else if ((item.FailureCount + 1) > WorkQueueProperties.MaxFailureCount) { Platform.Log(LogLevel.Error, "Failing {0} WorkQueue entry ({1}), reached max retry count of {2}. Failure Reason: {3}", item.WorkQueueTypeEnum, item.GetKey(), item.FailureCount + 1, item.FailureDescription); parms.WorkQueueStatusEnum = WorkQueueStatusEnum.Failed; parms.ScheduledTime = Platform.Time; parms.ExpirationTime = Platform.Time; // expire now RaiseAlert(item, AlertLevel.Error, String.Format("Failing {0} WorkQueue entry ({1}): {2}", item.WorkQueueTypeEnum, item.GetKey(), item.FailureDescription)); } else { Platform.Log(LogLevel.Error, "Resetting {0} WorkQueue entry ({1}) to Pending, current retry count {2}.", item.WorkQueueTypeEnum, item.GetKey(), item.FailureCount + 1); parms.WorkQueueStatusEnum = WorkQueueStatusEnum.Pending; parms.ScheduledTime = Platform.Time.AddSeconds(WorkQueueProperties.FailureDelaySeconds); parms.ExpirationTime = Platform.Time.AddSeconds((WorkQueueProperties.MaxFailureCount - item.FailureCount) * WorkQueueProperties.FailureDelaySeconds); } if (false == update.Execute(parms)) { Platform.Log(LogLevel.Error, "Unable to update {0} WorkQueue GUID: {1}", item.WorkQueueTypeEnum, item.GetKey().ToString()); } else updateContext.Commit(); } } ); break; // done #endregion } catch (Exception ex) { if (ex is PersistenceException || ex is SqlException) { if (retryCount > MAX_DB_RETRY) { Platform.Log(LogLevel.Error, ex, "Error occurred when calling PostProcessingFailure. Max db retry count has been reached."); // can't do anything except throwing it. throw; } Platform.Log(LogLevel.Error, ex, "Error occurred when calling PostProcessingFailure(). Retry later. GUID={0}", item.Key); SleepForRetry(); // If service is stoping then stop if (CancelPending) { Platform.Log(LogLevel.Warn, "Stop is requested. Attempt to fail WorkQueue entry is now terminated."); break; } retryCount++; } else throw; } } }
/// <summary> /// Updates the 'State' of the filesystem associated with the 'FilesystemDelete' <see cref="ServiceLock"/> item /// </summary> /// <param name="item"></param> /// <param name="fs"></param> private static void UpdateState(Model.ServiceLock item, ServerFilesystemInfo fs) { FilesystemState state = null; if (item.State != null && item.State.DocumentElement!=null) { //load from datatabase state = XmlUtils.Deserialize<FilesystemState>(item.State.DocumentElement); } if (state == null) state = new FilesystemState(); if (fs.AboveHighWatermark) { // we don't want to generate alert if the filesystem is offline or not accessible. if (fs.Online && (fs.Readable || fs.Writeable)) { TimeSpan ALERT_INTERVAL = TimeSpan.FromMinutes(ServiceLockSettings.Default.HighWatermarkAlertInterval); if (state.AboveHighWatermarkTimestamp == null) state.AboveHighWatermarkTimestamp = Platform.Time; TimeSpan elapse = (state.LastHighWatermarkAlertTimestamp != null) ? Platform.Time - state.LastHighWatermarkAlertTimestamp.Value : Platform.Time - state.AboveHighWatermarkTimestamp.Value; if (elapse.Duration() >= ALERT_INTERVAL) { ServerPlatform.Alert(AlertCategory.System, AlertLevel.Warning, "Filesystem", AlertTypeCodes.LowResources, null, TimeSpan.Zero, SR.AlertFilesystemAboveHW, fs.Filesystem.Description, TimeSpanFormatter.Format(Platform.Time - state.AboveHighWatermarkTimestamp.Value, true)); state.LastHighWatermarkAlertTimestamp = Platform.Time; } } else { state.AboveHighWatermarkTimestamp = null; state.LastHighWatermarkAlertTimestamp = null; } } else { state.AboveHighWatermarkTimestamp = null; state.LastHighWatermarkAlertTimestamp = null; } XmlDocument stateXml = new XmlDocument(); stateXml.AppendChild(stateXml.ImportNode(XmlUtils.Serialize(state), true)); IPersistentStore store = PersistentStoreRegistry.GetDefaultStore(); using (IUpdateContext ctx = store.OpenUpdateContext(UpdateContextSyncMode.Flush)) { ServiceLockUpdateColumns columns = new ServiceLockUpdateColumns(); columns.State = stateXml; IServiceLockEntityBroker broker = ctx.GetBroker<IServiceLockEntityBroker>(); broker.Update(item.GetKey(), columns); ctx.Commit(); } }
/// <summary> /// Simple routine for failing a work queue item. /// </summary> /// <param name="item">The item to fail.</param> /// <param name="failureDescription">The reason for the failure.</param> protected virtual void FailQueueItem(Model.WorkQueue item, string failureDescription) { int retryCount = 0; while (true) { try { DBUpdateTime.Add( delegate { #region Remove the WorkQueue entry using (IUpdateContext updateContext = PersistentStoreRegistry.GetDefaultStore().OpenUpdateContext(UpdateContextSyncMode.Flush)) { IUpdateWorkQueue update = updateContext.GetBroker<IUpdateWorkQueue>(); UpdateWorkQueueParameters parms = new UpdateWorkQueueParameters { ProcessorID = ServerPlatform.ProcessorId, WorkQueueKey = item.GetKey(), StudyStorageKey = item.StudyStorageKey, FailureCount = item.FailureCount + 1, FailureDescription = failureDescription }; var settings = WorkQueueSettings.Instance; if ((item.FailureCount + 1) > WorkQueueProperties.MaxFailureCount) { Platform.Log(LogLevel.Error, "Failing {0} WorkQueue entry ({1}), reached max retry count of {2}. Failure Reason: {3}", item.WorkQueueTypeEnum, item.GetKey(), item.FailureCount + 1, failureDescription); parms.WorkQueueStatusEnum = WorkQueueStatusEnum.Failed; parms.ScheduledTime = Platform.Time; parms.ExpirationTime = Platform.Time.AddDays(1); RaiseAlert(item, AlertLevel.Error, String.Format("Failing {0} WorkQueue entry ({1}), reached max retry count of {2}. Failure Reason: {3}", item.WorkQueueTypeEnum, item.GetKey(), item.FailureCount + 1, failureDescription)); } else { Platform.Log(LogLevel.Error, "Resetting {0} WorkQueue entry ({1}) to Pending, current retry count {2}", item.WorkQueueTypeEnum, item.GetKey(), item.FailureCount + 1); parms.WorkQueueStatusEnum = WorkQueueStatusEnum.Pending; parms.ScheduledTime = Platform.Time.AddMilliseconds(settings.WorkQueueQueryDelay); parms.ExpirationTime = Platform.Time.AddSeconds((WorkQueueProperties.MaxFailureCount - item.FailureCount) * WorkQueueProperties.FailureDelaySeconds); } if (false == update.Execute(parms)) { Platform.Log(LogLevel.Error, "Unable to update {0} WorkQueue GUID: {1}", item.WorkQueueTypeEnum, item.GetKey().ToString()); } else updateContext.Commit(); } #endregion }); break; // done } catch (Exception ex) { if (ex is PersistenceException || ex is SqlException) { if (retryCount > MAX_DB_RETRY) { Platform.Log(LogLevel.Error, ex, "Error occurred when calling FailQueueItem. Max db retry count has been reached."); throw; } Platform.Log(LogLevel.Error, ex, "Error occurred when calling FailQueueItem. Retry later. GUID={0}", item.Key); SleepForRetry(); // Service is stoping if (CancelPending) { Platform.Log(LogLevel.Warn, "Stop is requested. Attempt to fail WorkQueue entry is now terminated."); break; } retryCount++; } else throw; } } }
/// <summary> /// Process a <see cref="WorkQueue"/> item of type AutoRoute. /// </summary> protected override void ProcessItem(Model.WorkQueue item) { if (WorkQueueItem.ScheduledTime >= WorkQueueItem.ExpirationTime && !HasPendingItems) { Platform.Log(LogLevel.Debug, "Removing Idle {0} entry : {1}", item.WorkQueueTypeEnum, item.GetKey().Key); base.PostProcessing(item, WorkQueueProcessorStatus.Complete, WorkQueueProcessorDatabaseUpdate.None); return; } if (!HasPendingItems) { // nothing to process, change to idle state PostProcessing(item, WorkQueueProcessorStatus.Idle, WorkQueueProcessorDatabaseUpdate.None); return; } Platform.Log(LogLevel.Info, "Moving study {0} for Patient {1} (PatientId:{2} A#:{3}) on Partition {4} to {5}...", Study.StudyInstanceUid, Study.PatientsName, Study.PatientId, Study.AccessionNumber, ServerPartition.Description, DestinationDevice.AeTitle); // Load remote device information from the database. Device device = DestinationDevice; if (device == null) { item.FailureDescription = String.Format("Unknown auto-route destination \"{0}\"", item.DeviceKey); Platform.Log(LogLevel.Error, item.FailureDescription); PostProcessingFailure(item, WorkQueueProcessorFailureType.Fatal); // Fatal Error return; } if (device.Dhcp && device.IpAddress.Length == 0) { item.FailureDescription = String.Format("Auto-route destination is a DHCP device with no known IP address: \"{0}\"", device.AeTitle); Platform.Log(LogLevel.Error, item.FailureDescription); PostProcessingFailure(item, WorkQueueProcessorFailureType.Fatal); // Fatal error return; } // Now setup the StorageSCU component int sendCounter = 0; using (ImageServerStorageScu scu = new ImageServerStorageScu(ServerPartition, device)) { using (ServerExecutionContext context = new ServerExecutionContext()) // set the preferred syntax lists scu.LoadPreferredSyntaxes(context.ReadContext); // Load the Instances to Send into the SCU component scu.AddStorageInstanceList(InstanceList); // Set an event to be called when each image is transferred scu.ImageStoreCompleted += delegate(Object sender, StorageInstance instance) { if (instance.SendStatus.Status == DicomState.Success || instance.SendStatus.Status == DicomState.Warning || instance.SendStatus.Equals(DicomStatuses.SOPClassNotSupported)) { sendCounter++; OnInstanceSent(instance); } if (instance.SendStatus.Status == DicomState.Failure) { scu.FailureDescription = instance.SendStatus.Description; if (false == String.IsNullOrEmpty(instance.ExtendedFailureDescription)) { scu.FailureDescription = String.Format("{0} [{1}]", scu.FailureDescription, instance.ExtendedFailureDescription); } } if (CancelPending && !(this is WebMoveStudyItemProcessor) && !scu.Canceled) { Platform.Log(LogLevel.Info, "Auto-route canceled due to shutdown for study: {0}", StorageLocation.StudyInstanceUid); item.FailureDescription = "Operation was canceled due to server shutdown request."; scu.Cancel(); } }; try { // Block until send is complete scu.Send(); // Join for the thread to exit scu.Join(); } catch (Exception ex) { Platform.Log(LogLevel.Error, ex, "Error occurs while sending images to {0} : {1}", device.AeTitle, ex.Message); } finally { if (scu.FailureDescription.Length > 0) { item.FailureDescription = scu.FailureDescription; scu.Status = ScuOperationStatus.Failed; } // Reset the WorkQueue entry status if ((InstanceList.Count > 0 && sendCounter != InstanceList.Count) // not all sop were sent || scu.Status == ScuOperationStatus.Failed || scu.Status == ScuOperationStatus.ConnectFailed) { PostProcessingFailure(item, WorkQueueProcessorFailureType.NonFatal); // failures occurred} } else { OnComplete(); } } } }
private void SaveState(Model.WorkQueue item, ReprocessStudyQueueData queueData) { // Update the queue state using (IUpdateContext updateContext = PersistentStoreRegistry.GetDefaultStore().OpenUpdateContext(UpdateContextSyncMode.Flush)) { queueData.State.ExecuteAtLeastOnce = true; var broker = updateContext.GetBroker<IWorkQueueEntityBroker>(); var parms = new WorkQueueUpdateColumns {Data = XmlUtils.SerializeAsXmlDoc(_queueData)}; broker.Update(item.GetKey(), parms); updateContext.Commit(); } }
/// <summary> /// Reset the Lock for a specific <see cref="Model.ServiceLock"/> row. /// </summary> /// <param name="item">The row to reset the lock for.</param> private void ResetServiceLock(Model.ServiceLock item) { // keep trying just in case of db error while(true) { try { using (IUpdateContext updateContext = PersistentStoreRegistry.GetDefaultStore().OpenUpdateContext(UpdateContextSyncMode.Flush)) { // Update the ServiceLock item status and times. IUpdateServiceLock update = updateContext.GetBroker<IUpdateServiceLock>(); ServiceLockUpdateParameters parms = new ServiceLockUpdateParameters(); parms.ServiceLockKey = item.GetKey(); parms.Lock = false; parms.ScheduledTime = Platform.Time.AddMinutes(10); parms.ProcessorId = item.ProcessorId; if (false == update.Execute(parms)) { Platform.Log(LogLevel.Error, "Unable to update ServiceLock GUID Status: {0}", item.GetKey().ToString()); } updateContext.Commit(); } break; } catch(Exception ex) { Platform.Log(LogLevel.Error, ex, "Exception has occured when trying to reset the entry. Retry later"); _terminationEvent.WaitOne(2000, false); } } }