Example #1
0
        private static WorkQueueDetails CreateGeneralWorkQueueItemDetails(Model.WorkQueue item)
        {
            var detail = new WorkQueueDetails();

            detail.Key = item.Key;
            detail.ScheduledDateTime = item.ScheduledTime;
            detail.ExpirationTime = item.ExpirationTime;
            detail.InsertTime = item.InsertTime;
            detail.FailureCount = item.FailureCount;
            detail.Type = item.WorkQueueTypeEnum;
            detail.Status = item.WorkQueueStatusEnum;
            detail.Priority = item.WorkQueuePriorityEnum;
            detail.FailureDescription = item.FailureDescription;
            detail.ServerDescription = item.ProcessorID;

            StudyStorageLocation storage = WorkQueueController.GetLoadStorageLocation(item);
            detail.StorageLocationPath = storage.GetStudyPath();

            // Fetch UIDs
            var wqUidsAdaptor = new WorkQueueUidAdaptor();
            var uidCriteria = new WorkQueueUidSelectCriteria();
            uidCriteria.WorkQueueKey.EqualTo(item.GetKey());
            IList<WorkQueueUid> uids = wqUidsAdaptor.Get(uidCriteria);

            var mapSeries = new Hashtable();
            foreach (WorkQueueUid uid in uids)
            {
                if (mapSeries.ContainsKey(uid.SeriesInstanceUid) == false)
                    mapSeries.Add(uid.SeriesInstanceUid, uid.SopInstanceUid);
            }

            detail.NumInstancesPending = uids.Count;
            detail.NumSeriesPending = mapSeries.Count;


            // Fetch the study and patient info
            var ssAdaptor = new StudyStorageAdaptor();
            StudyStorage storages = ssAdaptor.Get(item.StudyStorageKey);

            var studyAdaptor = new StudyAdaptor();
            var studycriteria = new StudySelectCriteria();
            studycriteria.StudyInstanceUid.EqualTo(storages.StudyInstanceUid);
            studycriteria.ServerPartitionKey.EqualTo(item.ServerPartitionKey);
            Study study = studyAdaptor.GetFirst(studycriteria);

            // Study may not be available until the images are processed.
            if (study != null)
            {
                var studyAssembler = new StudyDetailsAssembler();
                detail.Study = studyAssembler.CreateStudyDetail(study);
            }
            return detail;
        }
        public ReconcileStudyWorkQueue(Model.WorkQueue workqueue)
        {
            Platform.CheckTrue(workqueue.WorkQueueTypeEnum.Equals(WorkQueueTypeEnum.ReconcileStudy),
                               String.Format("Cannot copy data from Work Queue record of type {0}",
                                             workqueue.WorkQueueTypeEnum));

            this.SetKey(workqueue.GetKey());
            this.Data= workqueue.Data;
            this.InsertTime = workqueue.InsertTime;
            this.DeviceKey = workqueue.DeviceKey;
            this.ExpirationTime = workqueue.ExpirationTime;
            this.FailureCount = workqueue.FailureCount;
            this.FailureDescription = workqueue.FailureDescription;
            this.GroupID = workqueue.GroupID;
            this.InsertTime = workqueue.InsertTime;
            this.ProcessorID = workqueue.ProcessorID;
            this.ScheduledTime = workqueue.ScheduledTime;
            this.ServerPartitionKey = workqueue.ServerPartitionKey;
            this.StudyHistoryKey = workqueue.StudyHistoryKey;
            this.StudyStorageKey = workqueue.StudyStorageKey;
            this.WorkQueuePriorityEnum = workqueue.WorkQueuePriorityEnum;
            this.WorkQueueStatusEnum = workqueue.WorkQueueStatusEnum;
            this.WorkQueueTypeEnum = this.WorkQueueTypeEnum;
        }
        /// <summary>
        /// Load the specific SOP Instance Uids in the database for the WorkQueue item.
        /// </summary>
        /// <param name="item">The WorkQueue item.</param>
        protected void LoadUids(Model.WorkQueue item)
        {
            
            if (_uidList==null)
            {
				UidsLoadTime.Add(delegate
				                 	{
				                 		using (ServerExecutionContext context = new ServerExecutionContext())
				                 		{

				                 			IWorkQueueUidEntityBroker select = context.ReadContext.GetBroker<IWorkQueueUidEntityBroker>();

				                 			WorkQueueUidSelectCriteria parms = new WorkQueueUidSelectCriteria();

				                 			parms.WorkQueueKey.EqualTo(item.GetKey());
				                 			_uidList = select.Find(parms);

				                 			_uidList = TruncateList(item, _uidList);
				                 		}
				                 	}
					);
            }
        }
		public void RaiseAlert(Model.WorkQueue queueItem, AlertLevel level, string message)
		{
			if (WorkQueueProperties.AlertFailedWorkQueue || level == AlertLevel.Critical)
			{
				ServerPlatform.Alert(AlertCategory.Application, level,
									 queueItem.WorkQueueTypeEnum.ToString(), AlertTypeCodes.UnableToProcess,
									 GetWorkQueueContextData(queueItem), TimeSpan.Zero,
									 "Work Queue item failed: Type={0}, GUID={1}: {2}",
									 queueItem.WorkQueueTypeEnum,
									 queueItem.GetKey(), message);
			}
		}
        private void InternalPostponeWorkQueue(Model.WorkQueue item, DateTime newScheduledTime, DateTime expireTime, string reasonText,
            bool updateWorkQueueEntry, WorkQueueProcessorFailureType? errorType)
        {
            if (errorType!=null)
            {
                Platform.Log(LogLevel.Info, "Postpone {0} entry until {1}: {2}. [GUID={3}.] (This transaction is treated as a failure)", 
                            item.WorkQueueTypeEnum, newScheduledTime, reasonText, item.GetKey());
                item.FailureDescription = reasonText;
                PostProcessingFailure(item, WorkQueueProcessorFailureType.NonFatal);
                return;
            }

            Platform.Log(LogLevel.Info, "Postpone {0} entry until {1}: {2}. [GUID={3}]", item.WorkQueueTypeEnum, newScheduledTime, reasonText, item.GetKey());

            using (IUpdateContext updateContext = PersistentStoreRegistry.GetDefaultStore().OpenUpdateContext(UpdateContextSyncMode.Flush))
            {
                IPostponeWorkQueue broker = updateContext.GetBroker<IPostponeWorkQueue>();


                PostponeWorkQueueParameters parameters = new PostponeWorkQueueParameters
                                                             {
                                                                 WorkQueueKey = item.Key,
                                                                 Reason = reasonText,
                                                                 ScheduledTime = newScheduledTime,
                                                                 ExpirationTime = expireTime,
                                                                 UpdateWorkQueue = updateWorkQueueEntry
                                                             };

                if (broker.Execute(parameters) == false)
                {
                    Platform.Log(LogLevel.Error, "Unable to reschedule {0} WorkQueue GUID: {1}", item.WorkQueueTypeEnum, item.GetKey().ToString());
                }
                else
                {
                    updateContext.Commit();
                }
            }
        }
        protected override void ProcessItem(Model.WorkQueue item)
        {
            Platform.CheckMemberIsSet(StorageLocation, "StorageLocation");
            Platform.CheckForNullReference(Study, "Study doesn't exist");
            
            if (WorkQueueUidList.Count == 0)
            {
                // we are done. Just need to cleanup the duplicate folder
                Platform.Log(LogLevel.Info, "{0} is completed. Cleaning up duplicate storage folder. (GUID={1}, action={2})",
                             item.WorkQueueTypeEnum, item.GetKey().Key, _processDuplicateEntry.QueueData.Action);
                
                CleanUpReconcileFolders();

                PostProcessing(item, WorkQueueProcessorStatus.Complete, WorkQueueProcessorDatabaseUpdate.ResetQueueState);
            }
            else
            {
                Platform.Log(LogLevel.Info, "Processing {0} entry (GUID={1}, action={2})",
                             item.WorkQueueTypeEnum, item.GetKey().Key, _processDuplicateEntry.QueueData.Action);

                Platform.CheckTrue(Directory.Exists(DuplicateFolder), String.Format("Duplicate Folder {0} doesn't exist.", DuplicateFolder));

                LogWorkQueueInfo();

                EnsureStorageLocationIsWritable(StorageLocation);

                _currentStudyInfo = StudyInformation.CreateFrom(Study);

                ImageSetDetails duplicateSopDetails = null;

                // If deleting duplicates then don't log the history
                if (_processDuplicateEntry.QueueData.Action != ProcessDuplicateAction.Delete && !HistoryLogged)
                {
                    duplicateSopDetails = LoadDuplicateDetails();
                }

                try
                {

                    UpdateStudyOrDuplicates();

                    int count = ProcessUidList();

                    // If deleting duplicates then don't log the history
                    if (_processDuplicateEntry.QueueData.Action != ProcessDuplicateAction.Delete &&
                        !HistoryLogged && duplicateSopDetails != null && count > 0)
                    {
                        LogHistory(duplicateSopDetails);
                    }

                    PostProcessing(item, WorkQueueProcessorStatus.Pending, WorkQueueProcessorDatabaseUpdate.None);
                }
                finally
                {
                    UpdateQueueData();
                }
            }

        }
Example #7
0
        private static WorkQueueDetails CreateEditWorkQueueItemDetails(Model.WorkQueue item)
        {
            string studyPath;
            try
            {
                StudyStorageLocation storage = WorkQueueController.GetLoadStorageLocation(item);
                studyPath = storage.GetStudyPath();
            }
            catch(Exception)
            {
                studyPath = string.Empty;
            }
            var detail = new WorkQueueDetails
                             {
                                 Key = item.Key,
                                 ScheduledDateTime = item.ScheduledTime,
                                 ExpirationTime = item.ExpirationTime,
                                 InsertTime = item.InsertTime,
                                 FailureCount = item.FailureCount,
                                 Type = item.WorkQueueTypeEnum,
                                 Status = item.WorkQueueStatusEnum,
                                 Priority = item.WorkQueuePriorityEnum,
                                 FailureDescription = item.FailureDescription,
                                 ServerDescription = item.ProcessorID,
                                 StorageLocationPath = studyPath
                             };



            // Fetch UIDs
            var wqUidsAdaptor = new WorkQueueUidAdaptor();
            var uidCriteria = new WorkQueueUidSelectCriteria();
            uidCriteria.WorkQueueKey.EqualTo(item.GetKey());
            IList<WorkQueueUid> uids = wqUidsAdaptor.Get(uidCriteria);

            var mapSeries = new Hashtable();
            foreach (WorkQueueUid uid in uids)
            {
                if (mapSeries.ContainsKey(uid.SeriesInstanceUid) == false)
                    mapSeries.Add(uid.SeriesInstanceUid, uid.SopInstanceUid);
            }

            detail.NumInstancesPending = uids.Count;
            detail.NumSeriesPending = mapSeries.Count;


            // Fetch the study and patient info
            var ssAdaptor = new StudyStorageAdaptor();
            StudyStorage storages = ssAdaptor.Get(item.StudyStorageKey);

            var studyAdaptor = new StudyAdaptor();
            var studycriteria = new StudySelectCriteria();
            studycriteria.StudyInstanceUid.EqualTo(storages.StudyInstanceUid);
            studycriteria.ServerPartitionKey.EqualTo(item.ServerPartitionKey);
            Study study = studyAdaptor.GetFirst(studycriteria);

            // Study may not be available until the images are processed.
            if (study != null)
            {
                var studyAssembler = new StudyDetailsAssembler();
                detail.Study = studyAssembler.CreateStudyDetail(study);
            }

            var parser = new EditStudyWorkQueueDataParser();
            EditStudyWorkQueueData data = parser.Parse(item.Data.DocumentElement);

            detail.EditUpdateItems = data.EditRequest.UpdateEntries.ToArray();

            return detail;
        }
 public WorkQueueProcessorContext(Model.WorkQueue item)
     :base(item.GetKey().Key.ToString())
 {
     _item = item;
     
 }
        /// <summary>
        /// Simple routine for abort (fail) a work queue item immediately.
        /// </summary>
        /// <param name="item">The item to fail.</param>
        /// <param name="failureDescription">The reason for the failure.</param>
        /// <param name="generateAlert"></param>
        protected virtual void AbortQueueItem(Model.WorkQueue item, string failureDescription, bool generateAlert)
        {
            int retryCount = 0;
            while (true)
            {
                try
                {
                    int count = retryCount;
                    DBUpdateTime.Add(
                        delegate
                        {
                            #region Fail the WorkQueue entry
                            using (IUpdateContext updateContext = PersistentStoreRegistry.GetDefaultStore().OpenUpdateContext(UpdateContextSyncMode.Flush))
                            {
                                if (count>0)
                                    Platform.Log(LogLevel.Error, "Abort {0} WorkQueue entry ({1}). Retry # {2}. Reason: {3}", item.WorkQueueTypeEnum, item.GetKey(), count, failureDescription);
                                else
                                    Platform.Log(LogLevel.Error, "Abort {0} WorkQueue entry ({1}). Reason: {2}", item.WorkQueueTypeEnum, item.GetKey(), failureDescription);
                                IUpdateWorkQueue broker = updateContext.GetBroker<IUpdateWorkQueue>();
                                UpdateWorkQueueParameters parms = new UpdateWorkQueueParameters
                                                                      {
                                                                          ProcessorID = ServerPlatform.ProcessorId,
                                                                          WorkQueueKey = item.GetKey(),
                                                                          StudyStorageKey = item.StudyStorageKey,
                                                                          FailureCount = item.FailureCount + 1,
                                                                          FailureDescription = failureDescription,
                                                                          WorkQueueStatusEnum = WorkQueueStatusEnum.Failed,
                                                                          ScheduledTime = Platform.Time,
                                                                          ExpirationTime = Platform.Time.AddDays(1)
                                                                      };

                                if (false == broker.Execute(parms))
                                {
                                    Platform.Log(LogLevel.Error, "Unable to update {0} WorkQueue GUID: {1}", item.WorkQueueTypeEnum, item.GetKey().ToString());
                                }
                                else
                                {
                                    updateContext.Commit();
                                }
                            }
                            #endregion
                        });
                    break; // done
                }
                catch (Exception ex)
                {

                    if (ex is PersistenceException || ex is SqlException)
                    {
                        if (retryCount > MAX_DB_RETRY)
                        {
                            Platform.Log(LogLevel.Error, ex, "Error occurred when calling AbortQueueItem. Max db retry count has been reached.");
                            throw;
                        }

                        Platform.Log(LogLevel.Error, ex, "Error occurred when calling AbortQueueItem. Retry later. GUID={0}", item.Key);
                        SleepForRetry();

                        // Service is stoping
                        if (CancelPending)
                        {
                            Platform.Log(LogLevel.Warn, "Stop is requested. Attempt to abort WorkQueue entry is now terminated.");
                            break;
                        }
                        retryCount++;
                    }
                    else
                        throw;
                }
            }

        }
        /// <summary>
        /// Updates the 'State' of the filesystem associated with the 'FilesystemDelete' <see cref="ServiceLock"/> item
        /// </summary>
        /// <param name="item"></param>
        /// <param name="fs"></param>
        private static void UpdateState(Model.ServiceLock item, ServerFilesystemInfo fs)
        {
            FilesystemState state = null;
            if (item.State != null && item.State.DocumentElement!=null)
            {
                //load from datatabase
                state = XmlUtils.Deserialize<FilesystemState>(item.State.DocumentElement);
            }

            if (state == null)
                state = new FilesystemState();

            if (fs.AboveHighWatermark)
            {
                // we don't want to generate alert if the filesystem is offline or not accessible.
                if (fs.Online && (fs.Readable || fs.Writeable))
                {
                    TimeSpan ALERT_INTERVAL = TimeSpan.FromMinutes(ServiceLockSettings.Default.HighWatermarkAlertInterval);

                    if (state.AboveHighWatermarkTimestamp == null)
                        state.AboveHighWatermarkTimestamp = Platform.Time;

                    TimeSpan elapse = (state.LastHighWatermarkAlertTimestamp != null) ? Platform.Time - state.LastHighWatermarkAlertTimestamp.Value : Platform.Time - state.AboveHighWatermarkTimestamp.Value;

                    if (elapse.Duration() >= ALERT_INTERVAL)
                    {
                        ServerPlatform.Alert(AlertCategory.System, AlertLevel.Warning, "Filesystem",
                                             AlertTypeCodes.LowResources, null, TimeSpan.Zero,
                                             SR.AlertFilesystemAboveHW,
                                             fs.Filesystem.Description,
                                             TimeSpanFormatter.Format(Platform.Time - state.AboveHighWatermarkTimestamp.Value, true));


                        state.LastHighWatermarkAlertTimestamp = Platform.Time;
                    }
                }
                else
                {
                    state.AboveHighWatermarkTimestamp = null;
                    state.LastHighWatermarkAlertTimestamp = null;
                }
            }
            else
            {
                state.AboveHighWatermarkTimestamp = null;
                state.LastHighWatermarkAlertTimestamp = null;
            }


            XmlDocument stateXml = new XmlDocument();
            stateXml.AppendChild(stateXml.ImportNode(XmlUtils.Serialize(state), true));

            IPersistentStore store = PersistentStoreRegistry.GetDefaultStore();
            using (IUpdateContext ctx = store.OpenUpdateContext(UpdateContextSyncMode.Flush))
            {
                ServiceLockUpdateColumns columns = new ServiceLockUpdateColumns();
                columns.State = stateXml;

                IServiceLockEntityBroker broker = ctx.GetBroker<IServiceLockEntityBroker>();
                broker.Update(item.GetKey(), columns);
                ctx.Commit();
            }
        }
        /// <summary>
        /// Set a <see cref="ServiceLock"/> entry to pending.
        /// </summary>
        /// <param name="item">The <see cref="ServiceLock"/> entry to set.</param>
        /// <param name="scheduledTime"></param>
        /// <param name="enabled">Bool telling if the ServiceLock entry should be enabled after unlock.</param>
        protected static void UnlockServiceLock(Model.ServiceLock item, bool enabled, DateTime scheduledTime)
        {
            using (IUpdateContext updateContext = PersistentStoreRegistry.GetDefaultStore().OpenUpdateContext(UpdateContextSyncMode.Flush))
            {
                // Update the WorkQueue item status and times.
                IUpdateServiceLock update = updateContext.GetBroker<IUpdateServiceLock>();

                ServiceLockUpdateParameters parms = new ServiceLockUpdateParameters
                                                    	{
                                                    		ServiceLockKey = item.GetKey(),
                                                    		Lock = false,
                                                    		ScheduledTime = scheduledTime,
                                                    		ProcessorId = item.ProcessorId,
                                                    		Enabled = enabled
                                                    	};

            	if (false == update.Execute(parms))
                {
                    Platform.Log(LogLevel.Error, "Unable to update StudyLock GUID Status: {0}",
                                 item.GetKey().ToString());
                }

                updateContext.Commit();
            }
        }
		protected override void ProcessItem(Model.WorkQueue item)
		{
			LoadUids(item);

			if (WorkQueueUidList.Count == 0)
			{
				// No UIDs associated with the WorkQueue item.  Set the status back to idle
				PostProcessing(item,
				               WorkQueueProcessorStatus.Idle,
				               WorkQueueProcessorDatabaseUpdate.ResetQueueState);
				return;
			}


			XmlElement element = item.Data.DocumentElement;

			string syntax = element.Attributes["syntax"].Value;

			TransferSyntax compressSyntax = TransferSyntax.GetTransferSyntax(syntax);
			if (compressSyntax == null)
			{
				item.FailureDescription =
					String.Format("Invalid transfer syntax in compression WorkQueue item: {0}", element.Attributes["syntax"].Value);
				Platform.Log(LogLevel.Error, "Error with work queue item {0}: {1}", item.GetKey(), item.FailureDescription);
				base.PostProcessingFailure(item, WorkQueueProcessorFailureType.Fatal);
				return;
			}

			if (Study == null)
			{
				item.FailureDescription =
					String.Format("Compression item does not have a linked Study record");
				Platform.Log(LogLevel.Error, "Error with work queue item {0}: {1}", item.GetKey(), item.FailureDescription);
				base.PostProcessingFailure(item, WorkQueueProcessorFailureType.Fatal);
				return;
			}

			Platform.Log(LogLevel.Info,
			             "Compressing study {0} for Patient {1} (PatientId:{2} A#:{3}) on partition {4} to {5}",
			             Study.StudyInstanceUid, Study.PatientsName, Study.PatientId,
			             Study.AccessionNumber, ServerPartition.Description, compressSyntax.Name);

			IDicomCodecFactory[] codecs = DicomCodecRegistry.GetCodecFactories();
			IDicomCodecFactory theCodecFactory = null;
			foreach (IDicomCodecFactory codec in codecs)
				if (codec.CodecTransferSyntax.Equals(compressSyntax))
				{
					theCodecFactory = codec;
					break;
				}

			if (theCodecFactory == null)
			{
				item.FailureDescription = String.Format("Unable to find codec for compression: {0}", compressSyntax.Name);
				Platform.Log(LogLevel.Error, "Error with work queue item {0}: {1}", item.GetKey(), item.FailureDescription);
				base.PostProcessingFailure(item, WorkQueueProcessorFailureType.Fatal);
				return;
			}

			if (!ProcessUidList(item, theCodecFactory))
				PostProcessingFailure(item, WorkQueueProcessorFailureType.NonFatal);
			else
			{
				Platform.Log(LogLevel.Info,
				             "Completed Compressing study {0} for Patient {1} (PatientId:{2} A#:{3}) on partition {4} to {5}",
				             Study.StudyInstanceUid, Study.PatientsName, Study.PatientId,
				             Study.AccessionNumber, ServerPartition.Description, compressSyntax.Name);


				if (compressSyntax.LossyCompressed)
					UpdateStudyStatus(StorageLocation, StudyStatusEnum.OnlineLossy, compressSyntax);
				else
					UpdateStudyStatus(StorageLocation, StudyStatusEnum.OnlineLossless, compressSyntax);

				PostProcessing(item,
				               WorkQueueProcessorStatus.Pending,
				               WorkQueueProcessorDatabaseUpdate.None); // batch processed, not complete
			}
		}
 /// <summary>
 /// Creates an instance of <see cref="ServiceLockProcessorContext"/>
 /// </summary>
 /// <param name="item"></param>
 public ServiceLockProcessorContext(Model.ServiceLock item)
     :base(item.GetKey().Key.ToString())
 {
     Platform.CheckForNullReference(item, "item");
     _item = item;
 }
Example #14
0
    	/// <summary>
        /// Simple routine for failing a work queue item.
        /// </summary>
        /// <param name="item">The item to fail.</param>
		/// <param name="failureDescription">The reason for the failure.</param>
        private void FailQueueItem(Model.WorkQueue item, string failureDescription)
        {
            // Must retry to reset the status of the entry in case of db error
            // Failure to do so will create stale work queue entry (stuck in "In Progress" state)
            // which can only be recovered by restarting the service.
            while(true) 
            {
                try
                {
                	WorkQueueTypeProperties prop = _propertiesDictionary[item.WorkQueueTypeEnum];
                    using (IUpdateContext updateContext = _store.OpenUpdateContext(UpdateContextSyncMode.Flush))
                    {
                        IUpdateWorkQueue update = updateContext.GetBroker<IUpdateWorkQueue>();
                        UpdateWorkQueueParameters parms = new UpdateWorkQueueParameters
                                                          	{
                                                          		ProcessorID = ServerPlatform.ProcessorId,
                                                          		WorkQueueKey = item.GetKey(),
                                                          		StudyStorageKey = item.StudyStorageKey,
                                                          		FailureCount = item.FailureCount + 1,
                                                          		FailureDescription = failureDescription
                                                          	};

                    	var settings = WorkQueueSettings.Instance;
                        if ((item.FailureCount + 1) > prop.MaxFailureCount)
                        {
                            Platform.Log(LogLevel.Error,
                                         "Failing {0} WorkQueue entry ({1}), reached max retry count of {2}. Failure Reason: {3}",
                                         item.WorkQueueTypeEnum, item.GetKey(), item.FailureCount + 1, failureDescription);
                            parms.WorkQueueStatusEnum = WorkQueueStatusEnum.Failed;
                            parms.ScheduledTime = Platform.Time;
                            parms.ExpirationTime = Platform.Time.AddDays(1);

                            OnWorkQueueEntryFailed(item, failureDescription);
                        }
                        else
                        {
                            Platform.Log(LogLevel.Error,
                                         "Resetting {0} WorkQueue entry ({1}) to Pending, current retry count {2}. Failure Reason: {3}",
                                         item.WorkQueueTypeEnum, item.GetKey(), item.FailureCount + 1, failureDescription);
                            parms.WorkQueueStatusEnum = WorkQueueStatusEnum.Pending;
                            parms.ScheduledTime = Platform.Time.AddMilliseconds(settings.WorkQueueQueryDelay);
                            parms.ExpirationTime =
                                Platform.Time.AddSeconds((prop.MaxFailureCount - item.FailureCount) *
                                                         prop.FailureDelaySeconds);
                        }

                        if (false == update.Execute(parms))
                        {
                            Platform.Log(LogLevel.Error, "Unable to update {0} WorkQueue GUID: {1}", item.WorkQueueTypeEnum,
                                         item.GetKey().ToString());
                        }
                        else
                        {
                            updateContext.Commit();
                            break; // done
                        }
                    }                    
                }
                catch(Exception ex)
                {
                    Platform.Log(LogLevel.Error,  "Error occurred when calling FailQueueItem. Retry later. {0}", ex.Message);
                    _terminateEvent.WaitOne(2000, false);
                    if (_stop)
                    {
                        Platform.Log(LogLevel.Warn, "Service is stopping. Retry to fail the entry is terminated.");
                        break;
                    }                        
                }                
            }
        }
Example #15
0
		/// <summary>
		/// The actual delegate 
		/// </summary>
		/// <param name="processor"></param>
		/// <param name="queueItem"></param>
		private void ExecuteProcessor(IWorkQueueItemProcessor processor, Model.WorkQueue queueItem)
		{
			try
			{
				processor.Process(queueItem);
			}
			catch (Exception e)
			{
				Platform.Log(LogLevel.Error, e,
				             "Unexpected exception when processing WorkQueue item of type {0}.  Failing Queue item. (GUID: {1})",
				             queueItem.WorkQueueTypeEnum,
				             queueItem.GetKey());
				String error = e.InnerException != null ? e.InnerException.Message : e.Message;

				FailQueueItem(queueItem, error);
			}
			finally
			{
                // Signal the parent thread, so it can query again
                _threadStop.Set();

                // Cleanup the processor
                processor.Dispose();
			}

		}
		/// <summary>
		/// Set a status of <see cref="WorkQueue"/> item after batch processing has been completed.
		/// </summary>
		/// <remarks>
		/// <para>
		/// This routine will set the status of the <paramref name="item"/> to one of the followings
		/// <list type="bullet">
		/// <item>Failed: if the current process failed and number of retries has been reached.</item>
		/// <item>Pending: if the current batch has been processed successfully</item>
		/// <item>Idle : if current batch size = 0.</item>
		/// <item>Completed: if batch size =0 (idle) and the item has expired.</item>
		/// </list>
		/// </para>
		/// </remarks>
		/// <param name="item">The <see cref="WorkQueue"/> item to set.</param>
		/// <param name="status">Indicates if complete.</param>
		/// <param name="resetQueueStudyState">Reset the queue study state back to Idle</param>
		protected virtual void PostProcessing(Model.WorkQueue item, WorkQueueProcessorStatus status, WorkQueueProcessorDatabaseUpdate resetQueueStudyState)
		{
		    Completed = status == WorkQueueProcessorStatus.Complete
		                     || (status == WorkQueueProcessorStatus.Idle && item.ExpirationTime < Platform.Time);
            if (Completed)
            {
                if (WorkQueueSettings.Instance.EnableStudyIntegrityValidation)
                {
                    Platform.Log(LogLevel.Info, "{0} has completed (GUID={1})", item.WorkQueueTypeEnum, item.GetKey().Key);
                    VerifyStudy(StorageLocation);
                }
            }

			DBUpdateTime.Add(
				delegate
				{
					using (
						IUpdateContext updateContext =
							PersistentStoreRegistry.GetDefaultStore().OpenUpdateContext(UpdateContextSyncMode.Flush))
					{
						IUpdateWorkQueue update = updateContext.GetBroker<IUpdateWorkQueue>();
						UpdateWorkQueueParameters parms = new UpdateWorkQueueParameters
						                                      {
						                                          WorkQueueKey = item.GetKey(),
						                                          StudyStorageKey = item.StudyStorageKey,
						                                          ProcessorID = item.ProcessorID
						                                      };

						DateTime now = Platform.Time;

						if (item.FailureDescription != null)
							parms.FailureDescription = item.FailureDescription;

						DateTime scheduledTime = now.AddSeconds(WorkQueueProperties.ProcessDelaySeconds);


						if (scheduledTime > item.ExpirationTime)
							scheduledTime = item.ExpirationTime;

                        if (status == WorkQueueProcessorStatus.CompleteDelayDelete)
                        {
                            parms.WorkQueueStatusEnum = WorkQueueStatusEnum.Idle;
                            parms.FailureCount = item.FailureCount;
                            parms.FailureDescription = "";
                            parms.ScheduledTime = parms.ExpirationTime = Platform.Time.AddSeconds(WorkQueueProperties.DeleteDelaySeconds);
                            if (resetQueueStudyState == WorkQueueProcessorDatabaseUpdate.ResetQueueState)
                                parms.QueueStudyStateEnum = QueueStudyStateEnum.Idle;
                        }
                        else if (status == WorkQueueProcessorStatus.Complete
							|| (status == WorkQueueProcessorStatus.Idle && item.ExpirationTime < Platform.Time))
						{
							parms.WorkQueueStatusEnum = WorkQueueStatusEnum.Completed;
							parms.FailureCount = item.FailureCount;
							parms.ScheduledTime = scheduledTime;
							if (resetQueueStudyState == WorkQueueProcessorDatabaseUpdate.ResetQueueState)
								parms.QueueStudyStateEnum = QueueStudyStateEnum.Idle;

							parms.ExpirationTime = item.ExpirationTime; // Keep the same
						    Completed = true;
						}
						else if (status == WorkQueueProcessorStatus.Idle
						      || status == WorkQueueProcessorStatus.IdleNoDelete)
						{
							scheduledTime = now.AddSeconds(WorkQueueProperties.DeleteDelaySeconds);
							if (scheduledTime > item.ExpirationTime)
								scheduledTime = item.ExpirationTime;

							parms.WorkQueueStatusEnum = WorkQueueStatusEnum.Idle;
							parms.ScheduledTime = scheduledTime;
							parms.ExpirationTime = item.ExpirationTime; // keep the same
							parms.FailureCount = item.FailureCount;
						}
						else
						{
							parms.WorkQueueStatusEnum = WorkQueueStatusEnum.Pending;

							parms.ExpirationTime = scheduledTime.AddSeconds(WorkQueueProperties.ExpireDelaySeconds);
							parms.ScheduledTime = scheduledTime;
							parms.FailureCount = item.FailureCount;
						}


						if (false == update.Execute(parms))
						{
							Platform.Log(LogLevel.Error, "Unable to update {0} WorkQueue Key: {1}", item.WorkQueueTypeEnum, item.Key.ToString());
						}
						else
							updateContext.Commit();
					}
				}
				);

		}
        /// <summary>
		/// Set a status of <see cref="WorkQueue"/> item after batch processing has been completed.
		/// </summary>
		/// <remarks>
		/// <para>
		/// This routine will set the status of the <paramref name="item"/> to one of the following
		/// <list type="bullet">
		/// <item>Failed: if the current process failed and number of retries has been reached or a fatal error.</item>
		/// <item>Pending: if the number of retries has not been reached</item>
		/// </list>
		/// </para>
		/// </remarks>
		/// <param name="item">The <see cref="WorkQueue"/> item to set.</param>
		/// <param name="processorFailureType">The failure is unrecoverable</param>
		protected virtual void PostProcessingFailure(Model.WorkQueue item, WorkQueueProcessorFailureType processorFailureType)
		{
    	    int retryCount = 0;
            while (true)
            {
                try
                {
                    #region Fail the entry

			DBUpdateTime.Add(
				delegate
					{
						using (
							IUpdateContext updateContext =
								PersistentStoreRegistry.GetDefaultStore().OpenUpdateContext(UpdateContextSyncMode.Flush))
						{
							IUpdateWorkQueue update = updateContext.GetBroker<IUpdateWorkQueue>();
							UpdateWorkQueueParameters parms = new UpdateWorkQueueParameters
							                                      {
							                                          WorkQueueKey = item.GetKey(),
							                                          StudyStorageKey = item.StudyStorageKey,
							                                          ProcessorID = item.ProcessorID
							                                      };

						    if (item.FailureDescription != null)
								parms.FailureDescription = item.FailureDescription;

							parms.FailureCount = item.FailureCount + 1;
							if (processorFailureType == WorkQueueProcessorFailureType.Fatal)
							{
								Platform.Log(LogLevel.Error,
											 "Failing {0} WorkQueue entry ({1}), fatal error: {2}",
											 item.WorkQueueTypeEnum, item.GetKey(), item.FailureDescription);

								parms.WorkQueueStatusEnum = WorkQueueStatusEnum.Failed;
								parms.ScheduledTime = Platform.Time;
								parms.ExpirationTime = Platform.Time; // expire now		

                                RaiseAlert(item, AlertLevel.Error, String.Format("Failing {0} WorkQueue entry ({1}), fatal error: {2}", item.WorkQueueTypeEnum, item.GetKey(), item.FailureDescription));
							}
							else if ((item.FailureCount + 1) > WorkQueueProperties.MaxFailureCount)
							{
								Platform.Log(LogLevel.Error,
                                             "Failing {0} WorkQueue entry ({1}), reached max retry count of {2}. Failure Reason: {3}",
											 item.WorkQueueTypeEnum, item.GetKey(), item.FailureCount + 1, item.FailureDescription);
								parms.WorkQueueStatusEnum = WorkQueueStatusEnum.Failed;
								parms.ScheduledTime = Platform.Time;
								parms.ExpirationTime = Platform.Time; // expire now


                                RaiseAlert(item, AlertLevel.Error, String.Format("Failing {0} WorkQueue entry ({1}): {2}", item.WorkQueueTypeEnum, item.GetKey(), item.FailureDescription));
							}
							else
							{
								Platform.Log(LogLevel.Error,
                                             "Resetting {0} WorkQueue entry ({1}) to Pending, current retry count {2}.",
								             item.WorkQueueTypeEnum, item.GetKey(), item.FailureCount + 1);
								parms.WorkQueueStatusEnum = WorkQueueStatusEnum.Pending;
								parms.ScheduledTime = Platform.Time.AddSeconds(WorkQueueProperties.FailureDelaySeconds);
								parms.ExpirationTime =
									   Platform.Time.AddSeconds((WorkQueueProperties.MaxFailureCount - item.FailureCount) *
									                        WorkQueueProperties.FailureDelaySeconds);
							}


							if (false == update.Execute(parms))
							{
								Platform.Log(LogLevel.Error, "Unable to update {0} WorkQueue GUID: {1}",
											 item.WorkQueueTypeEnum, item.GetKey().ToString());
							}
							else
								updateContext.Commit();
						}
					}
				);

                    break; // done
                    #endregion
		}
                catch (Exception ex)
                {
                    if (ex is PersistenceException || ex is SqlException)
                    {
                        if (retryCount > MAX_DB_RETRY)
                        {
                            Platform.Log(LogLevel.Error, ex, "Error occurred when calling PostProcessingFailure. Max db retry count has been reached.");
                            
                            // can't do anything except throwing it.
                            throw;
                        }

                        Platform.Log(LogLevel.Error, ex, "Error occurred when calling PostProcessingFailure(). Retry later. GUID={0}", item.Key);
                        SleepForRetry();

                        // If service is stoping then stop
                        if (CancelPending)
                        {
                            Platform.Log(LogLevel.Warn, "Stop is requested. Attempt to fail WorkQueue entry is now terminated.");
                            break;
                        }
                        retryCount++;
                    }
                    else
                        throw;
                }
            }
        
		}
        /// <summary>
        /// Process a <see cref="WorkQueue"/> item of type AutoRoute.
        /// </summary>
        protected override void ProcessItem(Model.WorkQueue item)
        {
            if (WorkQueueItem.ScheduledTime >= WorkQueueItem.ExpirationTime && !HasPendingItems)
            {
                Platform.Log(LogLevel.Debug, "Removing Idle {0} entry : {1}", item.WorkQueueTypeEnum, item.GetKey().Key);
                base.PostProcessing(item, WorkQueueProcessorStatus.Complete, WorkQueueProcessorDatabaseUpdate.None);
                return;
            }

            if (!HasPendingItems)
            {
                // nothing to process, change to idle state
                PostProcessing(item, WorkQueueProcessorStatus.Idle, WorkQueueProcessorDatabaseUpdate.None);
                return;
            }

            Platform.Log(LogLevel.Info,
						 "Moving study {0} for Patient {1} (PatientId:{2} A#:{3}) on Partition {4} to {5}...",
						 Study.StudyInstanceUid, Study.PatientsName, Study.PatientId, Study.AccessionNumber,
						 ServerPartition.Description, DestinationDevice.AeTitle);

            // Load remote device information from the database.
            Device device = DestinationDevice;
            if (device == null)
            {
                item.FailureDescription = String.Format("Unknown auto-route destination \"{0}\"", item.DeviceKey);
                Platform.Log(LogLevel.Error, item.FailureDescription);

                PostProcessingFailure(item, WorkQueueProcessorFailureType.Fatal); // Fatal Error
                return;
            }

            if (device.Dhcp && device.IpAddress.Length == 0)
            {
                item.FailureDescription = String.Format("Auto-route destination is a DHCP device with no known IP address: \"{0}\"", device.AeTitle);
                Platform.Log(LogLevel.Error,
                             item.FailureDescription);

                PostProcessingFailure(item, WorkQueueProcessorFailureType.Fatal); // Fatal error
                return;
            }

            
            // Now setup the StorageSCU component
            int sendCounter = 0;
			using (ImageServerStorageScu scu = new ImageServerStorageScu(ServerPartition, device))
			{
				using (ServerExecutionContext context = new ServerExecutionContext())
					// set the preferred syntax lists
					scu.LoadPreferredSyntaxes(context.ReadContext);

				// Load the Instances to Send into the SCU component
				scu.AddStorageInstanceList(InstanceList);

				// Set an event to be called when each image is transferred
				scu.ImageStoreCompleted += delegate(Object sender, StorageInstance instance)
				                           	{
				                           		if (instance.SendStatus.Status == DicomState.Success
				                           		    || instance.SendStatus.Status == DicomState.Warning
				                           		    || instance.SendStatus.Equals(DicomStatuses.SOPClassNotSupported))
				                           		{
				                           			sendCounter++;
				                           			OnInstanceSent(instance);
				                           		}

				                           		if (instance.SendStatus.Status == DicomState.Failure)
				                           		{
				                           			scu.FailureDescription = instance.SendStatus.Description;
				                           			if (false == String.IsNullOrEmpty(instance.ExtendedFailureDescription))
				                           			{
				                           				scu.FailureDescription = String.Format("{0} [{1}]", scu.FailureDescription,
				                           				                                       instance.ExtendedFailureDescription);
				                           			}
				                           		}


				                           		if (CancelPending && !(this is WebMoveStudyItemProcessor) && !scu.Canceled)
				                           		{
				                           			Platform.Log(LogLevel.Info, "Auto-route canceled due to shutdown for study: {0}",
				                           			             StorageLocation.StudyInstanceUid);
				                           			item.FailureDescription = "Operation was canceled due to server shutdown request.";
				                           			scu.Cancel();
				                           		}
				                           	};

				try
				{
					// Block until send is complete
					scu.Send();

					// Join for the thread to exit
					scu.Join();
				}
				catch (Exception ex)
				{
					Platform.Log(LogLevel.Error, ex, "Error occurs while sending images to {0} : {1}", device.AeTitle, ex.Message);
				}
				finally
				{
					if (scu.FailureDescription.Length > 0)
					{
						item.FailureDescription = scu.FailureDescription;
						scu.Status = ScuOperationStatus.Failed;
					}

					// Reset the WorkQueue entry status
					if ((InstanceList.Count > 0 && sendCounter != InstanceList.Count) // not all sop were sent
					    || scu.Status == ScuOperationStatus.Failed
					    || scu.Status == ScuOperationStatus.ConnectFailed)
					{
						PostProcessingFailure(item, WorkQueueProcessorFailureType.NonFatal); // failures occurred}
					}
					else
					{
						OnComplete();
					}
				}
			}
        }
        /// <summary>
        /// Simple routine for failing a work queue item.
        /// </summary>
        /// <param name="item">The item to fail.</param>
        /// <param name="failureDescription">The reason for the failure.</param>
        protected virtual void FailQueueItem(Model.WorkQueue item, string failureDescription)
        {
            int retryCount = 0;
            while (true)
            {
                try
                {
            DBUpdateTime.Add(
                delegate
                    {
                            #region Remove the WorkQueue entry
                        using (IUpdateContext updateContext = PersistentStoreRegistry.GetDefaultStore().OpenUpdateContext(UpdateContextSyncMode.Flush))
                        {
                            IUpdateWorkQueue update = updateContext.GetBroker<IUpdateWorkQueue>();
                            UpdateWorkQueueParameters parms = new UpdateWorkQueueParameters
                                                                  {
                                                                      ProcessorID = ServerPlatform.ProcessorId,
                                                                      WorkQueueKey = item.GetKey(),
                                                                      StudyStorageKey = item.StudyStorageKey,
                                                                      FailureCount = item.FailureCount + 1,
                                                                      FailureDescription = failureDescription
                                                                  };

                            var settings = WorkQueueSettings.Instance;
                            if ((item.FailureCount + 1) > WorkQueueProperties.MaxFailureCount)
                            {
                                Platform.Log(LogLevel.Error,
                                             "Failing {0} WorkQueue entry ({1}), reached max retry count of {2}. Failure Reason: {3}",
                                                 item.WorkQueueTypeEnum, item.GetKey(), item.FailureCount + 1,
                                                 failureDescription);
                                parms.WorkQueueStatusEnum = WorkQueueStatusEnum.Failed;
                                parms.ScheduledTime = Platform.Time;
                                parms.ExpirationTime = Platform.Time.AddDays(1);

                                RaiseAlert(item, AlertLevel.Error, String.Format("Failing {0} WorkQueue entry ({1}), reached max retry count of {2}. Failure Reason: {3}",
                                                         item.WorkQueueTypeEnum, item.GetKey(), item.FailureCount + 1, failureDescription));
                            }
                            else
                            {
                                Platform.Log(LogLevel.Error,
                                             "Resetting {0} WorkQueue entry ({1}) to Pending, current retry count {2}",
                                             item.WorkQueueTypeEnum, item.GetKey(), item.FailureCount + 1);
                                parms.WorkQueueStatusEnum = WorkQueueStatusEnum.Pending;
                                    parms.ScheduledTime =
                                        Platform.Time.AddMilliseconds(settings.WorkQueueQueryDelay);
                                parms.ExpirationTime =
                                        Platform.Time.AddSeconds((WorkQueueProperties.MaxFailureCount -
                                                                  item.FailureCount) *
															 WorkQueueProperties.FailureDelaySeconds);
                            }

                            if (false == update.Execute(parms))
                            {
                                    Platform.Log(LogLevel.Error, "Unable to update {0} WorkQueue GUID: {1}",
                                                 item.WorkQueueTypeEnum,
                                             item.GetKey().ToString());
                            }
                            else
                                updateContext.Commit();
                            } 
                            #endregion
                        });
                    break; // done
                        }
                catch (Exception ex)
                {
                    
                    if (ex is PersistenceException || ex is SqlException)
                    {
                        if (retryCount > MAX_DB_RETRY)
                        {
                            Platform.Log(LogLevel.Error, ex, "Error occurred when calling FailQueueItem. Max db retry count has been reached.");
                            throw;
                    }

                        Platform.Log(LogLevel.Error, ex, "Error occurred when calling FailQueueItem. Retry later. GUID={0}", item.Key);
                        SleepForRetry();

                        // Service is stoping
                        if (CancelPending)
                        {
                            Platform.Log(LogLevel.Warn, "Stop is requested. Attempt to fail WorkQueue entry is now terminated.");
                            break;
        }
                        retryCount++;
                    }
                    else
                        throw;
                }
            }
            
        }
Example #20
0
        /// <summary>
        /// Simple routine for failing a work queue item.
        /// </summary>
        /// <param name="item">The item to fail.</param>
        /// <param name="failureDescription">The reason for the failure.</param>
        protected override void FailQueueItem(Model.WorkQueue item, string failureDescription)
        {
            DBUpdateTime.Add(
                delegate
                {
                    using (IUpdateContext updateContext = PersistentStoreRegistry.GetDefaultStore().OpenUpdateContext(UpdateContextSyncMode.Flush))
                    {
                        IUpdateWorkQueue update = updateContext.GetBroker<IUpdateWorkQueue>();
                        UpdateWorkQueueParameters parms = new UpdateWorkQueueParameters
                                                          	{
                                                          		ProcessorID = ServerPlatform.ProcessorId,
                                                          		WorkQueueKey = item.GetKey(),
                                                          		StudyStorageKey = item.StudyStorageKey,
                                                          		FailureCount = item.FailureCount + 1,
                                                          		FailureDescription = failureDescription
                                                          	};

                    	Platform.Log(LogLevel.Error,
                                     "Failing {0} WorkQueue entry ({1}): {2}", 
                                     item.WorkQueueTypeEnum, 
                                     item.GetKey(), failureDescription);

                        parms.WorkQueueStatusEnum = WorkQueueStatusEnum.Failed;
                        parms.ScheduledTime = Platform.Time;
                        parms.ExpirationTime = Platform.Time.AddDays(1);
						
                        if (false == update.Execute(parms))
                        {
                            Platform.Log(LogLevel.Error, "Unable to update {0} WorkQueue GUID: {1}", item.WorkQueueTypeEnum,
                                         item.GetKey().ToString());
                        }
                        else
                            updateContext.Commit();
                    }
                }
                );


        }
 private void SaveState(Model.WorkQueue item, ReprocessStudyQueueData queueData)
 {
     // Update the queue state
     using (IUpdateContext updateContext = PersistentStoreRegistry.GetDefaultStore().OpenUpdateContext(UpdateContextSyncMode.Flush))
     {
         queueData.State.ExecuteAtLeastOnce = true;
         var broker = updateContext.GetBroker<IWorkQueueEntityBroker>();
         var parms = new WorkQueueUpdateColumns {Data = XmlUtils.SerializeAsXmlDoc(_queueData)};
         broker.Update(item.GetKey(), parms);
         updateContext.Commit();
     }
 }
        /// <summary>
        /// Reset the Lock for a specific <see cref="Model.ServiceLock"/> row.
        /// </summary>
        /// <param name="item">The row to reset the lock for.</param>
        private void ResetServiceLock(Model.ServiceLock item)
        {
            // keep trying just in case of db error
            while(true)
            {
                try
                {
                    using (IUpdateContext updateContext = PersistentStoreRegistry.GetDefaultStore().OpenUpdateContext(UpdateContextSyncMode.Flush))
                    {
                        // Update the ServiceLock item status and times.
                        IUpdateServiceLock update = updateContext.GetBroker<IUpdateServiceLock>();

                        ServiceLockUpdateParameters parms = new ServiceLockUpdateParameters();
                        parms.ServiceLockKey = item.GetKey();
                        parms.Lock = false;
                        parms.ScheduledTime = Platform.Time.AddMinutes(10);
                        parms.ProcessorId = item.ProcessorId;

                        if (false == update.Execute(parms))
                        {
                            Platform.Log(LogLevel.Error, "Unable to update ServiceLock GUID Status: {0}",
                                         item.GetKey().ToString());
                        }

                        updateContext.Commit();
                    }

                    break;
                }
                catch(Exception ex)
                {
                    Platform.Log(LogLevel.Error, ex, "Exception has occured when trying to reset the entry. Retry later");
                    _terminationEvent.WaitOne(2000, false);
                }
            }
            
        }