public void RemoveSerializedObject(Object instance)
{
var key = Model.GetKey(instance);
var searchKey = String.Format("{0}{1}*", key, Model.KeySeparator);
var keys = this.GetKeys(searchKey).ToArray();
if (keys != null && keys.Any())
{
this.Remove(keys);
}
}
private static WorkQueueDetails CreateGeneralWorkQueueItemDetails(Model.WorkQueue item)
{
var detail = new WorkQueueDetails();
detail.Key = item.Key;
detail.ScheduledDateTime = item.ScheduledTime;
detail.ExpirationTime = item.ExpirationTime;
detail.InsertTime = item.InsertTime;
detail.FailureCount = item.FailureCount;
detail.Type = item.WorkQueueTypeEnum;
detail.Status = item.WorkQueueStatusEnum;
detail.Priority = item.WorkQueuePriorityEnum;
detail.FailureDescription = item.FailureDescription;
detail.ServerDescription = item.ProcessorID;
StudyStorageLocation storage = WorkQueueController.GetLoadStorageLocation(item);
detail.StorageLocationPath = storage.GetStudyPath();
// Fetch UIDs
var wqUidsAdaptor = new WorkQueueUidAdaptor();
var uidCriteria = new WorkQueueUidSelectCriteria();
uidCriteria.WorkQueueKey.EqualTo(item.GetKey());
IList<WorkQueueUid> uids = wqUidsAdaptor.Get(uidCriteria);
var mapSeries = new Hashtable();
foreach (WorkQueueUid uid in uids)
{
if (mapSeries.ContainsKey(uid.SeriesInstanceUid) == false)
mapSeries.Add(uid.SeriesInstanceUid, uid.SopInstanceUid);
}
detail.NumInstancesPending = uids.Count;
detail.NumSeriesPending = mapSeries.Count;
// Fetch the study and patient info
var ssAdaptor = new StudyStorageAdaptor();
StudyStorage storages = ssAdaptor.Get(item.StudyStorageKey);
var studyAdaptor = new StudyAdaptor();
var studycriteria = new StudySelectCriteria();
studycriteria.StudyInstanceUid.EqualTo(storages.StudyInstanceUid);
studycriteria.ServerPartitionKey.EqualTo(item.ServerPartitionKey);
Study study = studyAdaptor.GetFirst(studycriteria);
// Study may not be available until the images are processed.
if (study != null)
{
var studyAssembler = new StudyDetailsAssembler();
detail.Study = studyAssembler.CreateStudyDetail(study);
}
return detail;
}
public void UpdateSerializedObjectValue <T, F>(T instance, Expression <Func <T, F> > expression, F value)
{
if (value == null)
{
throw new ArgumentException("Cannot be null.", "value");
}
var prefix = Model.GetKey(instance);
var memberKey = Model.GetKeyForMemberExpression(expression.Body as MemberExpression);
var key = Model.CreateKey(prefix, memberKey);
Set(key, RedisDataFormatUtil.FormatForStorage(value));
}
public void SerializeObject(Object instance, String keyPrefix = null, TimeSpan?slidingExpiration = null)
{
var type = instance.GetType();
var instanceKey = Model.GetKey(instance);
var fullPrefix = instanceKey;
if (!String.IsNullOrEmpty(keyPrefix))
{
fullPrefix = Model.CreateKey(keyPrefix, instanceKey);
}
var members = new Dictionary <Member, Object>();
ReflectionUtil.MapToDictionary(instance, members);
foreach (var member in members)
{
var fullKey = Model.CreateKey(fullPrefix, member.Key.FullyQualifiedName);
var value = member.Value;
if (member.Key.IsBinarySerialized)
{
BinarySerializeObject(fullKey, value);
ExpireByTimespan(fullKey, slidingExpiration);
}
else
{
if (value != null)
{
Set(fullKey, RedisDataFormatUtil.FormatForStorage(value));
ExpireByTimespan(fullKey, slidingExpiration);
}
}
}
}
public void RedisSessionStateItem_Tests()
{
RedisContext.AddHost("Default", "127.0.0.1");
var sessionState = new RedisSessionStateItem();
sessionState.SessionId = System.Guid.NewGuid().ToString("N").ToLower();
sessionState.ApplicationName = "TestApplication";
sessionState.Created = DateTime.UtcNow;
sessionState.SesssionItems = new MockObject()
{
Id = 2, EmailAddress = "*****@*****.**", Specialty = Speciality.Foos, Name = "Will", Tags = new List <String>()
{
"Stuff", "More Stuff"
}
};
using (var rc = RedisContext.GetClient())
{
rc.SerializeObject(sessionState);
var deserialized = rc.DeserializeObject <RedisSessionStateItem>(Model.GetKey(sessionState));
Assert.Equal(sessionState.SessionId, deserialized.SessionId);
Assert.Equal((sessionState.SesssionItems as MockObject).EmailAddress, (deserialized.SesssionItems as MockObject).EmailAddress);
rc.UpdateSerializedObjectValue(sessionState, s => s.Locked, true);
rc.UpdateSerializedObjectValue(sessionState, s => s.LockDate, DateTime.UtcNow);
deserialized = rc.DeserializeObject <RedisSessionStateItem>(Model.GetKey(sessionState));
Assert.True(deserialized.Locked);
Assert.True(deserialized.LockDate != default(DateTime));
rc.RemoveSerializedObject(sessionState);
}
}
public ReconcileStudyWorkQueue(Model.WorkQueue workqueue)
{
Platform.CheckTrue(workqueue.WorkQueueTypeEnum.Equals(WorkQueueTypeEnum.ReconcileStudy),
String.Format("Cannot copy data from Work Queue record of type {0}",
workqueue.WorkQueueTypeEnum));
this.SetKey(workqueue.GetKey());
this.Data= workqueue.Data;
this.InsertTime = workqueue.InsertTime;
this.DeviceKey = workqueue.DeviceKey;
this.ExpirationTime = workqueue.ExpirationTime;
this.FailureCount = workqueue.FailureCount;
this.FailureDescription = workqueue.FailureDescription;
this.GroupID = workqueue.GroupID;
this.InsertTime = workqueue.InsertTime;
this.ProcessorID = workqueue.ProcessorID;
this.ScheduledTime = workqueue.ScheduledTime;
this.ServerPartitionKey = workqueue.ServerPartitionKey;
this.StudyHistoryKey = workqueue.StudyHistoryKey;
this.StudyStorageKey = workqueue.StudyStorageKey;
this.WorkQueuePriorityEnum = workqueue.WorkQueuePriorityEnum;
this.WorkQueueStatusEnum = workqueue.WorkQueueStatusEnum;
this.WorkQueueTypeEnum = this.WorkQueueTypeEnum;
}
/// <summary>
/// Load the specific SOP Instance Uids in the database for the WorkQueue item.
/// </summary>
/// <param name="item">The WorkQueue item.</param>
protected void LoadUids(Model.WorkQueue item)
{
if (_uidList==null)
{
UidsLoadTime.Add(delegate
{
using (ServerExecutionContext context = new ServerExecutionContext())
{
IWorkQueueUidEntityBroker select = context.ReadContext.GetBroker<IWorkQueueUidEntityBroker>();
WorkQueueUidSelectCriteria parms = new WorkQueueUidSelectCriteria();
parms.WorkQueueKey.EqualTo(item.GetKey());
_uidList = select.Find(parms);
_uidList = TruncateList(item, _uidList);
}
}
);
}
}
public void RaiseAlert(Model.WorkQueue queueItem, AlertLevel level, string message)
{
if (WorkQueueProperties.AlertFailedWorkQueue || level == AlertLevel.Critical)
{
ServerPlatform.Alert(AlertCategory.Application, level,
queueItem.WorkQueueTypeEnum.ToString(), AlertTypeCodes.UnableToProcess,
GetWorkQueueContextData(queueItem), TimeSpan.Zero,
"Work Queue item failed: Type={0}, GUID={1}: {2}",
queueItem.WorkQueueTypeEnum,
queueItem.GetKey(), message);
}
}
private void InternalPostponeWorkQueue(Model.WorkQueue item, DateTime newScheduledTime, DateTime expireTime, string reasonText,
bool updateWorkQueueEntry, WorkQueueProcessorFailureType? errorType)
{
if (errorType!=null)
{
Platform.Log(LogLevel.Info, "Postpone {0} entry until {1}: {2}. [GUID={3}.] (This transaction is treated as a failure)",
item.WorkQueueTypeEnum, newScheduledTime, reasonText, item.GetKey());
item.FailureDescription = reasonText;
PostProcessingFailure(item, WorkQueueProcessorFailureType.NonFatal);
return;
}
Platform.Log(LogLevel.Info, "Postpone {0} entry until {1}: {2}. [GUID={3}]", item.WorkQueueTypeEnum, newScheduledTime, reasonText, item.GetKey());
using (IUpdateContext updateContext = PersistentStoreRegistry.GetDefaultStore().OpenUpdateContext(UpdateContextSyncMode.Flush))
{
IPostponeWorkQueue broker = updateContext.GetBroker<IPostponeWorkQueue>();
PostponeWorkQueueParameters parameters = new PostponeWorkQueueParameters
{
WorkQueueKey = item.Key,
Reason = reasonText,
ScheduledTime = newScheduledTime,
ExpirationTime = expireTime,
UpdateWorkQueue = updateWorkQueueEntry
};
if (broker.Execute(parameters) == false)
{
Platform.Log(LogLevel.Error, "Unable to reschedule {0} WorkQueue GUID: {1}", item.WorkQueueTypeEnum, item.GetKey().ToString());
}
else
{
updateContext.Commit();
}
}
}
protected override void ProcessItem(Model.WorkQueue item)
{
Platform.CheckMemberIsSet(StorageLocation, "StorageLocation");
Platform.CheckForNullReference(Study, "Study doesn't exist");
if (WorkQueueUidList.Count == 0)
{
// we are done. Just need to cleanup the duplicate folder
Platform.Log(LogLevel.Info, "{0} is completed. Cleaning up duplicate storage folder. (GUID={1}, action={2})",
item.WorkQueueTypeEnum, item.GetKey().Key, _processDuplicateEntry.QueueData.Action);
CleanUpReconcileFolders();
PostProcessing(item, WorkQueueProcessorStatus.Complete, WorkQueueProcessorDatabaseUpdate.ResetQueueState);
}
else
{
Platform.Log(LogLevel.Info, "Processing {0} entry (GUID={1}, action={2})",
item.WorkQueueTypeEnum, item.GetKey().Key, _processDuplicateEntry.QueueData.Action);
Platform.CheckTrue(Directory.Exists(DuplicateFolder), String.Format("Duplicate Folder {0} doesn't exist.", DuplicateFolder));
LogWorkQueueInfo();
EnsureStorageLocationIsWritable(StorageLocation);
_currentStudyInfo = StudyInformation.CreateFrom(Study);
ImageSetDetails duplicateSopDetails = null;
// If deleting duplicates then don't log the history
if (_processDuplicateEntry.QueueData.Action != ProcessDuplicateAction.Delete && !HistoryLogged)
{
duplicateSopDetails = LoadDuplicateDetails();
}
try
{
UpdateStudyOrDuplicates();
int count = ProcessUidList();
// If deleting duplicates then don't log the history
if (_processDuplicateEntry.QueueData.Action != ProcessDuplicateAction.Delete &&
!HistoryLogged && duplicateSopDetails != null && count > 0)
{
LogHistory(duplicateSopDetails);
}
PostProcessing(item, WorkQueueProcessorStatus.Pending, WorkQueueProcessorDatabaseUpdate.None);
}
finally
{
UpdateQueueData();
}
}
}
/// <summary>
/// Simple routine for failing a work queue item.
/// </summary>
/// <param name="item">The item to fail.</param>
/// <param name="failureDescription">The reason for the failure.</param>
protected override void FailQueueItem(Model.WorkQueue item, string failureDescription)
{
DBUpdateTime.Add(
delegate
{
using (IUpdateContext updateContext = PersistentStoreRegistry.GetDefaultStore().OpenUpdateContext(UpdateContextSyncMode.Flush))
{
IUpdateWorkQueue update = updateContext.GetBroker<IUpdateWorkQueue>();
UpdateWorkQueueParameters parms = new UpdateWorkQueueParameters
{
ProcessorID = ServerPlatform.ProcessorId,
WorkQueueKey = item.GetKey(),
StudyStorageKey = item.StudyStorageKey,
FailureCount = item.FailureCount + 1,
FailureDescription = failureDescription
};
Platform.Log(LogLevel.Error,
"Failing {0} WorkQueue entry ({1}): {2}",
item.WorkQueueTypeEnum,
item.GetKey(), failureDescription);
parms.WorkQueueStatusEnum = WorkQueueStatusEnum.Failed;
parms.ScheduledTime = Platform.Time;
parms.ExpirationTime = Platform.Time.AddDays(1);
if (false == update.Execute(parms))
{
Platform.Log(LogLevel.Error, "Unable to update {0} WorkQueue GUID: {1}", item.WorkQueueTypeEnum,
item.GetKey().ToString());
}
else
updateContext.Commit();
}
}
);
}
public WorkQueueProcessorContext(Model.WorkQueue item)
:base(item.GetKey().Key.ToString())
{
_item = item;
}
private static WorkQueueDetails CreateEditWorkQueueItemDetails(Model.WorkQueue item)
{
string studyPath;
try
{
StudyStorageLocation storage = WorkQueueController.GetLoadStorageLocation(item);
studyPath = storage.GetStudyPath();
}
catch(Exception)
{
studyPath = string.Empty;
}
var detail = new WorkQueueDetails
{
Key = item.Key,
ScheduledDateTime = item.ScheduledTime,
ExpirationTime = item.ExpirationTime,
InsertTime = item.InsertTime,
FailureCount = item.FailureCount,
Type = item.WorkQueueTypeEnum,
Status = item.WorkQueueStatusEnum,
Priority = item.WorkQueuePriorityEnum,
FailureDescription = item.FailureDescription,
ServerDescription = item.ProcessorID,
StorageLocationPath = studyPath
};
// Fetch UIDs
var wqUidsAdaptor = new WorkQueueUidAdaptor();
var uidCriteria = new WorkQueueUidSelectCriteria();
uidCriteria.WorkQueueKey.EqualTo(item.GetKey());
IList<WorkQueueUid> uids = wqUidsAdaptor.Get(uidCriteria);
var mapSeries = new Hashtable();
foreach (WorkQueueUid uid in uids)
{
if (mapSeries.ContainsKey(uid.SeriesInstanceUid) == false)
mapSeries.Add(uid.SeriesInstanceUid, uid.SopInstanceUid);
}
detail.NumInstancesPending = uids.Count;
detail.NumSeriesPending = mapSeries.Count;
// Fetch the study and patient info
var ssAdaptor = new StudyStorageAdaptor();
StudyStorage storages = ssAdaptor.Get(item.StudyStorageKey);
var studyAdaptor = new StudyAdaptor();
var studycriteria = new StudySelectCriteria();
studycriteria.StudyInstanceUid.EqualTo(storages.StudyInstanceUid);
studycriteria.ServerPartitionKey.EqualTo(item.ServerPartitionKey);
Study study = studyAdaptor.GetFirst(studycriteria);
// Study may not be available until the images are processed.
if (study != null)
{
var studyAssembler = new StudyDetailsAssembler();
detail.Study = studyAssembler.CreateStudyDetail(study);
}
var parser = new EditStudyWorkQueueDataParser();
EditStudyWorkQueueData data = parser.Parse(item.Data.DocumentElement);
detail.EditUpdateItems = data.EditRequest.UpdateEntries.ToArray();
return detail;
}
/// <summary>
/// Set a status of <see cref="WorkQueue"/> item after batch processing has been completed.
/// </summary>
/// <remarks>
/// <para>
/// This routine will set the status of the <paramref name="item"/> to one of the followings
/// <list type="bullet">
/// <item>Failed: if the current process failed and number of retries has been reached.</item>
/// <item>Pending: if the current batch has been processed successfully</item>
/// <item>Idle : if current batch size = 0.</item>
/// <item>Completed: if batch size =0 (idle) and the item has expired.</item>
/// </list>
/// </para>
/// </remarks>
/// <param name="item">The <see cref="WorkQueue"/> item to set.</param>
/// <param name="status">Indicates if complete.</param>
/// <param name="resetQueueStudyState">Reset the queue study state back to Idle</param>
protected virtual void PostProcessing(Model.WorkQueue item, WorkQueueProcessorStatus status, WorkQueueProcessorDatabaseUpdate resetQueueStudyState)
{
Completed = status == WorkQueueProcessorStatus.Complete
|| (status == WorkQueueProcessorStatus.Idle && item.ExpirationTime < Platform.Time);
if (Completed)
{
if (WorkQueueSettings.Instance.EnableStudyIntegrityValidation)
{
Platform.Log(LogLevel.Info, "{0} has completed (GUID={1})", item.WorkQueueTypeEnum, item.GetKey().Key);
VerifyStudy(StorageLocation);
}
}
DBUpdateTime.Add(
delegate
{
using (
IUpdateContext updateContext =
PersistentStoreRegistry.GetDefaultStore().OpenUpdateContext(UpdateContextSyncMode.Flush))
{
IUpdateWorkQueue update = updateContext.GetBroker<IUpdateWorkQueue>();
UpdateWorkQueueParameters parms = new UpdateWorkQueueParameters
{
WorkQueueKey = item.GetKey(),
StudyStorageKey = item.StudyStorageKey,
ProcessorID = item.ProcessorID
};
DateTime now = Platform.Time;
if (item.FailureDescription != null)
parms.FailureDescription = item.FailureDescription;
DateTime scheduledTime = now.AddSeconds(WorkQueueProperties.ProcessDelaySeconds);
if (scheduledTime > item.ExpirationTime)
scheduledTime = item.ExpirationTime;
if (status == WorkQueueProcessorStatus.CompleteDelayDelete)
{
parms.WorkQueueStatusEnum = WorkQueueStatusEnum.Idle;
parms.FailureCount = item.FailureCount;
parms.FailureDescription = "";
parms.ScheduledTime = parms.ExpirationTime = Platform.Time.AddSeconds(WorkQueueProperties.DeleteDelaySeconds);
if (resetQueueStudyState == WorkQueueProcessorDatabaseUpdate.ResetQueueState)
parms.QueueStudyStateEnum = QueueStudyStateEnum.Idle;
}
else if (status == WorkQueueProcessorStatus.Complete
|| (status == WorkQueueProcessorStatus.Idle && item.ExpirationTime < Platform.Time))
{
parms.WorkQueueStatusEnum = WorkQueueStatusEnum.Completed;
parms.FailureCount = item.FailureCount;
parms.ScheduledTime = scheduledTime;
if (resetQueueStudyState == WorkQueueProcessorDatabaseUpdate.ResetQueueState)
parms.QueueStudyStateEnum = QueueStudyStateEnum.Idle;
parms.ExpirationTime = item.ExpirationTime; // Keep the same
Completed = true;
}
else if (status == WorkQueueProcessorStatus.Idle
|| status == WorkQueueProcessorStatus.IdleNoDelete)
{
scheduledTime = now.AddSeconds(WorkQueueProperties.DeleteDelaySeconds);
if (scheduledTime > item.ExpirationTime)
scheduledTime = item.ExpirationTime;
parms.WorkQueueStatusEnum = WorkQueueStatusEnum.Idle;
parms.ScheduledTime = scheduledTime;
parms.ExpirationTime = item.ExpirationTime; // keep the same
parms.FailureCount = item.FailureCount;
}
else
{
parms.WorkQueueStatusEnum = WorkQueueStatusEnum.Pending;
parms.ExpirationTime = scheduledTime.AddSeconds(WorkQueueProperties.ExpireDelaySeconds);
parms.ScheduledTime = scheduledTime;
parms.FailureCount = item.FailureCount;
}
if (false == update.Execute(parms))
{
Platform.Log(LogLevel.Error, "Unable to update {0} WorkQueue Key: {1}", item.WorkQueueTypeEnum, item.Key.ToString());
}
else
updateContext.Commit();
}
}
);
}
/// <summary>
/// Simple routine for abort (fail) a work queue item immediately.
/// </summary>
/// <param name="item">The item to fail.</param>
/// <param name="failureDescription">The reason for the failure.</param>
/// <param name="generateAlert"></param>
protected virtual void AbortQueueItem(Model.WorkQueue item, string failureDescription, bool generateAlert)
{
int retryCount = 0;
while (true)
{
try
{
int count = retryCount;
DBUpdateTime.Add(
delegate
{
#region Fail the WorkQueue entry
using (IUpdateContext updateContext = PersistentStoreRegistry.GetDefaultStore().OpenUpdateContext(UpdateContextSyncMode.Flush))
{
if (count>0)
Platform.Log(LogLevel.Error, "Abort {0} WorkQueue entry ({1}). Retry # {2}. Reason: {3}", item.WorkQueueTypeEnum, item.GetKey(), count, failureDescription);
else
Platform.Log(LogLevel.Error, "Abort {0} WorkQueue entry ({1}). Reason: {2}", item.WorkQueueTypeEnum, item.GetKey(), failureDescription);
IUpdateWorkQueue broker = updateContext.GetBroker<IUpdateWorkQueue>();
UpdateWorkQueueParameters parms = new UpdateWorkQueueParameters
{
ProcessorID = ServerPlatform.ProcessorId,
WorkQueueKey = item.GetKey(),
StudyStorageKey = item.StudyStorageKey,
FailureCount = item.FailureCount + 1,
FailureDescription = failureDescription,
WorkQueueStatusEnum = WorkQueueStatusEnum.Failed,
ScheduledTime = Platform.Time,
ExpirationTime = Platform.Time.AddDays(1)
};
if (false == broker.Execute(parms))
{
Platform.Log(LogLevel.Error, "Unable to update {0} WorkQueue GUID: {1}", item.WorkQueueTypeEnum, item.GetKey().ToString());
}
else
{
updateContext.Commit();
}
}
#endregion
});
break; // done
}
catch (Exception ex)
{
if (ex is PersistenceException || ex is SqlException)
{
if (retryCount > MAX_DB_RETRY)
{
Platform.Log(LogLevel.Error, ex, "Error occurred when calling AbortQueueItem. Max db retry count has been reached.");
throw;
}
Platform.Log(LogLevel.Error, ex, "Error occurred when calling AbortQueueItem. Retry later. GUID={0}", item.Key);
SleepForRetry();
// Service is stoping
if (CancelPending)
{
Platform.Log(LogLevel.Warn, "Stop is requested. Attempt to abort WorkQueue entry is now terminated.");
break;
}
retryCount++;
}
else
throw;
}
}
}
/// <summary>
/// The actual delegate
/// </summary>
/// <param name="processor"></param>
/// <param name="queueItem"></param>
private void ExecuteProcessor(IWorkQueueItemProcessor processor, Model.WorkQueue queueItem)
{
try
{
processor.Process(queueItem);
}
catch (Exception e)
{
Platform.Log(LogLevel.Error, e,
"Unexpected exception when processing WorkQueue item of type {0}. Failing Queue item. (GUID: {1})",
queueItem.WorkQueueTypeEnum,
queueItem.GetKey());
String error = e.InnerException != null ? e.InnerException.Message : e.Message;
FailQueueItem(queueItem, error);
}
finally
{
// Signal the parent thread, so it can query again
_threadStop.Set();
// Cleanup the processor
processor.Dispose();
}
}
/// <summary>
/// Simple routine for failing a work queue item.
/// </summary>
/// <param name="item">The item to fail.</param>
/// <param name="failureDescription">The reason for the failure.</param>
private void FailQueueItem(Model.WorkQueue item, string failureDescription)
{
// Must retry to reset the status of the entry in case of db error
// Failure to do so will create stale work queue entry (stuck in "In Progress" state)
// which can only be recovered by restarting the service.
while(true)
{
try
{
WorkQueueTypeProperties prop = _propertiesDictionary[item.WorkQueueTypeEnum];
using (IUpdateContext updateContext = _store.OpenUpdateContext(UpdateContextSyncMode.Flush))
{
IUpdateWorkQueue update = updateContext.GetBroker<IUpdateWorkQueue>();
UpdateWorkQueueParameters parms = new UpdateWorkQueueParameters
{
ProcessorID = ServerPlatform.ProcessorId,
WorkQueueKey = item.GetKey(),
StudyStorageKey = item.StudyStorageKey,
FailureCount = item.FailureCount + 1,
FailureDescription = failureDescription
};
var settings = WorkQueueSettings.Instance;
if ((item.FailureCount + 1) > prop.MaxFailureCount)
{
Platform.Log(LogLevel.Error,
"Failing {0} WorkQueue entry ({1}), reached max retry count of {2}. Failure Reason: {3}",
item.WorkQueueTypeEnum, item.GetKey(), item.FailureCount + 1, failureDescription);
parms.WorkQueueStatusEnum = WorkQueueStatusEnum.Failed;
parms.ScheduledTime = Platform.Time;
parms.ExpirationTime = Platform.Time.AddDays(1);
OnWorkQueueEntryFailed(item, failureDescription);
}
else
{
Platform.Log(LogLevel.Error,
"Resetting {0} WorkQueue entry ({1}) to Pending, current retry count {2}. Failure Reason: {3}",
item.WorkQueueTypeEnum, item.GetKey(), item.FailureCount + 1, failureDescription);
parms.WorkQueueStatusEnum = WorkQueueStatusEnum.Pending;
parms.ScheduledTime = Platform.Time.AddMilliseconds(settings.WorkQueueQueryDelay);
parms.ExpirationTime =
Platform.Time.AddSeconds((prop.MaxFailureCount - item.FailureCount) *
prop.FailureDelaySeconds);
}
if (false == update.Execute(parms))
{
Platform.Log(LogLevel.Error, "Unable to update {0} WorkQueue GUID: {1}", item.WorkQueueTypeEnum,
item.GetKey().ToString());
}
else
{
updateContext.Commit();
break; // done
}
}
}
catch(Exception ex)
{
Platform.Log(LogLevel.Error, "Error occurred when calling FailQueueItem. Retry later. {0}", ex.Message);
_terminateEvent.WaitOne(2000, false);
if (_stop)
{
Platform.Log(LogLevel.Warn, "Service is stopping. Retry to fail the entry is terminated.");
break;
}
}
}
}
/// <summary>
/// Set a <see cref="ServiceLock"/> entry to pending.
/// </summary>
/// <param name="item">The <see cref="ServiceLock"/> entry to set.</param>
/// <param name="scheduledTime"></param>
/// <param name="enabled">Bool telling if the ServiceLock entry should be enabled after unlock.</param>
protected static void UnlockServiceLock(Model.ServiceLock item, bool enabled, DateTime scheduledTime)
{
using (IUpdateContext updateContext = PersistentStoreRegistry.GetDefaultStore().OpenUpdateContext(UpdateContextSyncMode.Flush))
{
// Update the WorkQueue item status and times.
IUpdateServiceLock update = updateContext.GetBroker<IUpdateServiceLock>();
ServiceLockUpdateParameters parms = new ServiceLockUpdateParameters
{
ServiceLockKey = item.GetKey(),
Lock = false,
ScheduledTime = scheduledTime,
ProcessorId = item.ProcessorId,
Enabled = enabled
};
if (false == update.Execute(parms))
{
Platform.Log(LogLevel.Error, "Unable to update StudyLock GUID Status: {0}",
item.GetKey().ToString());
}
updateContext.Commit();
}
}
protected override void ProcessItem(Model.WorkQueue item)
{
LoadUids(item);
if (WorkQueueUidList.Count == 0)
{
// No UIDs associated with the WorkQueue item. Set the status back to idle
PostProcessing(item,
WorkQueueProcessorStatus.Idle,
WorkQueueProcessorDatabaseUpdate.ResetQueueState);
return;
}
XmlElement element = item.Data.DocumentElement;
string syntax = element.Attributes["syntax"].Value;
TransferSyntax compressSyntax = TransferSyntax.GetTransferSyntax(syntax);
if (compressSyntax == null)
{
item.FailureDescription =
String.Format("Invalid transfer syntax in compression WorkQueue item: {0}", element.Attributes["syntax"].Value);
Platform.Log(LogLevel.Error, "Error with work queue item {0}: {1}", item.GetKey(), item.FailureDescription);
base.PostProcessingFailure(item, WorkQueueProcessorFailureType.Fatal);
return;
}
if (Study == null)
{
item.FailureDescription =
String.Format("Compression item does not have a linked Study record");
Platform.Log(LogLevel.Error, "Error with work queue item {0}: {1}", item.GetKey(), item.FailureDescription);
base.PostProcessingFailure(item, WorkQueueProcessorFailureType.Fatal);
return;
}
Platform.Log(LogLevel.Info,
"Compressing study {0} for Patient {1} (PatientId:{2} A#:{3}) on partition {4} to {5}",
Study.StudyInstanceUid, Study.PatientsName, Study.PatientId,
Study.AccessionNumber, ServerPartition.Description, compressSyntax.Name);
IDicomCodecFactory[] codecs = DicomCodecRegistry.GetCodecFactories();
IDicomCodecFactory theCodecFactory = null;
foreach (IDicomCodecFactory codec in codecs)
if (codec.CodecTransferSyntax.Equals(compressSyntax))
{
theCodecFactory = codec;
break;
}
if (theCodecFactory == null)
{
item.FailureDescription = String.Format("Unable to find codec for compression: {0}", compressSyntax.Name);
Platform.Log(LogLevel.Error, "Error with work queue item {0}: {1}", item.GetKey(), item.FailureDescription);
base.PostProcessingFailure(item, WorkQueueProcessorFailureType.Fatal);
return;
}
if (!ProcessUidList(item, theCodecFactory))
PostProcessingFailure(item, WorkQueueProcessorFailureType.NonFatal);
else
{
Platform.Log(LogLevel.Info,
"Completed Compressing study {0} for Patient {1} (PatientId:{2} A#:{3}) on partition {4} to {5}",
Study.StudyInstanceUid, Study.PatientsName, Study.PatientId,
Study.AccessionNumber, ServerPartition.Description, compressSyntax.Name);
if (compressSyntax.LossyCompressed)
UpdateStudyStatus(StorageLocation, StudyStatusEnum.OnlineLossy, compressSyntax);
else
UpdateStudyStatus(StorageLocation, StudyStatusEnum.OnlineLossless, compressSyntax);
PostProcessing(item,
WorkQueueProcessorStatus.Pending,
WorkQueueProcessorDatabaseUpdate.None); // batch processed, not complete
}
}
/// <summary>
/// Creates an instance of <see cref="ServiceLockProcessorContext"/>
/// </summary>
/// <param name="item"></param>
public ServiceLockProcessorContext(Model.ServiceLock item)
:base(item.GetKey().Key.ToString())
{
Platform.CheckForNullReference(item, "item");
_item = item;
}
/// <summary>
/// Set a status of <see cref="WorkQueue"/> item after batch processing has been completed.
/// </summary>
/// <remarks>
/// <para>
/// This routine will set the status of the <paramref name="item"/> to one of the following
/// <list type="bullet">
/// <item>Failed: if the current process failed and number of retries has been reached or a fatal error.</item>
/// <item>Pending: if the number of retries has not been reached</item>
/// </list>
/// </para>
/// </remarks>
/// <param name="item">The <see cref="WorkQueue"/> item to set.</param>
/// <param name="processorFailureType">The failure is unrecoverable</param>
protected virtual void PostProcessingFailure(Model.WorkQueue item, WorkQueueProcessorFailureType processorFailureType)
{
int retryCount = 0;
while (true)
{
try
{
#region Fail the entry
DBUpdateTime.Add(
delegate
{
using (
IUpdateContext updateContext =
PersistentStoreRegistry.GetDefaultStore().OpenUpdateContext(UpdateContextSyncMode.Flush))
{
IUpdateWorkQueue update = updateContext.GetBroker<IUpdateWorkQueue>();
UpdateWorkQueueParameters parms = new UpdateWorkQueueParameters
{
WorkQueueKey = item.GetKey(),
StudyStorageKey = item.StudyStorageKey,
ProcessorID = item.ProcessorID
};
if (item.FailureDescription != null)
parms.FailureDescription = item.FailureDescription;
parms.FailureCount = item.FailureCount + 1;
if (processorFailureType == WorkQueueProcessorFailureType.Fatal)
{
Platform.Log(LogLevel.Error,
"Failing {0} WorkQueue entry ({1}), fatal error: {2}",
item.WorkQueueTypeEnum, item.GetKey(), item.FailureDescription);
parms.WorkQueueStatusEnum = WorkQueueStatusEnum.Failed;
parms.ScheduledTime = Platform.Time;
parms.ExpirationTime = Platform.Time; // expire now
RaiseAlert(item, AlertLevel.Error, String.Format("Failing {0} WorkQueue entry ({1}), fatal error: {2}", item.WorkQueueTypeEnum, item.GetKey(), item.FailureDescription));
}
else if ((item.FailureCount + 1) > WorkQueueProperties.MaxFailureCount)
{
Platform.Log(LogLevel.Error,
"Failing {0} WorkQueue entry ({1}), reached max retry count of {2}. Failure Reason: {3}",
item.WorkQueueTypeEnum, item.GetKey(), item.FailureCount + 1, item.FailureDescription);
parms.WorkQueueStatusEnum = WorkQueueStatusEnum.Failed;
parms.ScheduledTime = Platform.Time;
parms.ExpirationTime = Platform.Time; // expire now
RaiseAlert(item, AlertLevel.Error, String.Format("Failing {0} WorkQueue entry ({1}): {2}", item.WorkQueueTypeEnum, item.GetKey(), item.FailureDescription));
}
else
{
Platform.Log(LogLevel.Error,
"Resetting {0} WorkQueue entry ({1}) to Pending, current retry count {2}.",
item.WorkQueueTypeEnum, item.GetKey(), item.FailureCount + 1);
parms.WorkQueueStatusEnum = WorkQueueStatusEnum.Pending;
parms.ScheduledTime = Platform.Time.AddSeconds(WorkQueueProperties.FailureDelaySeconds);
parms.ExpirationTime =
Platform.Time.AddSeconds((WorkQueueProperties.MaxFailureCount - item.FailureCount) *
WorkQueueProperties.FailureDelaySeconds);
}
if (false == update.Execute(parms))
{
Platform.Log(LogLevel.Error, "Unable to update {0} WorkQueue GUID: {1}",
item.WorkQueueTypeEnum, item.GetKey().ToString());
}
else
updateContext.Commit();
}
}
);
break; // done
#endregion
}
catch (Exception ex)
{
if (ex is PersistenceException || ex is SqlException)
{
if (retryCount > MAX_DB_RETRY)
{
Platform.Log(LogLevel.Error, ex, "Error occurred when calling PostProcessingFailure. Max db retry count has been reached.");
// can't do anything except throwing it.
throw;
}
Platform.Log(LogLevel.Error, ex, "Error occurred when calling PostProcessingFailure(). Retry later. GUID={0}", item.Key);
SleepForRetry();
// If service is stoping then stop
if (CancelPending)
{
Platform.Log(LogLevel.Warn, "Stop is requested. Attempt to fail WorkQueue entry is now terminated.");
break;
}
retryCount++;
}
else
throw;
}
}
}
/// <summary>
/// Updates the 'State' of the filesystem associated with the 'FilesystemDelete' <see cref="ServiceLock"/> item
/// </summary>
/// <param name="item"></param>
/// <param name="fs"></param>
private static void UpdateState(Model.ServiceLock item, ServerFilesystemInfo fs)
{
FilesystemState state = null;
if (item.State != null && item.State.DocumentElement!=null)
{
//load from datatabase
state = XmlUtils.Deserialize<FilesystemState>(item.State.DocumentElement);
}
if (state == null)
state = new FilesystemState();
if (fs.AboveHighWatermark)
{
// we don't want to generate alert if the filesystem is offline or not accessible.
if (fs.Online && (fs.Readable || fs.Writeable))
{
TimeSpan ALERT_INTERVAL = TimeSpan.FromMinutes(ServiceLockSettings.Default.HighWatermarkAlertInterval);
if (state.AboveHighWatermarkTimestamp == null)
state.AboveHighWatermarkTimestamp = Platform.Time;
TimeSpan elapse = (state.LastHighWatermarkAlertTimestamp != null) ? Platform.Time - state.LastHighWatermarkAlertTimestamp.Value : Platform.Time - state.AboveHighWatermarkTimestamp.Value;
if (elapse.Duration() >= ALERT_INTERVAL)
{
ServerPlatform.Alert(AlertCategory.System, AlertLevel.Warning, "Filesystem",
AlertTypeCodes.LowResources, null, TimeSpan.Zero,
SR.AlertFilesystemAboveHW,
fs.Filesystem.Description,
TimeSpanFormatter.Format(Platform.Time - state.AboveHighWatermarkTimestamp.Value, true));
state.LastHighWatermarkAlertTimestamp = Platform.Time;
}
}
else
{
state.AboveHighWatermarkTimestamp = null;
state.LastHighWatermarkAlertTimestamp = null;
}
}
else
{
state.AboveHighWatermarkTimestamp = null;
state.LastHighWatermarkAlertTimestamp = null;
}
XmlDocument stateXml = new XmlDocument();
stateXml.AppendChild(stateXml.ImportNode(XmlUtils.Serialize(state), true));
IPersistentStore store = PersistentStoreRegistry.GetDefaultStore();
using (IUpdateContext ctx = store.OpenUpdateContext(UpdateContextSyncMode.Flush))
{
ServiceLockUpdateColumns columns = new ServiceLockUpdateColumns();
columns.State = stateXml;
IServiceLockEntityBroker broker = ctx.GetBroker<IServiceLockEntityBroker>();
broker.Update(item.GetKey(), columns);
ctx.Commit();
}
}
/// <summary>
/// Simple routine for failing a work queue item.
/// </summary>
/// <param name="item">The item to fail.</param>
/// <param name="failureDescription">The reason for the failure.</param>
protected virtual void FailQueueItem(Model.WorkQueue item, string failureDescription)
{
int retryCount = 0;
while (true)
{
try
{
DBUpdateTime.Add(
delegate
{
#region Remove the WorkQueue entry
using (IUpdateContext updateContext = PersistentStoreRegistry.GetDefaultStore().OpenUpdateContext(UpdateContextSyncMode.Flush))
{
IUpdateWorkQueue update = updateContext.GetBroker<IUpdateWorkQueue>();
UpdateWorkQueueParameters parms = new UpdateWorkQueueParameters
{
ProcessorID = ServerPlatform.ProcessorId,
WorkQueueKey = item.GetKey(),
StudyStorageKey = item.StudyStorageKey,
FailureCount = item.FailureCount + 1,
FailureDescription = failureDescription
};
var settings = WorkQueueSettings.Instance;
if ((item.FailureCount + 1) > WorkQueueProperties.MaxFailureCount)
{
Platform.Log(LogLevel.Error,
"Failing {0} WorkQueue entry ({1}), reached max retry count of {2}. Failure Reason: {3}",
item.WorkQueueTypeEnum, item.GetKey(), item.FailureCount + 1,
failureDescription);
parms.WorkQueueStatusEnum = WorkQueueStatusEnum.Failed;
parms.ScheduledTime = Platform.Time;
parms.ExpirationTime = Platform.Time.AddDays(1);
RaiseAlert(item, AlertLevel.Error, String.Format("Failing {0} WorkQueue entry ({1}), reached max retry count of {2}. Failure Reason: {3}",
item.WorkQueueTypeEnum, item.GetKey(), item.FailureCount + 1, failureDescription));
}
else
{
Platform.Log(LogLevel.Error,
"Resetting {0} WorkQueue entry ({1}) to Pending, current retry count {2}",
item.WorkQueueTypeEnum, item.GetKey(), item.FailureCount + 1);
parms.WorkQueueStatusEnum = WorkQueueStatusEnum.Pending;
parms.ScheduledTime =
Platform.Time.AddMilliseconds(settings.WorkQueueQueryDelay);
parms.ExpirationTime =
Platform.Time.AddSeconds((WorkQueueProperties.MaxFailureCount -
item.FailureCount) *
WorkQueueProperties.FailureDelaySeconds);
}
if (false == update.Execute(parms))
{
Platform.Log(LogLevel.Error, "Unable to update {0} WorkQueue GUID: {1}",
item.WorkQueueTypeEnum,
item.GetKey().ToString());
}
else
updateContext.Commit();
}
#endregion
});
break; // done
}
catch (Exception ex)
{
if (ex is PersistenceException || ex is SqlException)
{
if (retryCount > MAX_DB_RETRY)
{
Platform.Log(LogLevel.Error, ex, "Error occurred when calling FailQueueItem. Max db retry count has been reached.");
throw;
}
Platform.Log(LogLevel.Error, ex, "Error occurred when calling FailQueueItem. Retry later. GUID={0}", item.Key);
SleepForRetry();
// Service is stoping
if (CancelPending)
{
Platform.Log(LogLevel.Warn, "Stop is requested. Attempt to fail WorkQueue entry is now terminated.");
break;
}
retryCount++;
}
else
throw;
}
}
}
/// <summary>
/// Process a <see cref="WorkQueue"/> item of type AutoRoute.
/// </summary>
protected override void ProcessItem(Model.WorkQueue item)
{
if (WorkQueueItem.ScheduledTime >= WorkQueueItem.ExpirationTime && !HasPendingItems)
{
Platform.Log(LogLevel.Debug, "Removing Idle {0} entry : {1}", item.WorkQueueTypeEnum, item.GetKey().Key);
base.PostProcessing(item, WorkQueueProcessorStatus.Complete, WorkQueueProcessorDatabaseUpdate.None);
return;
}
if (!HasPendingItems)
{
// nothing to process, change to idle state
PostProcessing(item, WorkQueueProcessorStatus.Idle, WorkQueueProcessorDatabaseUpdate.None);
return;
}
Platform.Log(LogLevel.Info,
"Moving study {0} for Patient {1} (PatientId:{2} A#:{3}) on Partition {4} to {5}...",
Study.StudyInstanceUid, Study.PatientsName, Study.PatientId, Study.AccessionNumber,
ServerPartition.Description, DestinationDevice.AeTitle);
// Load remote device information from the database.
Device device = DestinationDevice;
if (device == null)
{
item.FailureDescription = String.Format("Unknown auto-route destination \"{0}\"", item.DeviceKey);
Platform.Log(LogLevel.Error, item.FailureDescription);
PostProcessingFailure(item, WorkQueueProcessorFailureType.Fatal); // Fatal Error
return;
}
if (device.Dhcp && device.IpAddress.Length == 0)
{
item.FailureDescription = String.Format("Auto-route destination is a DHCP device with no known IP address: \"{0}\"", device.AeTitle);
Platform.Log(LogLevel.Error,
item.FailureDescription);
PostProcessingFailure(item, WorkQueueProcessorFailureType.Fatal); // Fatal error
return;
}
// Now setup the StorageSCU component
int sendCounter = 0;
using (ImageServerStorageScu scu = new ImageServerStorageScu(ServerPartition, device))
{
using (ServerExecutionContext context = new ServerExecutionContext())
// set the preferred syntax lists
scu.LoadPreferredSyntaxes(context.ReadContext);
// Load the Instances to Send into the SCU component
scu.AddStorageInstanceList(InstanceList);
// Set an event to be called when each image is transferred
scu.ImageStoreCompleted += delegate(Object sender, StorageInstance instance)
{
if (instance.SendStatus.Status == DicomState.Success
|| instance.SendStatus.Status == DicomState.Warning
|| instance.SendStatus.Equals(DicomStatuses.SOPClassNotSupported))
{
sendCounter++;
OnInstanceSent(instance);
}
if (instance.SendStatus.Status == DicomState.Failure)
{
scu.FailureDescription = instance.SendStatus.Description;
if (false == String.IsNullOrEmpty(instance.ExtendedFailureDescription))
{
scu.FailureDescription = String.Format("{0} [{1}]", scu.FailureDescription,
instance.ExtendedFailureDescription);
}
}
if (CancelPending && !(this is WebMoveStudyItemProcessor) && !scu.Canceled)
{
Platform.Log(LogLevel.Info, "Auto-route canceled due to shutdown for study: {0}",
StorageLocation.StudyInstanceUid);
item.FailureDescription = "Operation was canceled due to server shutdown request.";
scu.Cancel();
}
};
try
{
// Block until send is complete
scu.Send();
// Join for the thread to exit
scu.Join();
}
catch (Exception ex)
{
Platform.Log(LogLevel.Error, ex, "Error occurs while sending images to {0} : {1}", device.AeTitle, ex.Message);
}
finally
{
if (scu.FailureDescription.Length > 0)
{
item.FailureDescription = scu.FailureDescription;
scu.Status = ScuOperationStatus.Failed;
}
// Reset the WorkQueue entry status
if ((InstanceList.Count > 0 && sendCounter != InstanceList.Count) // not all sop were sent
|| scu.Status == ScuOperationStatus.Failed
|| scu.Status == ScuOperationStatus.ConnectFailed)
{
PostProcessingFailure(item, WorkQueueProcessorFailureType.NonFatal); // failures occurred}
}
else
{
OnComplete();
}
}
}
}
private void SaveState(Model.WorkQueue item, ReprocessStudyQueueData queueData)
{
// Update the queue state
using (IUpdateContext updateContext = PersistentStoreRegistry.GetDefaultStore().OpenUpdateContext(UpdateContextSyncMode.Flush))
{
queueData.State.ExecuteAtLeastOnce = true;
var broker = updateContext.GetBroker<IWorkQueueEntityBroker>();
var parms = new WorkQueueUpdateColumns {Data = XmlUtils.SerializeAsXmlDoc(_queueData)};
broker.Update(item.GetKey(), parms);
updateContext.Commit();
}
}
/// <summary>
/// Reset the Lock for a specific <see cref="Model.ServiceLock"/> row.
/// </summary>
/// <param name="item">The row to reset the lock for.</param>
private void ResetServiceLock(Model.ServiceLock item)
{
// keep trying just in case of db error
while(true)
{
try
{
using (IUpdateContext updateContext = PersistentStoreRegistry.GetDefaultStore().OpenUpdateContext(UpdateContextSyncMode.Flush))
{
// Update the ServiceLock item status and times.
IUpdateServiceLock update = updateContext.GetBroker<IUpdateServiceLock>();
ServiceLockUpdateParameters parms = new ServiceLockUpdateParameters();
parms.ServiceLockKey = item.GetKey();
parms.Lock = false;
parms.ScheduledTime = Platform.Time.AddMinutes(10);
parms.ProcessorId = item.ProcessorId;
if (false == update.Execute(parms))
{
Platform.Log(LogLevel.Error, "Unable to update ServiceLock GUID Status: {0}",
item.GetKey().ToString());
}
updateContext.Commit();
}
break;
}
catch(Exception ex)
{
Platform.Log(LogLevel.Error, ex, "Exception has occured when trying to reset the entry. Retry later");
_terminationEvent.WaitOne(2000, false);
}
}
}