/// <summary>
/// Simple routine for failing a work queue item.
/// </summary>
/// <param name="item">The item to fail.</param>
/// <param name="failureDescription">The reason for the failure.</param>
protected override void FailQueueItem(Model.WorkQueue item, string failureDescription)
{
DBUpdateTime.Add(
delegate
{
using (IUpdateContext updateContext = PersistentStoreRegistry.GetDefaultStore().OpenUpdateContext(UpdateContextSyncMode.Flush))
{
IUpdateWorkQueue update = updateContext.GetBroker <IUpdateWorkQueue>();
UpdateWorkQueueParameters parms = new UpdateWorkQueueParameters
{
ProcessorID = ServerPlatform.ProcessorId,
WorkQueueKey = item.GetKey(),
StudyStorageKey = item.StudyStorageKey,
FailureCount = item.FailureCount + 1,
FailureDescription = failureDescription
};
Platform.Log(LogLevel.Error,
"Failing {0} WorkQueue entry ({1}): {2}",
item.WorkQueueTypeEnum,
item.GetKey(), failureDescription);
parms.WorkQueueStatusEnum = WorkQueueStatusEnum.Failed;
parms.ScheduledTime = Platform.Time;
parms.ExpirationTime = Platform.Time.AddDays(1);
if (false == update.Execute(parms))
{
Platform.Log(LogLevel.Error, "Unable to update {0} WorkQueue GUID: {1}", item.WorkQueueTypeEnum,
item.GetKey().ToString());
}
else
{
updateContext.Commit();
}
}
}
);
}
/// <summary>
/// Simple routine for failing a work queue item.
/// </summary>
/// <param name="item">The item to fail.</param>
/// <param name="failureDescription">The reason for the failure.</param>
private void FailQueueItem(Model.WorkQueue item, string failureDescription)
{
// Must retry to reset the status of the entry in case of db error
// Failure to do so will create stale work queue entry (stuck in "In Progress" state)
// which can only be recovered by restarting the service.
while (true)
{
try
{
WorkQueueTypeProperties prop = _propertiesDictionary[item.WorkQueueTypeEnum];
using (IUpdateContext updateContext = _store.OpenUpdateContext(UpdateContextSyncMode.Flush))
{
IUpdateWorkQueue update = updateContext.GetBroker <IUpdateWorkQueue>();
UpdateWorkQueueParameters parms = new UpdateWorkQueueParameters
{
ProcessorID = ServerPlatform.ProcessorId,
WorkQueueKey = item.GetKey(),
StudyStorageKey = item.StudyStorageKey,
FailureCount = item.FailureCount + 1,
FailureDescription = failureDescription
};
var settings = WorkQueueSettings.Instance;
if ((item.FailureCount + 1) > prop.MaxFailureCount)
{
Platform.Log(LogLevel.Error,
"Failing {0} WorkQueue entry ({1}), reached max retry count of {2}. Failure Reason: {3}",
item.WorkQueueTypeEnum, item.GetKey(), item.FailureCount + 1, failureDescription);
parms.WorkQueueStatusEnum = WorkQueueStatusEnum.Failed;
parms.ScheduledTime = Platform.Time;
parms.ExpirationTime = Platform.Time.AddDays(1);
OnWorkQueueEntryFailed(item, failureDescription);
}
else
{
Platform.Log(LogLevel.Error,
"Resetting {0} WorkQueue entry ({1}) to Pending, current retry count {2}. Failure Reason: {3}",
item.WorkQueueTypeEnum, item.GetKey(), item.FailureCount + 1, failureDescription);
parms.WorkQueueStatusEnum = WorkQueueStatusEnum.Pending;
parms.ScheduledTime = Platform.Time.AddMilliseconds(settings.WorkQueueQueryDelay);
parms.ExpirationTime =
Platform.Time.AddSeconds((prop.MaxFailureCount - item.FailureCount) *
prop.FailureDelaySeconds);
}
if (false == update.Execute(parms))
{
Platform.Log(LogLevel.Error, "Unable to update {0} WorkQueue GUID: {1}", item.WorkQueueTypeEnum,
item.GetKey().ToString());
}
else
{
updateContext.Commit();
break; // done
}
}
}
catch (Exception ex)
{
Platform.Log(LogLevel.Error, "Error occurred when calling FailQueueItem. Retry later. {0}", ex.Message);
_terminateEvent.WaitOne(2000, false);
if (_stop)
{
Platform.Log(LogLevel.Warn, "Service is stopping. Retry to fail the entry is terminated.");
break;
}
}
}
}