/// <summary>
/// Simple routine for failing a work queue item.
/// </summary>
/// <param name="item">The item to fail.</param>
/// <param name="failureDescription">The reason for the failure.</param>
protected virtual void FailQueueItem(Model.WorkQueue item, string failureDescription)
{
int retryCount = 0;
while (true)
{
try
{
DBUpdateTime.Add(
delegate
{
#region Remove the WorkQueue entry
using (IUpdateContext updateContext = PersistentStoreRegistry.GetDefaultStore().OpenUpdateContext(UpdateContextSyncMode.Flush))
{
IUpdateWorkQueue update = updateContext.GetBroker<IUpdateWorkQueue>();
UpdateWorkQueueParameters parms = new UpdateWorkQueueParameters
{
ProcessorID = ServerPlatform.ProcessorId,
WorkQueueKey = item.GetKey(),
StudyStorageKey = item.StudyStorageKey,
FailureCount = item.FailureCount + 1,
FailureDescription = failureDescription
};
var settings = WorkQueueSettings.Instance;
if ((item.FailureCount + 1) > WorkQueueProperties.MaxFailureCount)
{
Platform.Log(LogLevel.Error,
"Failing {0} WorkQueue entry ({1}), reached max retry count of {2}. Failure Reason: {3}",
item.WorkQueueTypeEnum, item.GetKey(), item.FailureCount + 1,
failureDescription);
parms.WorkQueueStatusEnum = WorkQueueStatusEnum.Failed;
parms.ScheduledTime = Platform.Time;
parms.ExpirationTime = Platform.Time.AddDays(1);
RaiseAlert(item, AlertLevel.Error, String.Format("Failing {0} WorkQueue entry ({1}), reached max retry count of {2}. Failure Reason: {3}",
item.WorkQueueTypeEnum, item.GetKey(), item.FailureCount + 1, failureDescription));
}
else
{
Platform.Log(LogLevel.Error,
"Resetting {0} WorkQueue entry ({1}) to Pending, current retry count {2}",
item.WorkQueueTypeEnum, item.GetKey(), item.FailureCount + 1);
parms.WorkQueueStatusEnum = WorkQueueStatusEnum.Pending;
parms.ScheduledTime =
Platform.Time.AddMilliseconds(settings.WorkQueueQueryDelay);
parms.ExpirationTime =
Platform.Time.AddSeconds((WorkQueueProperties.MaxFailureCount -
item.FailureCount) *
WorkQueueProperties.FailureDelaySeconds);
}
if (false == update.Execute(parms))
{
Platform.Log(LogLevel.Error, "Unable to update {0} WorkQueue GUID: {1}",
item.WorkQueueTypeEnum,
item.GetKey().ToString());
}
else
updateContext.Commit();
}
#endregion
});
break; // done
}
catch (Exception ex)
{
if (ex is PersistenceException || ex is SqlException)
{
if (retryCount > MAX_DB_RETRY)
{
Platform.Log(LogLevel.Error, ex, "Error occurred when calling FailQueueItem. Max db retry count has been reached.");
throw;
}
Platform.Log(LogLevel.Error, ex, "Error occurred when calling FailQueueItem. Retry later. GUID={0}", item.Key);
SleepForRetry();
// Service is stoping
if (CancelPending)
{
Platform.Log(LogLevel.Warn, "Stop is requested. Attempt to fail WorkQueue entry is now terminated.");
break;
}
retryCount++;
}
else
throw;
}
}
}
/// <summary>
/// Simple routine for abort (fail) a work queue item immediately.
/// </summary>
/// <param name="item">The item to fail.</param>
/// <param name="failureDescription">The reason for the failure.</param>
/// <param name="generateAlert"></param>
protected virtual void AbortQueueItem(Model.WorkQueue item, string failureDescription, bool generateAlert)
{
int retryCount = 0;
while (true)
{
try
{
int count = retryCount;
DBUpdateTime.Add(
delegate
{
#region Fail the WorkQueue entry
using (IUpdateContext updateContext = PersistentStoreRegistry.GetDefaultStore().OpenUpdateContext(UpdateContextSyncMode.Flush))
{
if (count>0)
Platform.Log(LogLevel.Error, "Abort {0} WorkQueue entry ({1}). Retry # {2}. Reason: {3}", item.WorkQueueTypeEnum, item.GetKey(), count, failureDescription);
else
Platform.Log(LogLevel.Error, "Abort {0} WorkQueue entry ({1}). Reason: {2}", item.WorkQueueTypeEnum, item.GetKey(), failureDescription);
IUpdateWorkQueue broker = updateContext.GetBroker<IUpdateWorkQueue>();
UpdateWorkQueueParameters parms = new UpdateWorkQueueParameters
{
ProcessorID = ServerPlatform.ProcessorId,
WorkQueueKey = item.GetKey(),
StudyStorageKey = item.StudyStorageKey,
FailureCount = item.FailureCount + 1,
FailureDescription = failureDescription,
WorkQueueStatusEnum = WorkQueueStatusEnum.Failed,
ScheduledTime = Platform.Time,
ExpirationTime = Platform.Time.AddDays(1)
};
if (false == broker.Execute(parms))
{
Platform.Log(LogLevel.Error, "Unable to update {0} WorkQueue GUID: {1}", item.WorkQueueTypeEnum, item.GetKey().ToString());
}
else
{
updateContext.Commit();
}
}
#endregion
});
break; // done
}
catch (Exception ex)
{
if (ex is PersistenceException || ex is SqlException)
{
if (retryCount > MAX_DB_RETRY)
{
Platform.Log(LogLevel.Error, ex, "Error occurred when calling AbortQueueItem. Max db retry count has been reached.");
throw;
}
Platform.Log(LogLevel.Error, ex, "Error occurred when calling AbortQueueItem. Retry later. GUID={0}", item.Key);
SleepForRetry();
// Service is stoping
if (CancelPending)
{
Platform.Log(LogLevel.Warn, "Stop is requested. Attempt to abort WorkQueue entry is now terminated.");
break;
}
retryCount++;
}
else
throw;
}
}
}
/// <summary>
/// Set a status of <see cref="WorkQueue"/> item after batch processing has been completed.
/// </summary>
/// <remarks>
/// <para>
/// This routine will set the status of the <paramref name="item"/> to one of the following
/// <list type="bullet">
/// <item>Failed: if the current process failed and number of retries has been reached or a fatal error.</item>
/// <item>Pending: if the number of retries has not been reached</item>
/// </list>
/// </para>
/// </remarks>
/// <param name="item">The <see cref="WorkQueue"/> item to set.</param>
/// <param name="processorFailureType">The failure is unrecoverable</param>
protected virtual void PostProcessingFailure(Model.WorkQueue item, WorkQueueProcessorFailureType processorFailureType)
{
int retryCount = 0;
while (true)
{
try
{
#region Fail the entry
DBUpdateTime.Add(
delegate
{
using (
IUpdateContext updateContext =
PersistentStoreRegistry.GetDefaultStore().OpenUpdateContext(UpdateContextSyncMode.Flush))
{
IUpdateWorkQueue update = updateContext.GetBroker<IUpdateWorkQueue>();
UpdateWorkQueueParameters parms = new UpdateWorkQueueParameters
{
WorkQueueKey = item.GetKey(),
StudyStorageKey = item.StudyStorageKey,
ProcessorID = item.ProcessorID
};
if (item.FailureDescription != null)
parms.FailureDescription = item.FailureDescription;
parms.FailureCount = item.FailureCount + 1;
if (processorFailureType == WorkQueueProcessorFailureType.Fatal)
{
Platform.Log(LogLevel.Error,
"Failing {0} WorkQueue entry ({1}), fatal error: {2}",
item.WorkQueueTypeEnum, item.GetKey(), item.FailureDescription);
parms.WorkQueueStatusEnum = WorkQueueStatusEnum.Failed;
parms.ScheduledTime = Platform.Time;
parms.ExpirationTime = Platform.Time; // expire now
RaiseAlert(item, AlertLevel.Error, String.Format("Failing {0} WorkQueue entry ({1}), fatal error: {2}", item.WorkQueueTypeEnum, item.GetKey(), item.FailureDescription));
}
else if ((item.FailureCount + 1) > WorkQueueProperties.MaxFailureCount)
{
Platform.Log(LogLevel.Error,
"Failing {0} WorkQueue entry ({1}), reached max retry count of {2}. Failure Reason: {3}",
item.WorkQueueTypeEnum, item.GetKey(), item.FailureCount + 1, item.FailureDescription);
parms.WorkQueueStatusEnum = WorkQueueStatusEnum.Failed;
parms.ScheduledTime = Platform.Time;
parms.ExpirationTime = Platform.Time; // expire now
RaiseAlert(item, AlertLevel.Error, String.Format("Failing {0} WorkQueue entry ({1}): {2}", item.WorkQueueTypeEnum, item.GetKey(), item.FailureDescription));
}
else
{
Platform.Log(LogLevel.Error,
"Resetting {0} WorkQueue entry ({1}) to Pending, current retry count {2}.",
item.WorkQueueTypeEnum, item.GetKey(), item.FailureCount + 1);
parms.WorkQueueStatusEnum = WorkQueueStatusEnum.Pending;
parms.ScheduledTime = Platform.Time.AddSeconds(WorkQueueProperties.FailureDelaySeconds);
parms.ExpirationTime =
Platform.Time.AddSeconds((WorkQueueProperties.MaxFailureCount - item.FailureCount) *
WorkQueueProperties.FailureDelaySeconds);
}
if (false == update.Execute(parms))
{
Platform.Log(LogLevel.Error, "Unable to update {0} WorkQueue GUID: {1}",
item.WorkQueueTypeEnum, item.GetKey().ToString());
}
else
updateContext.Commit();
}
}
);
break; // done
#endregion
}
catch (Exception ex)
{
if (ex is PersistenceException || ex is SqlException)
{
if (retryCount > MAX_DB_RETRY)
{
Platform.Log(LogLevel.Error, ex, "Error occurred when calling PostProcessingFailure. Max db retry count has been reached.");
// can't do anything except throwing it.
throw;
}
Platform.Log(LogLevel.Error, ex, "Error occurred when calling PostProcessingFailure(). Retry later. GUID={0}", item.Key);
SleepForRetry();
// If service is stoping then stop
if (CancelPending)
{
Platform.Log(LogLevel.Warn, "Stop is requested. Attempt to fail WorkQueue entry is now terminated.");
break;
}
retryCount++;
}
else
throw;
}
}
}
/// <summary>
/// Set a status of <see cref="WorkQueue"/> item after batch processing has been completed.
/// </summary>
/// <remarks>
/// <para>
/// This routine will set the status of the <paramref name="item"/> to one of the followings
/// <list type="bullet">
/// <item>Failed: if the current process failed and number of retries has been reached.</item>
/// <item>Pending: if the current batch has been processed successfully</item>
/// <item>Idle : if current batch size = 0.</item>
/// <item>Completed: if batch size =0 (idle) and the item has expired.</item>
/// </list>
/// </para>
/// </remarks>
/// <param name="item">The <see cref="WorkQueue"/> item to set.</param>
/// <param name="status">Indicates if complete.</param>
/// <param name="resetQueueStudyState">Reset the queue study state back to Idle</param>
protected virtual void PostProcessing(Model.WorkQueue item, WorkQueueProcessorStatus status, WorkQueueProcessorDatabaseUpdate resetQueueStudyState)
{
Completed = status == WorkQueueProcessorStatus.Complete
|| (status == WorkQueueProcessorStatus.Idle && item.ExpirationTime < Platform.Time);
if (Completed)
{
if (WorkQueueSettings.Instance.EnableStudyIntegrityValidation)
{
Platform.Log(LogLevel.Info, "{0} has completed (GUID={1})", item.WorkQueueTypeEnum, item.GetKey().Key);
VerifyStudy(StorageLocation);
}
}
DBUpdateTime.Add(
delegate
{
using (
IUpdateContext updateContext =
PersistentStoreRegistry.GetDefaultStore().OpenUpdateContext(UpdateContextSyncMode.Flush))
{
IUpdateWorkQueue update = updateContext.GetBroker<IUpdateWorkQueue>();
UpdateWorkQueueParameters parms = new UpdateWorkQueueParameters
{
WorkQueueKey = item.GetKey(),
StudyStorageKey = item.StudyStorageKey,
ProcessorID = item.ProcessorID
};
DateTime now = Platform.Time;
if (item.FailureDescription != null)
parms.FailureDescription = item.FailureDescription;
DateTime scheduledTime = now.AddSeconds(WorkQueueProperties.ProcessDelaySeconds);
if (scheduledTime > item.ExpirationTime)
scheduledTime = item.ExpirationTime;
if (status == WorkQueueProcessorStatus.CompleteDelayDelete)
{
parms.WorkQueueStatusEnum = WorkQueueStatusEnum.Idle;
parms.FailureCount = item.FailureCount;
parms.FailureDescription = "";
parms.ScheduledTime = parms.ExpirationTime = Platform.Time.AddSeconds(WorkQueueProperties.DeleteDelaySeconds);
if (resetQueueStudyState == WorkQueueProcessorDatabaseUpdate.ResetQueueState)
parms.QueueStudyStateEnum = QueueStudyStateEnum.Idle;
}
else if (status == WorkQueueProcessorStatus.Complete
|| (status == WorkQueueProcessorStatus.Idle && item.ExpirationTime < Platform.Time))
{
parms.WorkQueueStatusEnum = WorkQueueStatusEnum.Completed;
parms.FailureCount = item.FailureCount;
parms.ScheduledTime = scheduledTime;
if (resetQueueStudyState == WorkQueueProcessorDatabaseUpdate.ResetQueueState)
parms.QueueStudyStateEnum = QueueStudyStateEnum.Idle;
parms.ExpirationTime = item.ExpirationTime; // Keep the same
Completed = true;
}
else if (status == WorkQueueProcessorStatus.Idle
|| status == WorkQueueProcessorStatus.IdleNoDelete)
{
scheduledTime = now.AddSeconds(WorkQueueProperties.DeleteDelaySeconds);
if (scheduledTime > item.ExpirationTime)
scheduledTime = item.ExpirationTime;
parms.WorkQueueStatusEnum = WorkQueueStatusEnum.Idle;
parms.ScheduledTime = scheduledTime;
parms.ExpirationTime = item.ExpirationTime; // keep the same
parms.FailureCount = item.FailureCount;
}
else
{
parms.WorkQueueStatusEnum = WorkQueueStatusEnum.Pending;
parms.ExpirationTime = scheduledTime.AddSeconds(WorkQueueProperties.ExpireDelaySeconds);
parms.ScheduledTime = scheduledTime;
parms.FailureCount = item.FailureCount;
}
if (false == update.Execute(parms))
{
Platform.Log(LogLevel.Error, "Unable to update {0} WorkQueue Key: {1}", item.WorkQueueTypeEnum, item.Key.ToString());
}
else
updateContext.Commit();
}
}
);
}
/// <summary>
/// Simple routine for failing a work queue item.
/// </summary>
/// <param name="item">The item to fail.</param>
/// <param name="failureDescription">The reason for the failure.</param>
protected override void FailQueueItem(Model.WorkQueue item, string failureDescription)
{
DBUpdateTime.Add(
delegate
{
using (IUpdateContext updateContext = PersistentStoreRegistry.GetDefaultStore().OpenUpdateContext(UpdateContextSyncMode.Flush))
{
IUpdateWorkQueue update = updateContext.GetBroker<IUpdateWorkQueue>();
UpdateWorkQueueParameters parms = new UpdateWorkQueueParameters
{
ProcessorID = ServerPlatform.ProcessorId,
WorkQueueKey = item.GetKey(),
StudyStorageKey = item.StudyStorageKey,
FailureCount = item.FailureCount + 1,
FailureDescription = failureDescription
};
Platform.Log(LogLevel.Error,
"Failing {0} WorkQueue entry ({1}): {2}",
item.WorkQueueTypeEnum,
item.GetKey(), failureDescription);
parms.WorkQueueStatusEnum = WorkQueueStatusEnum.Failed;
parms.ScheduledTime = Platform.Time;
parms.ExpirationTime = Platform.Time.AddDays(1);
if (false == update.Execute(parms))
{
Platform.Log(LogLevel.Error, "Unable to update {0} WorkQueue GUID: {1}", item.WorkQueueTypeEnum,
item.GetKey().ToString());
}
else
updateContext.Commit();
}
}
);
}
/// <summary>
/// Simple routine for failing a work queue item.
/// </summary>
/// <param name="item">The item to fail.</param>
/// <param name="failureDescription">The reason for the failure.</param>
private void FailQueueItem(Model.WorkQueue item, string failureDescription)
{
// Must retry to reset the status of the entry in case of db error
// Failure to do so will create stale work queue entry (stuck in "In Progress" state)
// which can only be recovered by restarting the service.
while(true)
{
try
{
WorkQueueTypeProperties prop = _propertiesDictionary[item.WorkQueueTypeEnum];
using (IUpdateContext updateContext = _store.OpenUpdateContext(UpdateContextSyncMode.Flush))
{
IUpdateWorkQueue update = updateContext.GetBroker<IUpdateWorkQueue>();
UpdateWorkQueueParameters parms = new UpdateWorkQueueParameters
{
ProcessorID = ServerPlatform.ProcessorId,
WorkQueueKey = item.GetKey(),
StudyStorageKey = item.StudyStorageKey,
FailureCount = item.FailureCount + 1,
FailureDescription = failureDescription
};
var settings = WorkQueueSettings.Instance;
if ((item.FailureCount + 1) > prop.MaxFailureCount)
{
Platform.Log(LogLevel.Error,
"Failing {0} WorkQueue entry ({1}), reached max retry count of {2}. Failure Reason: {3}",
item.WorkQueueTypeEnum, item.GetKey(), item.FailureCount + 1, failureDescription);
parms.WorkQueueStatusEnum = WorkQueueStatusEnum.Failed;
parms.ScheduledTime = Platform.Time;
parms.ExpirationTime = Platform.Time.AddDays(1);
OnWorkQueueEntryFailed(item, failureDescription);
}
else
{
Platform.Log(LogLevel.Error,
"Resetting {0} WorkQueue entry ({1}) to Pending, current retry count {2}. Failure Reason: {3}",
item.WorkQueueTypeEnum, item.GetKey(), item.FailureCount + 1, failureDescription);
parms.WorkQueueStatusEnum = WorkQueueStatusEnum.Pending;
parms.ScheduledTime = Platform.Time.AddMilliseconds(settings.WorkQueueQueryDelay);
parms.ExpirationTime =
Platform.Time.AddSeconds((prop.MaxFailureCount - item.FailureCount) *
prop.FailureDelaySeconds);
}
if (false == update.Execute(parms))
{
Platform.Log(LogLevel.Error, "Unable to update {0} WorkQueue GUID: {1}", item.WorkQueueTypeEnum,
item.GetKey().ToString());
}
else
{
updateContext.Commit();
break; // done
}
}
}
catch(Exception ex)
{
Platform.Log(LogLevel.Error, "Error occurred when calling FailQueueItem. Retry later. {0}", ex.Message);
_terminateEvent.WaitOne(2000, false);
if (_stop)
{
Platform.Log(LogLevel.Warn, "Service is stopping. Retry to fail the entry is terminated.");
break;
}
}
}
}