/// <summary>
/// Set a status of <see cref="WorkQueue"/> item after batch processing has been completed.
/// </summary>
/// <remarks>
/// <para>
/// This routine will set the status of the <paramref name="item"/> to one of the following
/// <list type="bullet">
/// <item>Failed: if the current process failed and number of retries has been reached or a fatal error.</item>
/// <item>Pending: if the number of retries has not been reached</item>
/// </list>
/// </para>
/// </remarks>
/// <param name="item">The <see cref="WorkQueue"/> item to set.</param>
/// <param name="processorFailureType">The failure is unrecoverable</param>
protected virtual void PostProcessingFailure(Model.WorkQueue item, WorkQueueProcessorFailureType processorFailureType)
{
int retryCount = 0;
while (true)
{
try
{
#region Fail the entry
DBUpdateTime.Add(
delegate
{
using (
IUpdateContext updateContext =
PersistentStoreRegistry.GetDefaultStore().OpenUpdateContext(UpdateContextSyncMode.Flush))
{
IUpdateWorkQueue update = updateContext.GetBroker<IUpdateWorkQueue>();
UpdateWorkQueueParameters parms = new UpdateWorkQueueParameters
{
WorkQueueKey = item.GetKey(),
StudyStorageKey = item.StudyStorageKey,
ProcessorID = item.ProcessorID
};
if (item.FailureDescription != null)
parms.FailureDescription = item.FailureDescription;
parms.FailureCount = item.FailureCount + 1;
if (processorFailureType == WorkQueueProcessorFailureType.Fatal)
{
Platform.Log(LogLevel.Error,
"Failing {0} WorkQueue entry ({1}), fatal error: {2}",
item.WorkQueueTypeEnum, item.GetKey(), item.FailureDescription);
parms.WorkQueueStatusEnum = WorkQueueStatusEnum.Failed;
parms.ScheduledTime = Platform.Time;
parms.ExpirationTime = Platform.Time; // expire now
RaiseAlert(item, AlertLevel.Error, String.Format("Failing {0} WorkQueue entry ({1}), fatal error: {2}", item.WorkQueueTypeEnum, item.GetKey(), item.FailureDescription));
}
else if ((item.FailureCount + 1) > WorkQueueProperties.MaxFailureCount)
{
Platform.Log(LogLevel.Error,
"Failing {0} WorkQueue entry ({1}), reached max retry count of {2}. Failure Reason: {3}",
item.WorkQueueTypeEnum, item.GetKey(), item.FailureCount + 1, item.FailureDescription);
parms.WorkQueueStatusEnum = WorkQueueStatusEnum.Failed;
parms.ScheduledTime = Platform.Time;
parms.ExpirationTime = Platform.Time; // expire now
RaiseAlert(item, AlertLevel.Error, String.Format("Failing {0} WorkQueue entry ({1}): {2}", item.WorkQueueTypeEnum, item.GetKey(), item.FailureDescription));
}
else
{
Platform.Log(LogLevel.Error,
"Resetting {0} WorkQueue entry ({1}) to Pending, current retry count {2}.",
item.WorkQueueTypeEnum, item.GetKey(), item.FailureCount + 1);
parms.WorkQueueStatusEnum = WorkQueueStatusEnum.Pending;
parms.ScheduledTime = Platform.Time.AddSeconds(WorkQueueProperties.FailureDelaySeconds);
parms.ExpirationTime =
Platform.Time.AddSeconds((WorkQueueProperties.MaxFailureCount - item.FailureCount) *
WorkQueueProperties.FailureDelaySeconds);
}
if (false == update.Execute(parms))
{
Platform.Log(LogLevel.Error, "Unable to update {0} WorkQueue GUID: {1}",
item.WorkQueueTypeEnum, item.GetKey().ToString());
}
else
updateContext.Commit();
}
}
);
break; // done
#endregion
}
catch (Exception ex)
{
if (ex is PersistenceException || ex is SqlException)
{
if (retryCount > MAX_DB_RETRY)
{
Platform.Log(LogLevel.Error, ex, "Error occurred when calling PostProcessingFailure. Max db retry count has been reached.");
// can't do anything except throwing it.
throw;
}
Platform.Log(LogLevel.Error, ex, "Error occurred when calling PostProcessingFailure(). Retry later. GUID={0}", item.Key);
SleepForRetry();
// If service is stoping then stop
if (CancelPending)
{
Platform.Log(LogLevel.Warn, "Stop is requested. Attempt to fail WorkQueue entry is now terminated.");
break;
}
retryCount++;
}
else
throw;
}
}
}
protected void PostponeItem(DateTime newScheduledTime, DateTime expireTime, string postponeReason, WorkQueueProcessorFailureType? errorType)
{
Model.WorkQueue item = WorkQueueItem;
DBUpdateTime.Add(
delegate
{
string stuckReason;
bool updatedBefore = item.LastUpdatedTime > DateTime.MinValue;
if (updatedBefore && AppearsStuck(item, out stuckReason))
{
string reason = String.IsNullOrEmpty(stuckReason)
? String.Format("Aborted because {0}", postponeReason)
: String.Format("Aborted because {0}. {1}", postponeReason, stuckReason);
AbortQueueItem(item, reason, true);
}
else
{
InternalPostponeWorkQueue(item, newScheduledTime, expireTime, postponeReason, !updatedBefore, errorType);
}
}
);
}
private void InternalPostponeWorkQueue(Model.WorkQueue item, DateTime newScheduledTime, DateTime expireTime, string reasonText,
bool updateWorkQueueEntry, WorkQueueProcessorFailureType? errorType)
{
if (errorType!=null)
{
Platform.Log(LogLevel.Info, "Postpone {0} entry until {1}: {2}. [GUID={3}.] (This transaction is treated as a failure)",
item.WorkQueueTypeEnum, newScheduledTime, reasonText, item.GetKey());
item.FailureDescription = reasonText;
PostProcessingFailure(item, WorkQueueProcessorFailureType.NonFatal);
return;
}
Platform.Log(LogLevel.Info, "Postpone {0} entry until {1}: {2}. [GUID={3}]", item.WorkQueueTypeEnum, newScheduledTime, reasonText, item.GetKey());
using (IUpdateContext updateContext = PersistentStoreRegistry.GetDefaultStore().OpenUpdateContext(UpdateContextSyncMode.Flush))
{
IPostponeWorkQueue broker = updateContext.GetBroker<IPostponeWorkQueue>();
PostponeWorkQueueParameters parameters = new PostponeWorkQueueParameters
{
WorkQueueKey = item.Key,
Reason = reasonText,
ScheduledTime = newScheduledTime,
ExpirationTime = expireTime,
UpdateWorkQueue = updateWorkQueueEntry
};
if (broker.Execute(parameters) == false)
{
Platform.Log(LogLevel.Error, "Unable to reschedule {0} WorkQueue GUID: {1}", item.WorkQueueTypeEnum, item.GetKey().ToString());
}
else
{
updateContext.Commit();
}
}
}
private void PostponeItem(string reason, WorkQueueProcessorFailureType errorType)
{
DateTime newScheduledTime = Platform.Time.AddSeconds(WorkQueueProperties.PostponeDelaySeconds);
DateTime expireTime = newScheduledTime.Add(TimeSpan.FromMinutes(2));
PostponeItem(newScheduledTime, expireTime, reason, errorType);
}
