protected void HandleExceptionForTransaction(T queueItem, IQueueTransaction queueTransaction, Action oldQueueItemAction = null)
{
if (queueTransaction == null)
{
throw new ArgumentNullException(nameof(queueTransaction), "The queue transaction is null.");
}
// The queue item is null, commit the transaction.
if (queueItem == null)
{
oldQueueItemAction?.Invoke();
queueTransaction.Commit();
return;
}
try
{
var isMessageOld = DateTime.UtcNow - queueItem.AssociationDateTime > _dequeueServiceConfig.MaximumQueueMessageAge;
// If the item is not null and we haven't dequeued too many times, add to the queue at the back
if (queueItem.DequeueCount < DequeueServiceConfig.MaxDequeueCount && !isMessageOld)
{
EnqueueMessage(queueItem, _dequeueQueuePath, queueTransaction);
}
// Enqueue onto the dead letter queue if the message isn't old
else if (!isMessageOld)
{
EnqueueMessage(queueItem, _deadLetterQueuePath, queueTransaction);
LogInformation(LogEntry.Create(MessageQueueStatus.EnqueuedDeadLetter,
queueItemBase: queueItem,
sourceMessageQueuePath: _deadLetterQueuePath));
}
// Remove from all queues if the dequeue count has been reached and the message is old. Invoke any clean-up
// tasks in the action.
else
{
// Make sure this is called before commiting the transaction. This will most likely use the transaction
// to enqueue to the delete service queue.
oldQueueItemAction?.Invoke();
LogError(LogEntry.Create(MessageQueueStatus.TooOldForDeadLetterError,
queueItemBase: queueItem,
sourceMessageQueuePath: _dequeueQueuePath),
new Exception("Message dequeued too many times. Remove from all queues."));
}
queueTransaction.Commit();
}
catch (Exception e)
{
LogError(LogEntry.Create(MessageQueueStatus.TransactionExceptionHandlerError,
queueItemBase: queueItem,
sourceMessageQueuePath: _dequeueQueuePath),
e);
queueTransaction.Abort();
}
}
public T Receive <T>(IQueueTransaction transaction)
{
var trans = ((AdoNetTransaction)transaction).Transaction;
AdoNetQueueEntry row;
using (var cmd = trans.Connection.CreateCommand())
{
cmd.Transaction = trans;
cmd.CommandText = @"SELECT TOP(1) * FROM Queue" + _queueName;
row = cmd.FirstOrDefault(_mapper);
}
if (row == null)
{
return(default(T));
}
using (var cmd = trans.Connection.CreateCommand())
{
cmd.Transaction = trans;
cmd.CommandText = @"DELETE FROM Queue" + _queueName + " WHERE Id = @id";
cmd.AddParameter("id", row.Id);
cmd.ExecuteNonQuery();
}
return(JsonConvert.DeserializeObject <T>(row.Body));
}
/// <summary>
/// Uploads the files.
/// </summary>
/// <param name="dicomFiles">The dicom files.</param>
/// <param name="uploadQueueItem">The upload queue item.</param>
/// <param name="queueTransaction">The queue transaction.</param>
/// <returns>The async task.</returns>
private async Task ProcessUploadQueueItem(UploadQueueItem uploadQueueItem, IQueueTransaction queueTransaction)
{
var clientConfiguration = ApplyAETModelConfigProvider.GetAETConfigs(
_aetConfigModels,
uploadQueueItem.CalledApplicationEntityTitle,
uploadQueueItem.CallingApplicationEntityTitle);
LogTrace(LogEntry.Create(AssociationStatus.UploadProcessQueueItem));
switch (clientConfiguration.Config.AETConfigType)
{
// ML Model or ML Model with Dry Run Result
case AETConfigType.Model:
case AETConfigType.ModelWithResultDryRun:
// Load all DICOM files in the received folder.
var dicomFiles = ReadDicomFiles(uploadQueueItem.AssociationFolderPath, uploadQueueItem);
await ProcessModelConfig(dicomFiles, uploadQueueItem, queueTransaction, clientConfiguration).ConfigureAwait(false);
break;
// ML Model dry run
case AETConfigType.ModelDryRun:
// Anonymize and save the files locally for the dry run using the segmentation anonymisation protocol
await AnonymiseAndSaveDicomFilesAsync(
anonymisationProtocolId : _innerEyeSegmentationClient.SegmentationAnonymisationProtocolId,
anonymisationProtocol : _innerEyeSegmentationClient.SegmentationAnonymisationProtocol,
uploadQueueItem : uploadQueueItem,
aETConfigType : clientConfiguration.Config.AETConfigType).ConfigureAwait(false);
break;
}
}
/// <summary>
/// Processes the model configuration.
/// </summary>
/// <param name="uploadQueueItem">The upload queue item.</param>
/// <param name="dicomFiles">The dicom files.</param>
/// <param name="queueTransaction">The queue transaction.</param>
/// <param name="clientConfiguration">The client configuration.</param>
/// <returns>The waitable task.</returns>
private async Task ProcessModelConfig(
IEnumerable <DicomFile> dicomFiles,
UploadQueueItem uploadQueueItem,
IQueueTransaction queueTransaction,
ClientAETConfig clientConfiguration)
{
var modelMatchResult = ApplyAETModelConfigProvider.ApplyAETModelConfig(clientConfiguration.Config.ModelsConfig, dicomFiles);
if (modelMatchResult.Matched)
{
var model = modelMatchResult.Result;
var queueItem = await StartSegmentationAsync(model.ChannelData, uploadQueueItem, model.ModelId, model.TagReplacements.ToArray(), clientConfiguration).ConfigureAwait(false);
EnqueueMessage(queueItem, _downloadQueuePath, queueTransaction);
}
else
{
var failedDicomTags = modelMatchResult.GetDicomConstraintsDicomTags();
// Log all the tags that did not match
LogError(LogEntry.Create(AssociationStatus.UploadErrorTagsDoNotMatch,
uploadQueueItem: uploadQueueItem,
failedDicomTags: string.Join(",", failedDicomTags.Select(x => x.DictionaryEntry.Name))),
new ProcessorServiceException("Failed to find a model for the received Dicom data."));
}
}
/// <summary>
/// Begins the message queue transaction.
/// MessageQueueTransaction.Begin() can throw exceptions. This method wraps the retry logic.
/// </summary>
/// <param name="queueTransaction">The queue transaction.</param>
/// <param name="retrySeconds">The time delay between every retry.</param>
/// <param name="maximumRetry">The maximum number of retries.</param>
/// <exception cref="InvalidOperationException">If the transaction has already been started.</exception>
/// <exception cref="MessageQueueTransactionBeginException">If we failed to start a transaction after retrying.</exception>
protected void BeginMessageQueueTransaction(IQueueTransaction queueTransaction, int retrySeconds = 30, int maximumRetry = 120)
{
if (queueTransaction == null)
{
throw new ArgumentNullException(nameof(queueTransaction), "The queue transaction is null.");
}
try
{
queueTransaction.Begin();
return;
}
catch (MessageQueueTransactionBeginException e)
{
LogError(LogEntry.Create(MessageQueueStatus.BeginTransactionError),
e);
// Throw if we reach the retry limit or cancellation has been requested (service stop)
if (maximumRetry <= 0 || _cancellationTokenSource.IsCancellationRequested)
{
throw;
}
}
// Delay before retrying
_cancellationTokenSource.Token.WaitHandle.WaitOne(TimeSpan.FromSeconds(retrySeconds));
BeginMessageQueueTransaction(queueTransaction, retrySeconds, maximumRetry - 1);
}
/// <summary>
/// Enqueues the message onto the message queue.
/// Note: This method will not commit or abort the queue transaction.
/// </summary>
/// <typeparam name="TMessageType">The enqueue message type.</typeparam>
/// <param name="message">The message to enqueue.</param>
/// <param name="messageQueuePath">The queue path to enqueue the message onto.</param>
/// <param name="queueTransaction">The queue transaction.</param>
/// <exception cref="ArgumentException">If the queue path is not correct.</exception>
/// <exception cref="MessageQueuePermissionsException">If we do not have permissions to write to the message queue.</exception>
/// <exception cref="MessageQueueWriteException">If the message queue could not write to the queue.</exception>
protected void EnqueueMessage <TMessageType>(
TMessageType message,
string messageQueuePath,
IQueueTransaction queueTransaction) where TMessageType : QueueItemBase
{
try
{
GetMessageQueue(messageQueuePath).Enqueue(message, queueTransaction);
LogInformation(LogEntry.Create(MessageQueueStatus.Enqueued,
queueItemBase: message,
sourceMessageQueuePath: messageQueuePath));
}
catch (MessageQueuePermissionsException e)
{
LogError(LogEntry.Create(MessageQueueStatus.EnqueueError,
queueItemBase: message,
sourceMessageQueuePath: messageQueuePath),
e);
// We cannot recover from access violations. We should stop the service.
StopServiceAsync();
throw;
}
}
/// <summary>
/// The clean up action if the queue item has completed or failed and is now an old message.
/// </summary>
/// <param name="queueItem">The queue item.</param>
/// <param name="transaction">The message queue transaction.</param>
private void CleanUp(DownloadQueueItem queueItem, IQueueTransaction transaction)
{
if (queueItem == null || transaction == null || string.IsNullOrWhiteSpace(queueItem.ResultsDirectory))
{
return;
}
EnqueueMessage(
new DeleteQueueItem(queueItem, queueItem.ResultsDirectory),
_deleteQueuePath,
transaction);
}
/// <summary>
/// The clean up action if the queue item has completed or failed and is now an old message.
/// </summary>
/// <param name="queueItem">The queue item.</param>
/// <param name="transaction">The message queue transaction.</param>
private void CleanUp(PushQueueItem queueItem, IQueueTransaction transaction)
{
if (queueItem == null || transaction == null)
{
return;
}
EnqueueMessage(
new DeleteQueueItem(queueItem, queueItem.FilePaths),
_deleteQueuePath,
transaction);
}
/// <summary>
/// The clean up action if the queue item has completed or failed and is now an old message.
/// </summary>
/// <param name="queueItem">The queue item.</param>
/// <param name="transaction">The message queue transaction.</param>
private void CleanUp(UploadQueueItem queueItem, IQueueTransaction transaction)
{
if (queueItem == null || transaction == null)
{
return;
}
// During a clean-up with remove everything in the association folder.
EnqueueMessage(
new DeleteQueueItem(queueItem, queueItem.AssociationFolderPath),
_deleteQueuePath,
transaction);
}
public T TryReceive <T>(IQueueTransaction transaction, TimeSpan waitTimeout)
{
try
{
var trans = ((MsmqTransactionAdapter)transaction).Transaction;
var message = _queue.Receive(waitTimeout, trans);
return(Deserialize <T>(message));
}
catch (MessageQueueException ex)
{
if (ex.MessageQueueErrorCode == MessageQueueErrorCode.IOTimeout)
{
return(default(T));
}
throw;
}
}
protected static T TryDequeue <T>(IMessageQueue messageQueue, IQueueTransaction messageQueueTransaction, int timeoutMs = 2000)
{
var startTime = DateTime.UtcNow;
while ((DateTime.UtcNow - startTime).TotalMilliseconds < timeoutMs)
{
try
{
return(messageQueue.DequeueNextMessage <T>(messageQueueTransaction));
}
catch (Exception)
{
Task.WaitAll(Task.Delay(500));
}
}
throw new MessageQueueReadException("Failed to transactional dequeue.");
}
/// <summary>
/// Dequeues the next message from the queue.
/// Note: This method will not commit or abort the queue transaction.
/// </summary>
/// <param name="queueTransaction">The queue transaction.</param>
/// <param name="cancellationToken">The cancellation token.</param>
/// <returns>The de-queued object.</returns>
/// <exception cref="ArgumentException">If the queue path is not correct.</exception>
/// <exception cref="MessageQueueReadException">The queue timed out when reading.</exception>
/// <exception cref="MessageQueuePermissionsException">The queue does not have permissions to read.</exception>
protected async Task <T> DequeueNextMessageAsync(
IQueueTransaction queueTransaction,
CancellationToken cancellationToken)
{
// Check if we should move all the dead letter messages on every dequeue.
CheckMoveDeadLetterMessages();
try
{
// Note: We do not put the message queue in a using block. The service manages the lifecycle
// of message queues.
var messageItem = GetMessageQueue(_dequeueQueuePath)
.DequeueNextMessage <T>(queueTransaction);
// Increase the count on the message.
// Note: We should not check if the maximum dequeue count has been reached here. We should do that on handling exceptions.
messageItem.DequeueCount++;
LogInformation(LogEntry.Create(MessageQueueStatus.Dequeued,
queueItemBase: messageItem,
sourceMessageQueuePath: _dequeueQueuePath));
return(messageItem);
}
catch (MessageQueueReadException)
{
// Delay here before throw the exception up the next level to delay message queue reads.
await Task.Delay(DequeueServiceConfig.DequeueTimeout, cancellationToken);
throw;
}
catch (MessageQueuePermissionsException e)
{
LogError(LogEntry.Create(MessageQueueStatus.DequeueError, sourceMessageQueuePath: _dequeueQueuePath),
e);
// We cannot recover from access violations. We should stop the service.
StopServiceAsync();
throw;
}
}
public T TryReceive <T>(IQueueTransaction transaction, TimeSpan waitTimeout)
{
return(Receive <T>());
}
