/// <summary>
/// Processes the model configuration.
/// </summary>
/// <param name="uploadQueueItem">The upload queue item.</param>
/// <param name="dicomFiles">The dicom files.</param>
/// <param name="queueTransaction">The queue transaction.</param>
/// <param name="clientConfiguration">The client configuration.</param>
/// <returns>The waitable task.</returns>
private async Task ProcessModelConfig(
IEnumerable <DicomFile> dicomFiles,
UploadQueueItem uploadQueueItem,
IQueueTransaction queueTransaction,
ClientAETConfig clientConfiguration)
{
var modelMatchResult = ApplyAETModelConfigProvider.ApplyAETModelConfig(clientConfiguration.Config.ModelsConfig, dicomFiles);
if (modelMatchResult.Matched)
{
var model = modelMatchResult.Result;
var queueItem = await StartSegmentationAsync(model.ChannelData, uploadQueueItem, model.ModelId, model.TagReplacements.ToArray(), clientConfiguration).ConfigureAwait(false);
EnqueueMessage(queueItem, _downloadQueuePath, queueTransaction);
}
else
{
var failedDicomTags = modelMatchResult.GetDicomConstraintsDicomTags();
// Log all the tags that did not match
LogError(LogEntry.Create(AssociationStatus.UploadErrorTagsDoNotMatch,
uploadQueueItem: uploadQueueItem,
failedDicomTags: string.Join(",", failedDicomTags.Select(x => x.DictionaryEntry.Name))),
new ProcessorServiceException("Failed to find a model for the received Dicom data."));
}
}
/// <summary>
/// Load T or Ts from a JSON file or folder.
/// </summary>
protected void Load()
{
if (File.Exists(_settingsFileOrFolderName))
{
_ts = null;
(_t, _) = LoadFile(_settingsFileOrFolderName);
}
else if (Directory.Exists(_settingsFileOrFolderName))
{
_t = default(T);
var ts = new List <T>();
foreach (var file in Directory.EnumerateFiles(_settingsFileOrFolderName, "*.json"))
{
var(t, loaded) = LoadFile(file);
if (loaded)
{
ts.Add(t);
}
}
_ts = ts.ToArray();
}
else
{
var logEntry = LogEntry.Create(ServiceStatus.NewConfigurationError,
string.Format("Settings is neither a file nor a folder: {0}", _settingsFileOrFolderName));
logEntry.Log(_logger, LogLevel.Error);
}
}
/// <summary>
/// Entry point for the worker thread. This invokes the update method and pauses depending on the defined interval.
/// </summary>
/// <param name="cancellationToken">The cancellation token.</param>
private void Execute(CancellationToken cancellationToken)
{
// Main execution loop - if we encounter an exception during start-up, this will be false
while (!cancellationToken.IsCancellationRequested)
{
try
{
OnUpdateTickAsync(cancellationToken).Wait(cancellationToken);
}
catch (OperationCanceledException e)
{
if (!cancellationToken.IsCancellationRequested)
{
// If we got this error when the cancellation token has not been set, this needs to be logged.
// Otherwise, the service is shutting down normally and the wait has thrown an operation canceled exception.
LogError(LogEntry.Create(ServiceStatus.ExecuteError), e);
}
}
#pragma warning disable CA1031 // Do not catch general exception types
catch (Exception e)
#pragma warning restore CA1031 // Do not catch general exception types
{
LogError(LogEntry.Create(ServiceStatus.ExecuteError), e);
}
}
}
/// <summary>
/// Load T from a JSON file.
/// </summary>
/// <param name="path">Path to file.</param>
/// <returns>New <see cref="LoadJsonResult"/>.</returns>
private LoadJsonResult LoadFile(string path)
{
try
{
var jsonText = File.ReadAllText(path);
return(new LoadJsonResult(true, true, JsonConvert.DeserializeObject <T>(jsonText)));
}
catch (JsonSerializationException e)
{
var logEntry = LogEntry.Create(ServiceStatus.NewConfigurationError,
string.Format(CultureInfo.InvariantCulture, "Unable to parse settings file {0}", path));
logEntry.Log(_logger, LogLevel.Error, e);
return(new LoadJsonResult(true, false, default(T)));
}
catch (IOException e)
{
var logEntry = LogEntry.Create(ServiceStatus.NewConfigurationError,
string.Format(CultureInfo.InvariantCulture, "Unable to load settings file {0}", path));
logEntry.Log(_logger, LogLevel.Error, e);
return(new LoadJsonResult(false, false, default(T)));
}
}
/// <summary>
/// Begins the message queue transaction.
/// MessageQueueTransaction.Begin() can throw exceptions. This method wraps the retry logic.
/// </summary>
/// <param name="queueTransaction">The queue transaction.</param>
/// <param name="retrySeconds">The time delay between every retry.</param>
/// <param name="maximumRetry">The maximum number of retries.</param>
/// <exception cref="InvalidOperationException">If the transaction has already been started.</exception>
/// <exception cref="MessageQueueTransactionBeginException">If we failed to start a transaction after retrying.</exception>
protected void BeginMessageQueueTransaction(IQueueTransaction queueTransaction, int retrySeconds = 30, int maximumRetry = 120)
{
if (queueTransaction == null)
{
throw new ArgumentNullException(nameof(queueTransaction), "The queue transaction is null.");
}
try
{
queueTransaction.Begin();
return;
}
catch (MessageQueueTransactionBeginException e)
{
LogError(LogEntry.Create(MessageQueueStatus.BeginTransactionError),
e);
// Throw if we reach the retry limit or cancellation has been requested (service stop)
if (maximumRetry <= 0 || _cancellationTokenSource.IsCancellationRequested)
{
throw;
}
}
// Delay before retrying
_cancellationTokenSource.Token.WaitHandle.WaitOne(TimeSpan.FromSeconds(retrySeconds));
BeginMessageQueueTransaction(queueTransaction, retrySeconds, maximumRetry - 1);
}
/// <summary>
/// Gets a message queue or throws an exception.
/// </summary>
/// <param name="messageQueuePath">The message queue path.</param>
/// <returns>The message queue.</returns>
/// <exception cref="Exception">If any exception occured when getting the message queue.</exception>
protected IMessageQueue GetMessageQueue(string messageQueuePath)
{
if (!_messageQueues.ContainsKey(messageQueuePath))
{
try
{
var messageQueue = GatewayMessageQueue.Get(messageQueuePath);
LogInformation(LogEntry.Create(MessageQueueStatus.InitialisedQueue,
sourceMessageQueuePath: messageQueuePath));
_messageQueues[messageQueuePath] = messageQueue;
}
catch (Exception e)
{
LogError(LogEntry.Create(MessageQueueStatus.InitialiseQueueError,
sourceMessageQueuePath: messageQueuePath),
e);
throw;
}
}
return(_messageQueues[messageQueuePath]);
}
/// <summary>
/// Disposes of all managed resources.
/// </summary>
/// <param name="disposing">If we are disposing.</param>
protected virtual void Dispose(bool disposing)
{
if (_isDisposed || !disposing)
{
return;
}
OnStop();
if (_cancellationTokenSource != null)
{
_cancellationTokenSource.Dispose();
_cancellationTokenSource = null;
}
foreach (var queue in _messageQueues)
{
try
{
queue.Value.Dispose();
}
#pragma warning disable CA1031 // Do not catch general exception types
catch (Exception e)
#pragma warning restore CA1031 // Do not catch general exception types
{
LogError(LogEntry.Create(MessageQueueStatus.DisposeError,
sourceMessageQueuePath: queue.Value.QueuePath),
e);
}
}
_messageQueues.Clear();
_isDisposed = true;
}
/// <summary>
/// Enqueues the message onto the message queue.
/// Note: This method will not commit or abort the queue transaction.
/// </summary>
/// <typeparam name="TMessageType">The enqueue message type.</typeparam>
/// <param name="message">The message to enqueue.</param>
/// <param name="messageQueuePath">The queue path to enqueue the message onto.</param>
/// <param name="queueTransaction">The queue transaction.</param>
/// <exception cref="ArgumentException">If the queue path is not correct.</exception>
/// <exception cref="MessageQueuePermissionsException">If we do not have permissions to write to the message queue.</exception>
/// <exception cref="MessageQueueWriteException">If the message queue could not write to the queue.</exception>
protected void EnqueueMessage <TMessageType>(
TMessageType message,
string messageQueuePath,
IQueueTransaction queueTransaction) where TMessageType : QueueItemBase
{
try
{
GetMessageQueue(messageQueuePath).Enqueue(message, queueTransaction);
LogInformation(LogEntry.Create(MessageQueueStatus.Enqueued,
queueItemBase: message,
sourceMessageQueuePath: messageQueuePath));
}
catch (MessageQueuePermissionsException e)
{
LogError(LogEntry.Create(MessageQueueStatus.EnqueueError,
queueItemBase: message,
sourceMessageQueuePath: messageQueuePath),
e);
// We cannot recover from access violations. We should stop the service.
StopServiceAsync();
throw;
}
}
/// <summary>
/// Disposes of all managed resources.
/// </summary>
/// <param name="disposing">If we are disposing.</param>
protected virtual void Dispose(bool disposing)
{
if (_isDisposed || !disposing)
{
return;
}
OnStop();
if (_cancellationTokenSource != null)
{
_cancellationTokenSource.Dispose();
_cancellationTokenSource = null;
}
foreach (var queue in _messageQueues)
{
try
{
queue.Value.Dispose();
}
catch (Exception e)
{
LogError(LogEntry.Create(MessageQueueStatus.DisposeError,
sourceMessageQueuePath: queue.Value.QueuePath),
e);
}
}
_messageQueues.Clear();
_isDisposed = true;
}
/// <summary>
/// Starts the service.
/// </summary>
public void Start()
{
_cancellationTokenSource = new CancellationTokenSource();
try
{
LogInformation(LogEntry.Create(ServiceStatus.Starting));
// Start the service
OnServiceStart();
// Note: We should only start-up the worker task and return. We should not execute any long running operations here.
for (var i = 0; i < _executionTasks.Length; i++)
{
_executionTasks[i] = Task.Run(() => Execute(_cancellationTokenSource.Token), _cancellationTokenSource.Token);
_startCount++;
}
_isRunning = true;
LogInformation(LogEntry.Create(ServiceStatus.Started));
}
#pragma warning disable CA1031 // Do not catch general exception types
catch (Exception e)
#pragma warning restore CA1031 // Do not catch general exception types
{
LogError(LogEntry.Create(ServiceStatus.StartError), e);
// Stop the service on any start-up failure
StopServiceAsync();
}
}
/// <summary>
/// Called when the service is stopping.
/// </summary>
public void OnStop()
{
LogInformation(LogEntry.Create(ServiceStatus.Stopping));
try
{
_cancellationTokenSource?.Cancel();
}
catch (Exception e)
{
LogError(LogEntry.Create(ServiceStatus.StoppingError,
information: "Cancellation token source cancel exception."),
e);
}
try
{
// Will only wait 10 seconds for all tasks to finish nicely
Task.WaitAll(_executionTasks.Where(x => x != null).ToArray(), TimeSpan.FromSeconds(10));
}
catch (Exception e)
{
LogError(LogEntry.Create(ServiceStatus.StoppingError,
information: "Unknown exception waiting for all execution tasks to end."),
e);
}
OnServiceStop();
_isRunning = false;
LogInformation(LogEntry.Create(ServiceStatus.Stopped));
}
protected void HandleExceptionForTransaction(T queueItem, IQueueTransaction queueTransaction, Action oldQueueItemAction = null)
{
if (queueTransaction == null)
{
throw new ArgumentNullException(nameof(queueTransaction), "The queue transaction is null.");
}
// The queue item is null, commit the transaction.
if (queueItem == null)
{
oldQueueItemAction?.Invoke();
queueTransaction.Commit();
return;
}
try
{
var isMessageOld = DateTime.UtcNow - queueItem.AssociationDateTime > _dequeueServiceConfig.MaximumQueueMessageAge;
// If the item is not null and we haven't dequeued too many times, add to the queue at the back
if (queueItem.DequeueCount < DequeueServiceConfig.MaxDequeueCount && !isMessageOld)
{
EnqueueMessage(queueItem, _dequeueQueuePath, queueTransaction);
}
// Enqueue onto the dead letter queue if the message isn't old
else if (!isMessageOld)
{
EnqueueMessage(queueItem, _deadLetterQueuePath, queueTransaction);
LogInformation(LogEntry.Create(MessageQueueStatus.EnqueuedDeadLetter,
queueItemBase: queueItem,
sourceMessageQueuePath: _deadLetterQueuePath));
}
// Remove from all queues if the dequeue count has been reached and the message is old. Invoke any clean-up
// tasks in the action.
else
{
// Make sure this is called before commiting the transaction. This will most likely use the transaction
// to enqueue to the delete service queue.
oldQueueItemAction?.Invoke();
LogError(LogEntry.Create(MessageQueueStatus.TooOldForDeadLetterError,
queueItemBase: queueItem,
sourceMessageQueuePath: _dequeueQueuePath),
new Exception("Message dequeued too many times. Remove from all queues."));
}
queueTransaction.Commit();
}
catch (Exception e)
{
LogError(LogEntry.Create(MessageQueueStatus.TransactionExceptionHandlerError,
queueItemBase: queueItem,
sourceMessageQueuePath: _dequeueQueuePath),
e);
queueTransaction.Abort();
}
}
/// <summary>
/// Uploads the files.
/// </summary>
/// <param name="dicomFiles">The dicom files.</param>
/// <param name="uploadQueueItem">The upload queue item.</param>
/// <param name="queueTransaction">The queue transaction.</param>
/// <returns>The async task.</returns>
private async Task ProcessUploadQueueItem(UploadQueueItem uploadQueueItem, IQueueTransaction queueTransaction)
{
var clientConfiguration = ApplyAETModelConfigProvider.GetAETConfigs(
_aetConfigModels,
uploadQueueItem.CalledApplicationEntityTitle,
uploadQueueItem.CallingApplicationEntityTitle);
LogTrace(LogEntry.Create(AssociationStatus.UploadProcessQueueItem));
switch (clientConfiguration.Config.AETConfigType)
{
// ML Model or ML Model with Dry Run Result
case AETConfigType.Model:
case AETConfigType.ModelWithResultDryRun:
// Load all DICOM files in the received folder.
var dicomFiles = ReadDicomFiles(uploadQueueItem.AssociationFolderPath, uploadQueueItem);
await ProcessModelConfig(dicomFiles, uploadQueueItem, queueTransaction, clientConfiguration).ConfigureAwait(false);
break;
// ML Model dry run
case AETConfigType.ModelDryRun:
// Anonymize and save the files locally for the dry run using the segmentation anonymisation protocol
await AnonymiseAndSaveDicomFilesAsync(
anonymisationProtocolId : _innerEyeSegmentationClient.SegmentationAnonymisationProtocolId,
anonymisationProtocol : _innerEyeSegmentationClient.SegmentationAnonymisationProtocol,
uploadQueueItem : uploadQueueItem,
aETConfigType : clientConfiguration.Config.AETConfigType).ConfigureAwait(false);
break;
}
}
/// <summary>
/// Given a set of possible relative paths, find one that is a directory.
/// </summary>
/// <param name="relativePaths">List of relative paths to test.</param>
/// <param name="logger">Logger.</param>
/// <returns>Full path to existing directory or Empty if none exist.</returns>
public static string FindRelativeDirectory(IEnumerable <string> relativePaths, ILogger logger)
{
relativePaths = relativePaths ?? throw new ArgumentNullException(nameof(relativePaths));
var parentDirectory = new DirectoryInfo(Assembly.GetExecutingAssembly().Location).Parent.FullName;
foreach (var relativePath in relativePaths)
{
var configurationPath = Path.GetFullPath(Path.Combine(parentDirectory, relativePath));
if (Directory.Exists(configurationPath))
{
var logEntry = LogEntry.Create(ServiceStatus.Starting,
string.Format(CultureInfo.InvariantCulture, "Settings location: {0}", configurationPath));
logEntry.Log(logger, LogLevel.Information);
return(configurationPath);
}
}
var logEntry2 = LogEntry.Create(ServiceStatus.Starting);
logEntry2.Log(logger, LogLevel.Error, new ConfigurationException("Cannot find configuration directory."));
return(string.Empty);
}
/// <summary>
/// Moves the queue messages from the source queue to the destination queue.
/// </summary>
/// <param name="sourceQueuePath">The source queue path.</param>
/// <param name="destinationQueuePath">The destination queue path.</param>
/// <exception cref="ArgumentException">If the queue path is not correct.</exception>
private void MoveQueueMessages(string sourceQueuePath, string destinationQueuePath)
{
// Move messages from the source queue to the destination
var sourceQueue = GetMessageQueue(sourceQueuePath);
var destinationQueue = GetMessageQueue(destinationQueuePath);
var moreMessages = true;
// Keep de-queueing until no more messages on the dead letter queue
while (moreMessages)
{
// Create a new transaction for-each move request
using (var transaction = CreateQueueTransaction(sourceQueuePath))
{
transaction.Begin();
try
{
var message = sourceQueue.DequeueNextMessage <T>(transaction);
LogInformation(LogEntry.Create(MessageQueueStatus.Moved,
queueItemBase: message,
destinationMessageQueuePath: destinationQueuePath,
sourceMessageQueuePath: sourceQueuePath));
destinationQueue.Enqueue(message, transaction);
transaction.Commit();
}
catch (MessageQueueReadException)
{
// No more messages
moreMessages = false;
transaction.Commit();
}
catch (MessageQueueWriteException e)
{
LogError(LogEntry.Create(MessageQueueStatus.MoveError,
information: "Failed to wite a message from the dead letter queue to origin queue",
destinationMessageQueuePath: destinationQueuePath,
sourceMessageQueuePath: sourceQueuePath),
e);
// Abort on a write exception
transaction.Abort();
}
#pragma warning disable CA1031 // Do not catch general exception types
catch (Exception e)
#pragma warning restore CA1031 // Do not catch general exception types
{
LogError(LogEntry.Create(MessageQueueStatus.MoveError,
information: "Failed to dequeue a message from the dead letter queue",
destinationMessageQueuePath: destinationQueuePath,
sourceMessageQueuePath: sourceQueuePath),
e);
transaction.Commit();
}
}
}
}
/// <summary>
/// receive unity log message. see
/// <see cref="UnityEngine.Application.logMessageReceived"/>
/// </summary>
/// <param name="logString"></param>
/// <param name="stackTrace"></param>
/// <param name="logType"></param>
private void Application_logMessageReceived(string logString, string stackTrace, UnityEngine.LogType logType)
{
var entry = LogEntry.Create(logString, stackTrace, logType);
lock (m_LogQueueLock)
{
m_LogQueue.Enqueue(entry);
}
}
/// <summary>
/// Called when [update tick] is called. This will wait for all work to execute then will pause for desired interval delay.
/// </summary>
/// <param name="cancellationToken">The cancellation token.</param>
/// <returns>
/// The async task.
/// </returns>
protected override async Task OnUpdateTickAsync(CancellationToken cancellationToken)
{
using (var transaction = CreateQueueTransaction())
{
BeginMessageQueueTransaction(transaction);
UploadQueueItem queueItem = null;
try
{
queueItem = await DequeueNextMessageAsync(transaction, cancellationToken).ConfigureAwait(false);
// If the directory does not exist we cannot process this queue item.
// Lets log but remove this queue item.
if (Directory.Exists(queueItem.AssociationFolderPath))
{
await ProcessUploadQueueItem(queueItem, transaction).ConfigureAwait(false);
// Enqueue the message to delete the association folder
CleanUp(queueItem, transaction);
}
else
{
LogError(LogEntry.Create(AssociationStatus.UploadErrorAssocationFolderDeleted, uploadQueueItem: queueItem),
new ProcessorServiceException("The association folder has been deleted."));
}
transaction.Commit();
}
catch (MessageQueueReadException)
{
// We timed out trying to de-queue (no items on the queue).
// This exception doesn't need to be logged.
transaction.Abort();
}
catch (OperationCanceledException)
{
// Throw operation canceled exceptions up to the worker thread. It will handle
// logging correctly.
transaction.Abort();
throw;
}
#pragma warning disable CA1031 // Do not catch general exception types
catch (Exception e)
#pragma warning restore CA1031 // Do not catch general exception types
{
LogError(LogEntry.Create(AssociationStatus.UploadError, uploadQueueItem: queueItem),
e);
HandleExceptionForTransaction(
queueItem: queueItem,
queueTransaction: transaction,
oldQueueItemAction: () => CleanUp(queueItem, transaction));
}
}
}
/// <summary>
/// Gets the segmentation result.
/// </summary>
/// <param name="downloadQueueItem">The download queue item.</param>
/// <param name="retryDelay">The delay between getting segmentation progress.</param>
/// <param name="timeout">The maximum time we will wait for a segmentation result.</param>
/// <param name="cancellationToken">The cancellation token.</param>
/// <returns>The segmentation result Dicom file or null.</returns>
private async Task <DicomFile> GetSegmentationResultAsync(DownloadQueueItem downloadQueueItem, TimeSpan retryDelay, TimeSpan timeout, CancellationToken cancellationToken)
{
var referenceDicomFiles = OpenDicomFiles(downloadQueueItem.ReferenceDicomFiles);
// Attempting to download result
LogInformation(LogEntry.Create(AssociationStatus.Downloading,
downloadQueueItem: downloadQueueItem,
downloadProgress: 0,
downloadError: string.Empty));
var tagReplacements = JsonConvert.DeserializeObject <IEnumerable <TagReplacement> >(downloadQueueItem.TagReplacementJsonString);
// Create a new token for the maximum time we will sit and wait for a result.
// Note: We need to check both the passed cancellation token and this new token for cancellation requests
using (var cancellationTokenSource = new CancellationTokenSource(timeout))
{
while (!cancellationTokenSource.IsCancellationRequested && !cancellationToken.IsCancellationRequested)
{
var modelResult = await _innerEyeSegmentationClient.SegmentationResultAsync(
modelId : downloadQueueItem.ModelId,
segmentationId : downloadQueueItem.SegmentationID,
referenceDicomFiles : referenceDicomFiles,
userReplacements : tagReplacements);
if (modelResult.Progress == 100 && modelResult.DicomResult != null)
{
return(modelResult.DicomResult);
}
else if (!string.IsNullOrEmpty(modelResult.Error))
{
LogError(LogEntry.Create(AssociationStatus.Downloading,
downloadQueueItem: downloadQueueItem,
downloadProgress: modelResult.Progress,
downloadError: modelResult.Error),
new Exception("Failed to get a segmentation result."));
// We cannot recover from this error, so we log and continue.
return(null);
}
else
{
LogInformation(LogEntry.Create(AssociationStatus.Downloading,
downloadQueueItem: downloadQueueItem,
downloadProgress: modelResult.Progress,
downloadError: modelResult.Error));
}
// Make sure you pass the cancellation token, not the timeout token, so the service can stop timely
await Task.Delay(retryDelay, cancellationToken);
}
}
return(null);
}
protected List <LogEntry> CreateLogEntries(IEnumerable <JObject> jObjects)
{
var logEntries = new List <LogEntry>();
foreach (var jObject in jObjects)
{
if (string.Equals(_type, jObject.Value <string>("type"), StringComparison.Ordinal))
{
logEntries.Add(LogEntry.Create(jObject));
}
}
return(logEntries);
}
/// <summary>
/// Load T from a JSON file or folder.
/// </summary>
/// <returns>True if new config has been loaded, false otherwise.</returns>
private bool Load()
{
if (File.Exists(_settingsFileOrFolderName))
{
var loadJsonResult = LoadFile(_settingsFileOrFolderName);
if (!loadJsonResult.Loaded || !loadJsonResult.Parsed)
{
return(false);
}
Config = loadJsonResult.Result;
return(true);
}
else if (Directory.Exists(_settingsFileOrFolderName))
{
var ts = new List <T>();
foreach (var file in Directory.EnumerateFiles(_settingsFileOrFolderName, "*.json"))
{
var loadJsonResult = LoadFile(file);
if (!loadJsonResult.Loaded)
{
// File still in use, FileWatcher has reported file changed but
// the other process has not finished yet.
return(false);
}
if (loadJsonResult.Parsed)
{
ts.Add(loadJsonResult.Result);
}
}
Config = _flatMap(ts);
return(true);
}
else
{
var logEntry = LogEntry.Create(ServiceStatus.NewConfigurationError,
string.Format(CultureInfo.InvariantCulture, "Settings is neither a file nor a folder: {0}", _settingsFileOrFolderName));
logEntry.Log(_logger, LogLevel.Error);
return(false);
}
}
/// <summary>
/// Called when [update tick] is called. This will wait for all work to execute then will pause for desired interval delay.
/// </summary>
/// <param name="cancellationToken">The cancellation token.</param>
/// <returns>
/// The async task.
/// </returns>
protected override async Task OnUpdateTickAsync(CancellationToken cancellationToken)
{
using (var transaction = CreateQueueTransaction())
{
BeginMessageQueueTransaction(transaction);
DeleteQueueItem deleteQueueItem = null;
try
{
deleteQueueItem = await DequeueNextMessageAsync(transaction, cancellationToken).ConfigureAwait(false);
// Delete every path in the queue item. Each path could be a directory or a file.
foreach (var path in deleteQueueItem.Paths)
{
DeletePath(path);
}
LogInformation(LogEntry.Create(AssociationStatus.Deleted,
deleteQueueItem: deleteQueueItem));
transaction.Commit();
}
catch (MessageQueueReadException)
{
// We timed out trying to de-queue (no items on the queue).
// This exception doesn't need to be logged.
transaction.Abort();
}
catch (OperationCanceledException)
{
// Throw operation canceled exceptions up to the worker thread. It will handle
// logging correctly.
transaction.Abort();
throw;
}
#pragma warning disable CA1031 // Do not catch general exception types
catch (Exception e)
#pragma warning restore CA1031 // Do not catch general exception types
{
LogError(LogEntry.Create(AssociationStatus.DeleteError,
deleteQueueItem: deleteQueueItem),
e);
HandleExceptionForTransaction(deleteQueueItem, transaction);
}
}
}
/// <summary>
/// Create a new <see cref="IInnerEyeSegmentationClient"/> based on settings in JSON file.
/// </summary>
/// <param name="logger">Optional logger for client.</param>
/// <returns>New <see cref="IInnerEyeSegmentationClient"/>.</returns>
public Func <IInnerEyeSegmentationClient> CreateInnerEyeSegmentationClient(ILogger logger = null) =>
() =>
{
var processorSettings = ProcessorSettings();
var licenseKey = processorSettings.LicenseKey;
if (string.IsNullOrEmpty(licenseKey))
{
var message = string.Format(CultureInfo.InvariantCulture, "License key for the service `{0}` has not been set correctly in environment variable `{1}`. It needs to be a system variable.",
processorSettings.InferenceUri, processorSettings.LicenseKeyEnvVar);
var logEntry = LogEntry.Create(ServiceStatus.Starting);
logEntry.Log(logger, LogLevel.Error, new ConfigurationException(message));
}
return(new InnerEyeSegmentationClient(processorSettings.InferenceUri, licenseKey));
};
/// <summary>
/// Load T from a JSON file.
/// </summary>
/// <param name="path">Path to file.</param>
/// <returns>Pair of T and true if file loaded correctly, false otherwise.</returns>
private (T, bool) LoadFile(string path)
{
try
{
var jsonText = File.ReadAllText(path);
return(JsonConvert.DeserializeObject <T>(jsonText), true);
}
catch (Exception e)
{
var logEntry = LogEntry.Create(ServiceStatus.NewConfigurationError,
string.Format("Unable to load settings file {0}", path));
logEntry.Log(_logger, LogLevel.Error, e);
return(default(T), false);
}
}
public TestLogService()
{
_logEntries = new List <LogEntry>();
using (StreamReader reader = new StreamReader("..\\..\\sampleMessages.txt"))
{
int index = 0;
while (!reader.EndOfStream)
{
var line = reader.ReadLine();
var logEntry = LogEntry.Create(false, index++, DateTime.Now);
logEntry.Message = line;
_logEntries.Add(logEntry);
}
}
}
public void RecordLog(PacketObject rPacket, bool rStack, int rFrame)
{
lock (logQueue)
{
StackTrace st = null;
if (rStack)
{
st = new StackTrace(false);
}
if (m_useThread)
{
logQueue.Enqueue(LogEntry.Create(rPacket, rFrame, st));
}
else
{
gameLogger.Log(rPacket, rFrame, st);
}
}
}
/// <summary>
/// File watcher Changed event handler. Filter the events, reload the config and if successful invoke ConfigChanged.
/// </summary>
/// <param name="sender">Sender.</param>
/// <param name="e">File system event args.</param>
private void OnChanged(object sender, FileSystemEventArgs e)
{
if (e.ChangeType != WatcherChangeTypes.Changed)
{
return;
}
var logEntry = LogEntry.Create(ServiceStatus.NewConfigurationDetetected,
string.Format(CultureInfo.InvariantCulture, "Settings have changed: {0}", e.FullPath));
logEntry.Log(_logger, LogLevel.Information);
if (!Load())
{
return;
}
ConfigChanged?.Invoke(this, new EventArgs());
}
/// <summary>
/// Dequeues the next message from the queue.
/// Note: This method will not commit or abort the queue transaction.
/// </summary>
/// <param name="queueTransaction">The queue transaction.</param>
/// <param name="cancellationToken">The cancellation token.</param>
/// <returns>The de-queued object.</returns>
/// <exception cref="ArgumentException">If the queue path is not correct.</exception>
/// <exception cref="MessageQueueReadException">The queue timed out when reading.</exception>
/// <exception cref="MessageQueuePermissionsException">The queue does not have permissions to read.</exception>
protected async Task <T> DequeueNextMessageAsync(
IQueueTransaction queueTransaction,
CancellationToken cancellationToken)
{
// Check if we should move all the dead letter messages on every dequeue.
CheckMoveDeadLetterMessages();
try
{
// Note: We do not put the message queue in a using block. The service manages the lifecycle
// of message queues.
var messageItem = GetMessageQueue(_dequeueQueuePath)
.DequeueNextMessage <T>(queueTransaction);
// Increase the count on the message.
// Note: We should not check if the maximum dequeue count has been reached here. We should do that on handling exceptions.
messageItem.DequeueCount++;
LogInformation(LogEntry.Create(MessageQueueStatus.Dequeued,
queueItemBase: messageItem,
sourceMessageQueuePath: _dequeueQueuePath));
return(messageItem);
}
catch (MessageQueueReadException)
{
// Delay here before throw the exception up the next level to delay message queue reads.
await Task.Delay(DequeueServiceConfig.DequeueTimeout, cancellationToken);
throw;
}
catch (MessageQueuePermissionsException e)
{
LogError(LogEntry.Create(MessageQueueStatus.DequeueError, sourceMessageQueuePath: _dequeueQueuePath),
e);
// We cannot recover from access violations. We should stop the service.
StopServiceAsync();
throw;
}
}
/// <summary>
/// Enqueues the message using its own message queue transaction.
/// </summary>
/// <typeparam name="MessageType">The enqueue message type.</typeparam>
/// <param name="message">The message to enqueue.</param>
/// <param name="messageQueuePath">The queue path to enqueue the message onto.</param>
protected void EnqueueMessage <TMessageType>(TMessageType message, string messageQueuePath) where TMessageType : QueueItemBase
{
using (var transaction = CreateQueueTransaction(messageQueuePath))
{
BeginMessageQueueTransaction(transaction);
try
{
EnqueueMessage(message, messageQueuePath, transaction);
transaction.Commit();
}
catch (Exception e)
{
LogError(LogEntry.Create(AssociationStatus.BaseEnqueueMessageError,
queueItemBase: message),
e);
transaction.Abort();
}
}
}
/// <summary>
/// Pings the segmentation client and stops the service on authentication exceptions.
/// </summary>
/// <param name="stopServiceOnAuthFailures">Will stop the service on any authentication failures.</param>
/// <exception cref="AuthenticationException">If the license key is incorrect for the segmentation client.</exception>
private async Task PingAsync(bool stopServiceOnAuthFailures = true)
{
try
{
if (_innerEyeSegmentationClient != null)
{
await _innerEyeSegmentationClient.PingAsync().ConfigureAwait(false);
}
}
catch (AuthenticationException e)
{
LogError(LogEntry.Create(ServiceStatus.PingError), e);
if (stopServiceOnAuthFailures)
{
StopServiceAsync();
}
throw;
}
}
/// <summary>
/// Gets the accepted SOP classes and transfer syntaxes.
/// </summary>
/// <returns>The accepted SOP classes and transfer syntaxes.</returns>
private IReadOnlyDictionary <DicomUID, DicomTransferSyntax[]> GetAcceptedSopClassesAndTransferSyntaxes()
{
try
{
var gatewayReceiveConfig = _getReceiveServiceConfig();
if (gatewayReceiveConfig != null)
{
_receiveServiceConfig = gatewayReceiveConfig;
}
}
// We catch all exceptions and return the latest cached result on error. We do not want to stop accepting data
// just because we cannot communicate with our API. We should stop processing further down the chain.
catch (Exception e)
{
LogError(LogEntry.Create(ServiceStatus.GetAcceptedSopClassesAndTransferSyntaxesError),
e);
}
return(_receiveServiceConfig.AcceptedSopClassesAndTransferSyntaxes);
}
