public void SendMessages()
{
_logger.Debug("Sending messages in buffer");
lock (_oBufferLock)
{
var newBatchHeaders = new List <IMessageHeader>();
foreach ((DicomFileMessage message, IMessageHeader header) in _messageBuffer)
{
newBatchHeaders.Add(_producerModel.SendMessage(message, header, _reprocessingRoutingKey));
}
// Confirm all messages in the batch
_producerModel.WaitForConfirms();
TotalProcessed += _messageBuffer.Count;
foreach (IMessageHeader newHeader in newBatchHeaders)
{
newHeader.Log(_logger, LogLevel.Trace, "Sent");
}
_messageBuffer.Clear();
}
}
public void ProcessMessage(
[NotNull] ExtractFileMessage message,
[NotNull] IMessageHeader header)
{
string fullSrc = _fileSystem.Path.Combine(_fileSystemRoot, message.DicomFilePath);
ExtractedFileStatusMessage statusMessage;
if (!_fileSystem.File.Exists(fullSrc))
{
statusMessage = new ExtractedFileStatusMessage(message)
{
DicomFilePath = message.DicomFilePath,
Status = ExtractedFileStatus.FileMissing,
StatusMessage = $"Could not find '{fullSrc}'"
};
_ = _copyStatusProducerModel.SendMessage(statusMessage, header, _options.NoVerifyRoutingKey);
return;
}
string fullDest = _fileSystem.Path.Combine(_extractionRoot, message.ExtractionDirectory, message.OutputPath);
if (_fileSystem.File.Exists(fullDest))
{
_logger.Warn($"Output file '{fullDest}' already exists. Will overwrite.");
}
IDirectoryInfo parent = _fileSystem.Directory.GetParent(fullDest);
if (!parent.Exists)
{
_logger.Debug($"Creating directory '{parent}'");
parent.Create();
}
_logger.Debug($"Copying source file to '{message.OutputPath}'");
_fileSystem.File.Copy(fullSrc, fullDest, overwrite: true);
statusMessage = new ExtractedFileStatusMessage(message)
{
DicomFilePath = message.DicomFilePath,
Status = ExtractedFileStatus.Copied,
OutputFilePath = message.OutputPath,
};
_ = _copyStatusProducerModel.SendMessage(statusMessage, header, _options.NoVerifyRoutingKey);
}
public override void Start()
{
foreach (var upd in _source.GetUpdates())
{
_producer.SendMessage(upd, null);
}
Stop("Update detection process finished");
}
public void TestMultipleConfirmsOk()
{
IProducerModel pm = _testAdapter.SetupProducer(_testProducerOptions, true);
pm.SendMessage(new TestMessage(), null);
for (var i = 0; i < 10; ++i)
{
pm.WaitForConfirms();
}
}
public IdentifierMapperQueueConsumer(IProducerModel producer, ISwapIdentifiers swapper)
{
_producer = producer;
_swapper = swapper;
acker = new Thread(() =>
{
try
{
while (true)
{
List <Tuple <IMessageHeader, ulong> > done = new List <Tuple <IMessageHeader, ulong> >();
Tuple <DicomFileMessage, IMessageHeader, ulong> t;
t = msgq.Take();
lock (_producer)
{
_producer.SendMessage(t.Item1, t.Item2, "");
done.Add(new Tuple <IMessageHeader, ulong>(t.Item2, t.Item3));
while (msgq.TryTake(out t))
{
_producer.SendMessage(t.Item1, t.Item2, "");
done.Add(new Tuple <IMessageHeader, ulong>(t.Item2, t.Item3));
}
_producer.WaitForConfirms();
foreach (var ack in done)
{
Ack(ack.Item1, ack.Item2);
}
}
}
}
catch (InvalidOperationException)
{
// The BlockingCollection will throw this exception when closed by Shutdown()
return;
}
});
acker.IsBackground = true;
acker.Start();
}
protected override void ProcessMessageImpl(IMessageHeader header, ExtractedFileStatusMessage message, ulong tag)
{
bool isClean = true;
object resultObject;
try
{
// We should only ever receive messages regarding anonymised images
if (message.Status != ExtractedFileStatus.Anonymised)
{
throw new ApplicationException($"Received a message with anonymised status of {message.Status}");
}
IFileInfo toProcess = _fileSystem.FileInfo.FromFileName(Path.Combine(_extractionRoot, message.ExtractionDirectory, message.OutputFilePath));
if (!toProcess.Exists)
{
throw new ApplicationException("IsIdentifiable service cannot find file " + toProcess.FullName);
}
var result = _classifier.Classify(toProcess);
foreach (Failure f in result)
{
Logger.Log(LogLevel.Info, $"Validation failed for {f.Resource} Problem Value:{f.ProblemValue}");
isClean = false;
}
resultObject = result;
}
catch (ApplicationException e)
{
// Catch specific exceptions we are aware of, any uncaught will bubble up to the wrapper
ErrorAndNack(header, tag, "Error while processing AnonSuccessMessage", e);
return;
}
_producer.SendMessage(new ExtractedFileVerificationMessage(message)
{
IsIdentifiable = !isClean,
Report = JsonConvert.SerializeObject(resultObject)
}, header);
Ack(header, tag);
}
/// <summary>
/// Handled when a new DICOM directory is found. Writes an AccessionDirectoryMessage to the message exchange
/// </summary>
/// <param name="dir">Full path to a directory that has been found to contain a DICOM file</param>
protected void FoundNewDicomDirectory(string dir)
{
Logger.Debug("DicomDirectoryFinder: Found " + dir);
string dirPath = Path.GetFullPath(dir).TrimEnd(Path.DirectorySeparatorChar);
if (dirPath.StartsWith(FileSystemRoot))
{
dirPath = dirPath.Remove(0, FileSystemRoot.Length);
}
dirPath = dirPath.TrimStart(Path.DirectorySeparatorChar);
var message = new AccessionDirectoryMessage
{
DirectoryPath = dirPath,
NationalPACSAccessionNumber = GetNationalPacsAccessionNumberFrom(dirPath)
};
_directoriesProducerModel.SendMessage(message, null);
++TotalSent;
}
/// <summary>
/// Fatal essentially just calls <see cref="Stop"/>, but attempts to send a FatalErrorMessage to RabbitMQ first
/// </summary>
/// <param name="msg"></param>
/// <param name="exception"></param>
public void Fatal(string msg, Exception exception)
{
if (_stopCalled)
{
return;
}
Logger.Fatal(exception, msg);
if (_fatalLoggingProducer != null)
{
try
{
_fatalLoggingProducer.SendMessage(new FatalErrorMessage(msg, exception), null);
}
catch (Exception e)
{
Logger.Error(e, "Failed to log fatal error");
}
}
Stop("Fatal error in MicroserviceHost (" + msg + ")");
}
/// <summary>
/// Process files from the directory referenced in the message
/// </summary>
/// <param name="header"></param>
/// <param name="message"></param>
public void ReadTags(IMessageHeader header, AccessionDirectoryMessage message)
{
_stopwatch.Restart();
string dirPath = message.GetAbsolutePath(_filesystemRoot);
Logger.Debug("TagReader: About to process files in " + dirPath);
if (!_fs.Directory.Exists(dirPath))
{
throw new ApplicationException("Directory not found: " + dirPath);
}
if (!dirPath.StartsWith(_filesystemRoot, StringComparison.CurrentCultureIgnoreCase))
{
throw new ApplicationException("Directory " + dirPath + " is not below the given FileSystemRoot (" +
_filesystemRoot + ")");
}
long beginEnumerate = _stopwatch.ElapsedTicks;
string[] dicomFilePaths = _fs.Directory.EnumerateFiles(dirPath, _searchPattern).Where(Include).ToArray();
string[] zipFilePaths = _fs.Directory.EnumerateFiles(dirPath).Where(ZipHelper.IsZip).Where(Include).ToArray();
_swTotals[0] += _stopwatch.ElapsedTicks - beginEnumerate;
Logger.Debug("TagReader: Found " + dicomFilePaths.Length + " dicom files to process");
Logger.Debug("TagReader: Found " + zipFilePaths.Length + " zip files to process");
int toProcess = dicomFilePaths.Length + zipFilePaths.Length;
if (toProcess == 0)
{
throw new ApplicationException("No dicom/zip files found in " + dirPath);
}
// We have files to process, let's do it!
long beginRead = _stopwatch.ElapsedTicks;
List <DicomFileMessage> fileMessages = ReadTagsImpl(dicomFilePaths.Select(p => new FileInfo(p)), message);
fileMessages.AddRange(ReadZipFilesImpl(zipFilePaths.Select(p => new FileInfo(p)), message));
_swTotals[1] += (_stopwatch.ElapsedTicks - beginRead) / toProcess;
var seriesMessages = new Dictionary <string, SeriesMessage>();
foreach (DicomFileMessage fileMessage in fileMessages)
{
string seriesUID = fileMessage.SeriesInstanceUID;
// If we've already seen this seriesUID, just update the image count
if (seriesMessages.ContainsKey(seriesUID))
{
seriesMessages[seriesUID].ImagesInSeries++;
continue;
}
// Else create a new SeriesMessage
var seriesMessage = new SeriesMessage
{
NationalPACSAccessionNumber = fileMessage.NationalPACSAccessionNumber,
DirectoryPath = message.DirectoryPath,
StudyInstanceUID = fileMessage.StudyInstanceUID,
SeriesInstanceUID = seriesUID,
ImagesInSeries = 1,
DicomDataset = fileMessage.DicomDataset
};
seriesMessages.Add(seriesUID, seriesMessage);
}
Logger.Debug("TagReader: Finished processing directory, sending messages");
// Only send if have processed all files in the directory ok
if (!fileMessages.Any())
{
throw new ApplicationException("No DicomFileMessage(s) to send after processing the directory");
}
if (!seriesMessages.Any())
{
throw new ApplicationException("No SeriesMessage(s) to send but we have file messages");
}
Logger.Info($"Sending {fileMessages.Count} DicomFileMessage(s)");
long beginSend = _stopwatch.ElapsedTicks;
var headers = new List <IMessageHeader>();
foreach (DicomFileMessage fileMessage in fileMessages)
{
headers.Add(_fileMessageProducerModel.SendMessage(fileMessage, header));
}
_fileMessageProducerModel.WaitForConfirms();
headers.ForEach(x => x.Log(Logger, LogLevel.Trace, $"Sent {header?.MessageGuid}"));
Logger.Info($"Sending {seriesMessages.Count} SeriesMessage(s)");
headers.Clear();
foreach (KeyValuePair <string, SeriesMessage> kvp in seriesMessages)
{
headers.Add(_seriesMessageProducerModel.SendMessage(kvp.Value, header));
}
_seriesMessageProducerModel.WaitForConfirms();
headers.ForEach(x => x.Log(Logger, LogLevel.Trace, $"Sent {x.MessageGuid}"));
_swTotals[2] += _stopwatch.ElapsedTicks - beginSend;
_swTotals[3] += _stopwatch.ElapsedTicks;
_nMessagesSent += fileMessages.Count + seriesMessages.Count;
if (++_nAccMessagesProcessed % 10 == 0)
{
LogRates();
}
}
protected override void ProcessMessageImpl(IMessageHeader header, ExtractionRequestMessage request, ulong tag)
{
Logger.Info($"Received message: {request}");
_auditor.AuditExtractionRequest(request);
if (!request.ExtractionDirectory.StartsWith(request.ProjectNumber))
{
Logger.Debug("ExtractionDirectory did not start with the project number, doing ErrorAndNack");
ErrorAndNack(header, tag, "", new InvalidEnumArgumentException("ExtractionDirectory"));
}
string extractionDirectory = request.ExtractionDirectory.TrimEnd('/', '\\');
string extractFileRoutingKey = request.IsIdentifiableExtraction ? _options.ExtractIdentRoutingKey : _options.ExtractAnonRoutingKey;
foreach (ExtractImageCollection matchedFiles in _fulfiller.GetAllMatchingFiles(request, _auditor))
{
Logger.Info($"Accepted {matchedFiles.Accepted.Count} and rejected {matchedFiles.Rejected.Count} files for KeyValue {matchedFiles.KeyValue}");
var infoMessage = new ExtractFileCollectionInfoMessage(request);
foreach (QueryToExecuteResult accepted in matchedFiles.Accepted)
{
var extractFileMessage = new ExtractFileMessage(request)
{
// Path to the original file
DicomFilePath = accepted.FilePathValue.TrimStart('/', '\\'),
// Extraction directory relative to the extract root
ExtractionDirectory = extractionDirectory,
// Output path for the anonymised file, relative to the extraction directory
OutputPath = _resolver.GetOutputPath(accepted, request).Replace('\\', '/')
};
Logger.Debug($"DicomFilePath={extractFileMessage.DicomFilePath}, OutputPath={extractFileMessage.OutputPath}");
// Send the extract file message
var sentHeader = (MessageHeader)_fileMessageProducer.SendMessage(extractFileMessage, header, extractFileRoutingKey);
// Record that we sent it
infoMessage.ExtractFileMessagesDispatched.Add(sentHeader, extractFileMessage.OutputPath);
}
// Wait for confirms from the batched messages
Logger.Debug($"All file messages sent for {request.ExtractionJobIdentifier}, calling WaitForConfirms");
_fileMessageProducer.WaitForConfirms();
// For all the rejected messages log why (in the info message)
foreach (QueryToExecuteResult rejectedResults in matchedFiles.Rejected)
{
if (!infoMessage.RejectionReasons.ContainsKey(rejectedResults.RejectReason))
{
infoMessage.RejectionReasons.Add(rejectedResults.RejectReason, 0);
}
infoMessage.RejectionReasons[rejectedResults.RejectReason]++;
}
_auditor.AuditExtractFiles(request, matchedFiles);
infoMessage.KeyValue = matchedFiles.KeyValue;
_fileMessageInfoProducer.SendMessage(infoMessage, header);
if (_fileMessageInfoProducer.GetType() == typeof(BatchProducerModel))
{
_fileMessageInfoProducer.WaitForConfirms();
}
Logger.Info($"All messages sent and acknowledged for {matchedFiles.KeyValue}");
}
Logger.Info("Finished processing message");
Ack(header, tag);
}
