public void SendMessages()
{
_logger.Debug("Sending messages in buffer");
lock (_oBufferLock)
{
var newBatchHeaders = new List <IMessageHeader>();
foreach ((DicomFileMessage message, IMessageHeader header) in _messageBuffer)
{
newBatchHeaders.Add(_producerModel.SendMessage(message, header, _reprocessingRoutingKey));
}
// Confirm all messages in the batch
_producerModel.WaitForConfirms();
TotalProcessed += _messageBuffer.Count;
foreach (IMessageHeader newHeader in newBatchHeaders)
{
newHeader.Log(_logger, LogLevel.Trace, "Sent");
}
_messageBuffer.Clear();
}
}
public void TestMultipleConfirmsOk()
{
IProducerModel pm = _testAdapter.SetupProducer(_testProducerOptions, true);
pm.SendMessage(new TestMessage(), null);
for (var i = 0; i < 10; ++i)
{
pm.WaitForConfirms();
}
}
public override void Stop(string reason)
{
if (_fileMessageProducer != null)
{
try
{
_fileMessageProducer.WaitForConfirms();
}
catch (AlreadyClosedException) { /* Ignored */ }
}
base.Stop(reason);
}
public override void Stop(string reason)
{
if (_consumerId != Guid.Empty)
{
RabbitMqAdapter.StopConsumer(_consumerId, Smi.Common.RabbitMqAdapter.DefaultOperationTimeout);
}
try
{
// Wait for any unconfirmed messages before calling stop
_producerModel.WaitForConfirms();
}
catch (AlreadyClosedException)
{
}
_swapper?.LogProgress(Logger, LogLevel.Info);
base.Stop(reason);
}
public IdentifierMapperQueueConsumer(IProducerModel producer, ISwapIdentifiers swapper)
{
_producer = producer;
_swapper = swapper;
acker = new Thread(() =>
{
try
{
while (true)
{
List <Tuple <IMessageHeader, ulong> > done = new List <Tuple <IMessageHeader, ulong> >();
Tuple <DicomFileMessage, IMessageHeader, ulong> t;
t = msgq.Take();
lock (_producer)
{
_producer.SendMessage(t.Item1, t.Item2, "");
done.Add(new Tuple <IMessageHeader, ulong>(t.Item2, t.Item3));
while (msgq.TryTake(out t))
{
_producer.SendMessage(t.Item1, t.Item2, "");
done.Add(new Tuple <IMessageHeader, ulong>(t.Item2, t.Item3));
}
_producer.WaitForConfirms();
foreach (var ack in done)
{
Ack(ack.Item1, ack.Item2);
}
}
}
}
catch (InvalidOperationException)
{
// The BlockingCollection will throw this exception when closed by Shutdown()
return;
}
});
acker.IsBackground = true;
acker.Start();
}
/// <summary>
/// Process files from the directory referenced in the message
/// </summary>
/// <param name="header"></param>
/// <param name="message"></param>
public void ReadTags(IMessageHeader header, AccessionDirectoryMessage message)
{
_stopwatch.Restart();
string dirPath = message.GetAbsolutePath(_filesystemRoot);
Logger.Debug("TagReader: About to process files in " + dirPath);
if (!_fs.Directory.Exists(dirPath))
{
throw new ApplicationException("Directory not found: " + dirPath);
}
if (!dirPath.StartsWith(_filesystemRoot, StringComparison.CurrentCultureIgnoreCase))
{
throw new ApplicationException("Directory " + dirPath + " is not below the given FileSystemRoot (" +
_filesystemRoot + ")");
}
long beginEnumerate = _stopwatch.ElapsedTicks;
string[] dicomFilePaths = _fs.Directory.EnumerateFiles(dirPath, _searchPattern).Where(Include).ToArray();
string[] zipFilePaths = _fs.Directory.EnumerateFiles(dirPath).Where(ZipHelper.IsZip).Where(Include).ToArray();
_swTotals[0] += _stopwatch.ElapsedTicks - beginEnumerate;
Logger.Debug("TagReader: Found " + dicomFilePaths.Length + " dicom files to process");
Logger.Debug("TagReader: Found " + zipFilePaths.Length + " zip files to process");
int toProcess = dicomFilePaths.Length + zipFilePaths.Length;
if (toProcess == 0)
{
throw new ApplicationException("No dicom/zip files found in " + dirPath);
}
// We have files to process, let's do it!
long beginRead = _stopwatch.ElapsedTicks;
List <DicomFileMessage> fileMessages = ReadTagsImpl(dicomFilePaths.Select(p => new FileInfo(p)), message);
fileMessages.AddRange(ReadZipFilesImpl(zipFilePaths.Select(p => new FileInfo(p)), message));
_swTotals[1] += (_stopwatch.ElapsedTicks - beginRead) / toProcess;
var seriesMessages = new Dictionary <string, SeriesMessage>();
foreach (DicomFileMessage fileMessage in fileMessages)
{
string seriesUID = fileMessage.SeriesInstanceUID;
// If we've already seen this seriesUID, just update the image count
if (seriesMessages.ContainsKey(seriesUID))
{
seriesMessages[seriesUID].ImagesInSeries++;
continue;
}
// Else create a new SeriesMessage
var seriesMessage = new SeriesMessage
{
NationalPACSAccessionNumber = fileMessage.NationalPACSAccessionNumber,
DirectoryPath = message.DirectoryPath,
StudyInstanceUID = fileMessage.StudyInstanceUID,
SeriesInstanceUID = seriesUID,
ImagesInSeries = 1,
DicomDataset = fileMessage.DicomDataset
};
seriesMessages.Add(seriesUID, seriesMessage);
}
Logger.Debug("TagReader: Finished processing directory, sending messages");
// Only send if have processed all files in the directory ok
if (!fileMessages.Any())
{
throw new ApplicationException("No DicomFileMessage(s) to send after processing the directory");
}
if (!seriesMessages.Any())
{
throw new ApplicationException("No SeriesMessage(s) to send but we have file messages");
}
Logger.Info($"Sending {fileMessages.Count} DicomFileMessage(s)");
long beginSend = _stopwatch.ElapsedTicks;
var headers = new List <IMessageHeader>();
foreach (DicomFileMessage fileMessage in fileMessages)
{
headers.Add(_fileMessageProducerModel.SendMessage(fileMessage, header));
}
_fileMessageProducerModel.WaitForConfirms();
headers.ForEach(x => x.Log(Logger, LogLevel.Trace, $"Sent {header?.MessageGuid}"));
Logger.Info($"Sending {seriesMessages.Count} SeriesMessage(s)");
headers.Clear();
foreach (KeyValuePair <string, SeriesMessage> kvp in seriesMessages)
{
headers.Add(_seriesMessageProducerModel.SendMessage(kvp.Value, header));
}
_seriesMessageProducerModel.WaitForConfirms();
headers.ForEach(x => x.Log(Logger, LogLevel.Trace, $"Sent {x.MessageGuid}"));
_swTotals[2] += _stopwatch.ElapsedTicks - beginSend;
_swTotals[3] += _stopwatch.ElapsedTicks;
_nMessagesSent += fileMessages.Count + seriesMessages.Count;
if (++_nAccMessagesProcessed % 10 == 0)
{
LogRates();
}
}
protected override void ProcessMessageImpl(IMessageHeader header, ExtractionRequestMessage request, ulong tag)
{
Logger.Info($"Received message: {request}");
_auditor.AuditExtractionRequest(request);
if (!request.ExtractionDirectory.StartsWith(request.ProjectNumber))
{
Logger.Debug("ExtractionDirectory did not start with the project number, doing ErrorAndNack");
ErrorAndNack(header, tag, "", new InvalidEnumArgumentException("ExtractionDirectory"));
}
string extractionDirectory = request.ExtractionDirectory.TrimEnd('/', '\\');
string extractFileRoutingKey = request.IsIdentifiableExtraction ? _options.ExtractIdentRoutingKey : _options.ExtractAnonRoutingKey;
foreach (ExtractImageCollection matchedFiles in _fulfiller.GetAllMatchingFiles(request, _auditor))
{
Logger.Info($"Accepted {matchedFiles.Accepted.Count} and rejected {matchedFiles.Rejected.Count} files for KeyValue {matchedFiles.KeyValue}");
var infoMessage = new ExtractFileCollectionInfoMessage(request);
foreach (QueryToExecuteResult accepted in matchedFiles.Accepted)
{
var extractFileMessage = new ExtractFileMessage(request)
{
// Path to the original file
DicomFilePath = accepted.FilePathValue.TrimStart('/', '\\'),
// Extraction directory relative to the extract root
ExtractionDirectory = extractionDirectory,
// Output path for the anonymised file, relative to the extraction directory
OutputPath = _resolver.GetOutputPath(accepted, request).Replace('\\', '/')
};
Logger.Debug($"DicomFilePath={extractFileMessage.DicomFilePath}, OutputPath={extractFileMessage.OutputPath}");
// Send the extract file message
var sentHeader = (MessageHeader)_fileMessageProducer.SendMessage(extractFileMessage, header, extractFileRoutingKey);
// Record that we sent it
infoMessage.ExtractFileMessagesDispatched.Add(sentHeader, extractFileMessage.OutputPath);
}
// Wait for confirms from the batched messages
Logger.Debug($"All file messages sent for {request.ExtractionJobIdentifier}, calling WaitForConfirms");
_fileMessageProducer.WaitForConfirms();
// For all the rejected messages log why (in the info message)
foreach (QueryToExecuteResult rejectedResults in matchedFiles.Rejected)
{
if (!infoMessage.RejectionReasons.ContainsKey(rejectedResults.RejectReason))
{
infoMessage.RejectionReasons.Add(rejectedResults.RejectReason, 0);
}
infoMessage.RejectionReasons[rejectedResults.RejectReason]++;
}
_auditor.AuditExtractFiles(request, matchedFiles);
infoMessage.KeyValue = matchedFiles.KeyValue;
_fileMessageInfoProducer.SendMessage(infoMessage, header);
if (_fileMessageInfoProducer.GetType() == typeof(BatchProducerModel))
{
_fileMessageInfoProducer.WaitForConfirms();
}
Logger.Info($"All messages sent and acknowledged for {matchedFiles.KeyValue}");
}
Logger.Info("Finished processing message");
Ack(header, tag);
}
