public void Test_FileCopierHost_HappyPath()
{
GlobalOptions globals = new GlobalOptionsFactory().Load();
globals.FileSystemOptions.FileSystemRoot = "root";
globals.FileSystemOptions.ExtractRoot = "exroot";
using var tester = new MicroserviceTester(globals.RabbitOptions, globals.FileCopierOptions);
string outputQueueName = globals.FileCopierOptions.CopyStatusProducerOptions.ExchangeName.Replace("Exchange", "Queue");
tester.CreateExchange(
globals.FileCopierOptions.CopyStatusProducerOptions.ExchangeName,
outputQueueName,
false,
globals.FileCopierOptions.NoVerifyRoutingKey);
var mockFileSystem = new MockFileSystem();
mockFileSystem.AddDirectory(globals.FileSystemOptions.FileSystemRoot);
mockFileSystem.AddDirectory(globals.FileSystemOptions.ExtractRoot);
mockFileSystem.AddFile(mockFileSystem.Path.Combine(globals.FileSystemOptions.FileSystemRoot, "file.dcm"), MockFileData.NullObject);
var host = new FileCopierHost(globals, mockFileSystem);
tester.StopOnDispose.Add(host);
host.Start();
var message = new ExtractFileMessage
{
ExtractionJobIdentifier = Guid.NewGuid(),
JobSubmittedAt = DateTime.UtcNow,
ProjectNumber = "1234",
ExtractionDirectory = "1234/foo",
DicomFilePath = "file.dcm",
IsIdentifiableExtraction = true,
OutputPath = "output.dcm",
};
tester.SendMessage(globals.FileCopierOptions, message);
using IConnection conn = tester.Factory.CreateConnection();
using IModel model = conn.CreateModel();
var consumer = new EventingBasicConsumer(model);
ExtractedFileStatusMessage statusMessage = null;
consumer.Received += (_, ea) => statusMessage = JsonConvert.DeserializeObject <ExtractedFileStatusMessage>(Encoding.UTF8.GetString(ea.Body.ToArray()));
model.BasicConsume(outputQueueName, true, "", consumer);
new TestTimelineAwaiter().Await(() => statusMessage != null);
Assert.AreEqual(ExtractedFileStatus.Copied, statusMessage.Status);
}
public void ProcessMessage(
[NotNull] ExtractFileMessage message,
[NotNull] IMessageHeader header)
{
string fullSrc = _fileSystem.Path.Combine(_fileSystemRoot, message.DicomFilePath);
ExtractedFileStatusMessage statusMessage;
if (!_fileSystem.File.Exists(fullSrc))
{
statusMessage = new ExtractedFileStatusMessage(message)
{
DicomFilePath = message.DicomFilePath,
Status = ExtractedFileStatus.FileMissing,
StatusMessage = $"Could not find '{fullSrc}'"
};
_ = _copyStatusProducerModel.SendMessage(statusMessage, header, _options.NoVerifyRoutingKey);
return;
}
string fullDest = _fileSystem.Path.Combine(_extractionRoot, message.ExtractionDirectory, message.OutputPath);
if (_fileSystem.File.Exists(fullDest))
{
_logger.Warn($"Output file '{fullDest}' already exists. Will overwrite.");
}
IDirectoryInfo parent = _fileSystem.Directory.GetParent(fullDest);
if (!parent.Exists)
{
_logger.Debug($"Creating directory '{parent}'");
parent.Create();
}
_logger.Debug($"Copying source file to '{message.OutputPath}'");
_fileSystem.File.Copy(fullSrc, fullDest, overwrite: true);
statusMessage = new ExtractedFileStatusMessage(message)
{
DicomFilePath = message.DicomFilePath,
Status = ExtractedFileStatus.Copied,
OutputFilePath = message.OutputPath,
};
_ = _copyStatusProducerModel.SendMessage(statusMessage, header, _options.NoVerifyRoutingKey);
}
private static BasicDeliverEventArgs GetMockDeliverArgs(ExtractFileMessage message)
{
var mockDeliverArgs = Mock.Of <BasicDeliverEventArgs>(MockBehavior.Strict);
mockDeliverArgs.DeliveryTag = 1;
mockDeliverArgs.Body = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(message));
mockDeliverArgs.BasicProperties = new BasicProperties {
Headers = new Dictionary <string, object>()
};
var header = new MessageHeader();
header.Populate(mockDeliverArgs.BasicProperties.Headers);
// Have to convert these to bytes since RabbitMQ normally does that when sending
mockDeliverArgs.BasicProperties.Headers["MessageGuid"] = Encoding.UTF8.GetBytes(header.MessageGuid.ToString());
mockDeliverArgs.BasicProperties.Headers["ProducerExecutableName"] = Encoding.UTF8.GetBytes(header.ProducerExecutableName);
mockDeliverArgs.BasicProperties.Headers["Parents"] = Encoding.UTF8.GetBytes(string.Join("->", header.Parents));
return(mockDeliverArgs);
}
public void SetUp()
{
_message = new ExtractFileMessage
{
JobSubmittedAt = DateTime.UtcNow,
ExtractionJobIdentifier = Guid.NewGuid(),
ProjectNumber = "1234",
ExtractionDirectory = "foo",
DicomFilePath = "foo.dcm",
IsIdentifiableExtraction = true,
OutputPath = "bar",
};
_mockModel = new Mock <IModel>(MockBehavior.Strict);
_mockModel.Setup(x => x.IsClosed).Returns(false);
_mockModel.Setup(x => x.BasicAck(It.IsAny <ulong>(), It.IsAny <bool>()));
_mockModel.Setup(x => x.BasicNack(It.IsAny <ulong>(), It.IsAny <bool>(), It.IsAny <bool>()));
_mockFileCopier = new Mock <IFileCopier>(MockBehavior.Strict);
_mockFileCopier.Setup(x => x.ProcessMessage(It.IsAny <ExtractFileMessage>(), It.IsAny <IMessageHeader>()));
}
protected override void ProcessMessageImpl(IMessageHeader header, ExtractionRequestMessage request, ulong tag)
{
Logger.Info($"Received message: {request}");
_auditor.AuditExtractionRequest(request);
if (!request.ExtractionDirectory.StartsWith(request.ProjectNumber))
{
Logger.Debug("ExtractionDirectory did not start with the project number, doing ErrorAndNack");
ErrorAndNack(header, tag, "", new InvalidEnumArgumentException("ExtractionDirectory"));
}
string extractionDirectory = request.ExtractionDirectory.TrimEnd('/', '\\');
string extractFileRoutingKey = request.IsIdentifiableExtraction ? _options.ExtractIdentRoutingKey : _options.ExtractAnonRoutingKey;
foreach (ExtractImageCollection matchedFiles in _fulfiller.GetAllMatchingFiles(request, _auditor))
{
Logger.Info($"Accepted {matchedFiles.Accepted.Count} and rejected {matchedFiles.Rejected.Count} files for KeyValue {matchedFiles.KeyValue}");
var infoMessage = new ExtractFileCollectionInfoMessage(request);
foreach (QueryToExecuteResult accepted in matchedFiles.Accepted)
{
var extractFileMessage = new ExtractFileMessage(request)
{
// Path to the original file
DicomFilePath = accepted.FilePathValue.TrimStart('/', '\\'),
// Extraction directory relative to the extract root
ExtractionDirectory = extractionDirectory,
// Output path for the anonymised file, relative to the extraction directory
OutputPath = _resolver.GetOutputPath(accepted, request).Replace('\\', '/')
};
Logger.Debug($"DicomFilePath={extractFileMessage.DicomFilePath}, OutputPath={extractFileMessage.OutputPath}");
// Send the extract file message
var sentHeader = (MessageHeader)_fileMessageProducer.SendMessage(extractFileMessage, header, extractFileRoutingKey);
// Record that we sent it
infoMessage.ExtractFileMessagesDispatched.Add(sentHeader, extractFileMessage.OutputPath);
}
// Wait for confirms from the batched messages
Logger.Debug($"All file messages sent for {request.ExtractionJobIdentifier}, calling WaitForConfirms");
_fileMessageProducer.WaitForConfirms();
// For all the rejected messages log why (in the info message)
foreach (QueryToExecuteResult rejectedResults in matchedFiles.Rejected)
{
if (!infoMessage.RejectionReasons.ContainsKey(rejectedResults.RejectReason))
{
infoMessage.RejectionReasons.Add(rejectedResults.RejectReason, 0);
}
infoMessage.RejectionReasons[rejectedResults.RejectReason]++;
}
_auditor.AuditExtractFiles(request, matchedFiles);
infoMessage.KeyValue = matchedFiles.KeyValue;
_fileMessageInfoProducer.SendMessage(infoMessage, header);
if (_fileMessageInfoProducer.GetType() == typeof(BatchProducerModel))
{
_fileMessageInfoProducer.WaitForConfirms();
}
Logger.Info($"All messages sent and acknowledged for {matchedFiles.KeyValue}");
}
Logger.Info("Finished processing message");
Ack(header, tag);
}
