public IsIdentifiableQueueConsumer(IProducerModel producer, string fileSystemRoot, string extractionRoot, IClassifier classifier)
{
_producer = producer;
_fileSystemRoot = fileSystemRoot;
_extractionRoot = extractionRoot;
_classifier = classifier;
}
public ExtractionFileCopier(
[NotNull] FileCopierOptions options,
[NotNull] IProducerModel copyStatusCopyStatusProducerModel,
[NotNull] string fileSystemRoot,
[NotNull] string extractionRoot,
[CanBeNull] IFileSystem fileSystem = null)
{
_options = options;
_copyStatusProducerModel = copyStatusCopyStatusProducerModel;
_fileSystemRoot = fileSystemRoot;
_extractionRoot = extractionRoot;
_fileSystem = fileSystem ?? new FileSystem();
if (!_fileSystem.Directory.Exists(_fileSystemRoot))
{
throw new ArgumentException($"Cannot find the specified fileSystemRoot: '{_fileSystemRoot}'");
}
if (!_fileSystem.Directory.Exists(_extractionRoot))
{
throw new ArgumentException($"Cannot find the specified extractionRoot: '{_extractionRoot}'");
}
_logger = LogManager.GetLogger(GetType().Name);
_logger.Info($"fileSystemRoot={_fileSystemRoot}, extractionRoot={_extractionRoot}");
}
/// <summary>
/// Starts up service and begins listening with the <see cref="Consumer"/>
/// </summary>
public override void Start()
{
FansiImplementations.Load();
IRDMPPlatformRepositoryServiceLocator repositoryLocator = Globals.RDMPOptions.GetRepositoryProvider();
var startup = new Startup(new EnvironmentInfo("netcoreapp2.2"), repositoryLocator);
var toMemory = new ToMemoryCheckNotifier();
startup.DoStartup(toMemory);
foreach (CheckEventArgs args in toMemory.Messages.Where(m => m.Result == CheckResult.Fail))
{
Logger.Log(LogLevel.Warn, args.Ex, args.Message);
}
_fileMessageProducer = RabbitMqAdapter.SetupProducer(Globals.CohortExtractorOptions.ExtractFilesProducerOptions, isBatch: true);
IProducerModel fileMessageInfoProducer = RabbitMqAdapter.SetupProducer(Globals.CohortExtractorOptions.ExtractFilesInfoProducerOptions, isBatch: false);
InitializeExtractionSources(repositoryLocator);
Consumer = new ExtractionRequestQueueConsumer(Globals.CohortExtractorOptions, _fulfiller, _auditor, _pathResolver, _fileMessageProducer, fileMessageInfoProducer);
RabbitMqAdapter.StartConsumer(_consumerOptions, Consumer, isSolo: false);
}
public IsIdentifiableHost(
[NotNull] GlobalOptions globals,
[NotNull] IsIdentifiableServiceOptions serviceOpts
)
: base(globals)
{
_consumerOptions = globals.IsIdentifiableOptions;
string classifierTypename = globals.IsIdentifiableOptions.ClassifierType;
string dataDirectory = globals.IsIdentifiableOptions.DataDirectory;
if (string.IsNullOrWhiteSpace(classifierTypename))
{
throw new ArgumentException("No IClassifier has been set in options. Enter a value for ClassifierType", nameof(globals));
}
if (string.IsNullOrWhiteSpace(dataDirectory))
{
throw new ArgumentException("A DataDirectory must be set", nameof(globals));
}
var objectFactory = new MicroserviceObjectFactory();
var classifier = objectFactory.CreateInstance <IClassifier>(classifierTypename, typeof(IClassifier).Assembly, new DirectoryInfo(dataDirectory), serviceOpts);
if (classifier == null)
{
throw new TypeLoadException($"Could not find IClassifier Type { classifierTypename }");
}
_producerModel = RabbitMqAdapter.SetupProducer(globals.IsIdentifiableOptions.IsIdentifiableProducerOptions, isBatch: false);
Consumer = new IsIdentifiableQueueConsumer(_producerModel, globals.FileSystemOptions.FileSystemRoot, globals.FileSystemOptions.ExtractRoot, classifier);
}
protected DicomDirectoryFinder(string fileSystemRoot, IFileSystem fileSystem, string dicomSearchPattern, IProducerModel directoriesProducerModel)
{
FileSystemRoot = fileSystemRoot;
FileSystem = fileSystem;
SearchPattern = dicomSearchPattern;
_directoriesProducerModel = directoriesProducerModel;
Logger = LogManager.GetLogger(GetType().Name);
}
public DicomFileProcessor(DicomReprocessorOptions options, IProducerModel producerModel, string reprocessingRoutingKey)
{
_logger = LogManager.GetLogger(GetType().Name);
_options = options;
_producerModel = producerModel;
_reprocessingRoutingKey = reprocessingRoutingKey;
}
public void TestMultipleConfirmsOk()
{
IProducerModel pm = _testAdapter.SetupProducer(_testProducerOptions, true);
pm.SendMessage(new TestMessage(), null);
for (var i = 0; i < 10; ++i)
{
pm.WaitForConfirms();
}
}
public ParallelTagReader(DicomTagReaderOptions options, FileSystemOptions fileSystemOptions,
IProducerModel seriesMessageProducerModel, IProducerModel fileMessageProducerModel, IFileSystem fs)
: base(options, fileSystemOptions, seriesMessageProducerModel, fileMessageProducerModel, fs)
{
_parallelOptions = new ParallelOptions
{
MaxDegreeOfParallelism = options.MaxIoThreads
};
Logger.Info($"Using MaxDegreeOfParallelism={_parallelOptions.MaxDegreeOfParallelism} for parallel IO operations");
}
public ExtractionRequestQueueConsumer(
CohortExtractorOptions options,
IExtractionRequestFulfiller fulfiller, IAuditExtractions auditor,
IProjectPathResolver pathResolver, IProducerModel fileMessageProducer,
IProducerModel fileMessageInfoProducer)
{
_options = options;
_fulfiller = fulfiller;
_auditor = auditor;
_resolver = pathResolver;
_fileMessageProducer = fileMessageProducer;
_fileMessageInfoProducer = fileMessageInfoProducer;
}
public TagPromotionProcessor(DicomReprocessorOptions options, IProducerModel producerModel, string reprocessingRoutingKey)
{
_logger = LogManager.GetLogger(GetType().Name);
_options = options;
var asBatchProducer = producerModel as BatchProducerModel;
if (asBatchProducer == null)
{
throw new ArgumentException("producerModel must be a batch producer");
}
_producerModel = asBatchProducer;
_reprocessingRoutingKey = reprocessingRoutingKey;
}
public IdentifierMapperHost(GlobalOptions options, ISwapIdentifiers swapper = null)
: base(options)
{
_consumerOptions = options.IdentifierMapperOptions;
FansiImplementations.Load();
if (swapper == null)
{
Logger.Info("Not passed a swapper, creating one of type " + options.IdentifierMapperOptions.SwapperType);
_swapper = ObjectFactory.CreateInstance <ISwapIdentifiers>(options.IdentifierMapperOptions.SwapperType, typeof(ISwapIdentifiers).Assembly);
}
else
{
_swapper = swapper;
}
// If we want to use a Redis server to cache answers then wrap the mapper in a Redis caching swapper
if (!string.IsNullOrWhiteSpace(options.IdentifierMapperOptions.RedisConnectionString))
{
try
{
_swapper = new RedisSwapper(options.IdentifierMapperOptions.RedisConnectionString, _swapper);
}
catch (RedisConnectionException e)
{
// NOTE(rkm 2020-03-30) Log & throw! I hate this, but if we don't log here using NLog, then the exception will bubble-up
// and only be printed to STDERR instead of to the log file and may be lost
Logger.Error(e, "Could not connect to Redis");
throw;
}
}
_swapper.Setup(_consumerOptions);
Logger.Info($"Swapper of type {_swapper.GetType()} created");
// Batching now handled implicitly as backlog demands
_producerModel = RabbitMqAdapter.SetupProducer(options.IdentifierMapperOptions.AnonImagesProducerOptions, isBatch: true);
Consumer = new IdentifierMapperQueueConsumer(_producerModel, _swapper)
{
AllowRegexMatching = options.IdentifierMapperOptions.AllowRegexMatching
};
// Add our event handler for control messages
AddControlHandler(new IdentifierMapperControlMessageHandler(_swapper));
}
/// <summary>
/// Interrogates directory tree for dicom files and produces series info and individual file info
/// </summary>
/// <param name="options"></param>
/// <param name="fileSystemOptions"></param>
/// <param name="seriesMessageProducerModel"></param>
/// <param name="fileMessageProducerModel"></param>
/// <param name="fs">File system to use</param>
public TagReaderBase(DicomTagReaderOptions options, FileSystemOptions fileSystemOptions, IProducerModel seriesMessageProducerModel, IProducerModel fileMessageProducerModel, IFileSystem fs)
{
Logger = LogManager.GetLogger(GetType().Name);
_filesystemRoot = fileSystemOptions.FileSystemRoot;
NackIfAnyFileErrors = options.NackIfAnyFileErrors;
_searchPattern = fileSystemOptions.DicomSearchPattern;
_fileReadOption = options.GetReadOption();
Logger.Debug($"FileReadOption is: {_fileReadOption}");
_seriesMessageProducerModel = seriesMessageProducerModel;
_fileMessageProducerModel = fileMessageProducerModel;
_fs = fs;
Logger.Info($"Stopwatch implementation - IsHighResolution: {Stopwatch.IsHighResolution}. Frequency: {Stopwatch.Frequency} ticks/s");
}
public DicomReprocessorHost(GlobalOptions options, DicomReprocessorCliOptions cliOptions)
: base(options)
{
string key = cliOptions.ReprocessingRoutingKey;
if (string.IsNullOrWhiteSpace(key))
{
throw new ArgumentException("ReprocessingRoutingKey");
}
// Set the initial sleep time
Globals.DicomReprocessorOptions.SleepTime = TimeSpan.FromMilliseconds(cliOptions.SleepTimeMs);
IProducerModel reprocessingProducerModel = RabbitMqAdapter.SetupProducer(options.DicomReprocessorOptions.ReprocessingProducerOptions, true);
Logger.Info("Documents will be reprocessed to " +
options.DicomReprocessorOptions.ReprocessingProducerOptions.ExchangeName + " on vhost " +
options.RabbitOptions.RabbitMqVirtualHost + " with routing key \"" + key + "\"");
if (!string.IsNullOrWhiteSpace(cliOptions.QueryFile))
{
_queryString = File.ReadAllText(cliOptions.QueryFile);
}
//TODO Make this into a CreateInstance<> call
switch (options.DicomReprocessorOptions.ProcessingMode)
{
case ProcessingMode.TagPromotion:
_processor = new TagPromotionProcessor(options.DicomReprocessorOptions, reprocessingProducerModel, key);
break;
case ProcessingMode.ImageReprocessing:
_processor = new DicomFileProcessor(options.DicomReprocessorOptions, reprocessingProducerModel, key);
break;
default:
throw new ArgumentException("ProcessingMode " + options.DicomReprocessorOptions.ProcessingMode + " not supported");
}
_mongoReader = new MongoDbReader(options.MongoDatabases.DicomStoreOptions, cliOptions, HostProcessName + "-" + HostProcessID);
AddControlHandler(new DicomReprocessorControlMessageHandler(Globals.DicomReprocessorOptions));
}
/// <summary>
/// Start this separately so we don't block the thread if the host constructor throws an exception
/// </summary>
public void StartAuxConnections()
{
lock (_oAdapterLock)
{
if (_auxConnectionsCreated)
{
return;
}
_auxConnectionsCreated = true;
// Ensures no consumers have been started until we explicitly call Start()
if (RabbitMqAdapter.HasConsumers)
{
throw new ApplicationException("Rabbit adapter has consumers before aux. connections created");
}
_fatalLoggingProducer = RabbitMqAdapter.SetupProducer(_fatalLoggingProducerOptions, isBatch: false);
RabbitMqAdapter.StartConsumer(_controlMessageConsumer.ControlConsumerOptions, _controlMessageConsumer, isSolo: false);
}
}
public IdentifierMapperQueueConsumer(IProducerModel producer, ISwapIdentifiers swapper)
{
_producer = producer;
_swapper = swapper;
acker = new Thread(() =>
{
try
{
while (true)
{
List <Tuple <IMessageHeader, ulong> > done = new List <Tuple <IMessageHeader, ulong> >();
Tuple <DicomFileMessage, IMessageHeader, ulong> t;
t = msgq.Take();
lock (_producer)
{
_producer.SendMessage(t.Item1, t.Item2, "");
done.Add(new Tuple <IMessageHeader, ulong>(t.Item2, t.Item3));
while (msgq.TryTake(out t))
{
_producer.SendMessage(t.Item1, t.Item2, "");
done.Add(new Tuple <IMessageHeader, ulong>(t.Item2, t.Item3));
}
_producer.WaitForConfirms();
foreach (var ack in done)
{
Ack(ack.Item1, ack.Item2);
}
}
}
}
catch (InvalidOperationException)
{
// The BlockingCollection will throw this exception when closed by Shutdown()
return;
}
});
acker.IsBackground = true;
acker.Start();
}
public FileCopierHost(
[NotNull] GlobalOptions options,
[CanBeNull] IFileSystem fileSystem = null
)
: base(
options
)
{
Logger.Debug("Creating FileCopierHost with FileSystemRoot: " + Globals.FileSystemOptions.FileSystemRoot);
IProducerModel copyStatusProducerModel = RabbitMqAdapter.SetupProducer(Globals.FileCopierOptions.CopyStatusProducerOptions, isBatch: false);
var fileCopier = new ExtractionFileCopier(
Globals.FileCopierOptions,
copyStatusProducerModel,
Globals.FileSystemOptions.FileSystemRoot,
Globals.FileSystemOptions.ExtractRoot,
fileSystem
);
_consumer = new FileCopyQueueConsumer(fileCopier);
}
public BasicDicomDirectoryFinder(string fileSystemRoot, string dicomSearchPattern, IProducerModel directoriesProducerModel)
: this(fileSystemRoot, new FileSystem(), dicomSearchPattern, directoriesProducerModel)
{
}
public BasicDicomDirectoryFinder(string fileSystemRoot, IFileSystem fileSystem, string dicomSearchPattern, IProducerModel directoriesProducerModel)
: base(fileSystemRoot, fileSystem, dicomSearchPattern, directoriesProducerModel)
{
}
public AccessionDirectoryLister(string fileSystemRoot, string dicomSearchPattern, IProducerModel directoriesProducerModel)
: this(fileSystemRoot, new FileSystem(), dicomSearchPattern, directoriesProducerModel)
{
}
public TriggerUpdatesHost(GlobalOptions options, ITriggerUpdatesSource source, IRabbitMqAdapter rabbitMqAdapter = null)
: base(options, rabbitMqAdapter)
{
this._source = source;
_producer = RabbitMqAdapter.SetupProducer(options.TriggerUpdatesOptions, isBatch: false);
}
public ZipDicomDirectoryFinder(string fileSystemRoot, IFileSystem fileSystem, string dicomSearchPattern, IProducerModel directoriesProducerModel)
: base(fileSystemRoot, fileSystem, dicomSearchPattern, directoriesProducerModel)
{
AlwaysSearchSubdirectories = true;
}
public AccessionDirectoryLister(string fileSystemRoot, IFileSystem fileSystem, string dicomSearchPattern, IProducerModel directoriesProducerModel)
: base(fileSystemRoot, fileSystem, dicomSearchPattern, directoriesProducerModel)
{
}
public SerialTagReader(DicomTagReaderOptions options, FileSystemOptions fileSystemOptions,
IProducerModel seriesMessageProducerModel, IProducerModel fileMessageProducerModel, IFileSystem fs)
: base(options, fileSystemOptions, seriesMessageProducerModel, fileMessageProducerModel, fs)
{
}
public PacsDirectoryFinder(string fileSystemRoot, string dicomSearchPattern, IProducerModel directoriesProducerModel)
: base(fileSystemRoot, new FileSystem(), dicomSearchPattern, directoriesProducerModel)
{
}
