private static int RecreateReport(GlobalOptions globalOptions, CohortPackagerCliOptions cliOptions)
{
Logger logger = LogManager.GetCurrentClassLogger();
logger.Info($"Recreating report for job {cliOptions.ExtractionId}");
MongoDbOptions mongoDbOptions = globalOptions.MongoDatabases.ExtractionStoreOptions;
MongoClient client = MongoClientHelpers.GetMongoClient(mongoDbOptions, SmiCliInit.HostProcessName);
var jobStore = new MongoExtractJobStore(client, mongoDbOptions.DatabaseName);
// NOTE(rkm 2020-10-22) Sets the extraction root to the current directory
IJobReporter reporter = JobReporterFactory.GetReporter(
"FileReporter",
jobStore,
new FileSystem(),
Directory.GetCurrentDirectory(),
cliOptions.ReportFormat.ToString(),
cliOptions.OutputNewLine ?? globalOptions.CohortPackagerOptions.ReportNewLine,
createJobIdFile: false
);
try
{
reporter.CreateReport(cliOptions.ExtractionId);
}
catch (Exception e)
{
logger.Error(e);
return(1);
}
return(0);
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="applicationName">Name to identify the connection to MongoDb with</param>
/// <param name="mongoDbOptions"></param>
/// <param name="defaultCollectionName">Default collectionNamePostfix to write to unless overridden</param>
public MongoDbAdapter(string applicationName, MongoDbOptions mongoDbOptions, string defaultCollectionName)
{
if (string.IsNullOrWhiteSpace(defaultCollectionName))
{
throw new ArgumentException("defaultCollectionName");
}
_logger.Debug("MongoDbAdapter: Creating connection to MongoDb on " + mongoDbOptions.HostName + ":" + mongoDbOptions.Port);
//TODO Standardise AppId
MongoClient mongoClient = MongoClientHelpers.GetMongoClient(mongoDbOptions, "MongoDbPopulator::" + applicationName, string.IsNullOrWhiteSpace(mongoDbOptions.UserName));
_logger.Debug("MongoDbAdapter: Getting reference to database " + mongoDbOptions.DatabaseName);
_database = mongoClient.GetDatabase(mongoDbOptions.DatabaseName);
_logger.Debug("MongoDbAdapter: Getting reference to collection " + defaultCollectionName);
_defaultCollectionName = defaultCollectionName;
_defaultCollection = _database.GetCollection <BsonDocument>(defaultCollectionName);
_logger.Debug("MongoDbAdapter: Checking initial collection");
bool isLive = _database.RunCommandAsync((Command <BsonDocument>) "{ping:1}").Wait(1000);
if (!isLive)
{
throw new ArgumentException($"Could not connect to the MongoDB server/database on startup at {mongoDbOptions.HostName}:{mongoDbOptions.Port}");
}
_logger.Debug("MongoDbAdapter: Connection setup successfully");
}
public void TestParseQuery(string jsonQuery, int?expectedSkip, int?expectedLimit)
{
MongoClient mongoClient = MongoClientHelpers.GetMongoClient(_mongoOptions, "MongoQueryParserTests");
IMongoDatabase database = mongoClient.GetDatabase("test");
IMongoCollection <BsonDocument> coll = database.GetCollection <BsonDocument>("test");
var findOptions = new FindOptions <BsonDocument> {
BatchSize = 1
};
Task <IAsyncCursor <BsonDocument> > t = MongoQueryParser.GetCursor(coll, findOptions, jsonQuery);
t.Wait(1_000);
Assert.IsTrue(t.IsCompleted);
Assert.IsFalse(t.IsFaulted);
using (IAsyncCursor <BsonDocument> _ = t.Result)
{
_logger.Info("Received new batch");
Assert.AreEqual(expectedSkip, findOptions.Skip);
Assert.AreEqual(expectedLimit, findOptions.Limit);
}
}
/// <summary>
/// Default constructor
/// </summary>
/// <param name="options">Options for connecting to MongoDB</param>
/// <param name="rabbitMqVirtualHost">RabbitMQ vhost where the messages are located. Used as part of the MongoDB collection names if provided</param>
public MongoDeadLetterStore(MongoDbOptions options, string rabbitMqVirtualHost = null)
{
_logger = LogManager.GetLogger(GetType().Name);
MongoClient client = MongoClientHelpers.GetMongoClient(options, "DeadLetterReprocessor");
_database = client.GetDatabase(options.DatabaseName);
try
{
Ping();
}
catch (ApplicationException e)
{
throw new ArgumentException(
"Could not connect to the MongoDB server/database on startup at: " + options.HostName + ":" +
options.Port, e);
}
var re = new Regex("^[A-Z_]+$", RegexOptions.IgnoreCase);
if (string.IsNullOrWhiteSpace(rabbitMqVirtualHost) || !re.IsMatch(rabbitMqVirtualHost))
{
_logger.Info("Not provided a valid string to label the collections (\"" + rabbitMqVirtualHost + "\"), using the default");
DeadLetterStoreCollectionName = DeadLetterStoreBaseCollectionName;
DeadLetterGraveyardCollectionName = DeadLetterGraveyardBaseCollectionName;
}
else
{
_logger.Info("Provided vhost \"" + rabbitMqVirtualHost + "\" for naming the storage collections");
DeadLetterStoreCollectionName = DeadLetterStoreBaseCollectionName + "-" + rabbitMqVirtualHost;
DeadLetterGraveyardCollectionName = DeadLetterGraveyardBaseCollectionName + "-" + rabbitMqVirtualHost;
}
_logger.Info("Connecting to dead letter store: " + options.DatabaseName + "." + DeadLetterStoreCollectionName);
_deadLetterStore = _database.GetCollection <MongoDeadLetterDocument>(DeadLetterStoreCollectionName);
long count = _deadLetterStore.CountDocuments(FilterDefinition <MongoDeadLetterDocument> .Empty);
_logger.Info("Connected to " +
(count > 0
? "dead letter store containing " + count + " existing messages"
: "empty dead letter store"));
_logger.Info("Connecting to dead letter graveyard: " + options.DatabaseName + "." + DeadLetterGraveyardCollectionName);
_deadLetterGraveyard = _database.GetCollection <MongoDeadLetterGraveyardDocument>(DeadLetterGraveyardCollectionName);
count = _deadLetterGraveyard.CountDocuments(FilterDefinition <MongoDeadLetterGraveyardDocument> .Empty);
_logger.Info("Connected to " +
(count > 0
? "dead letter graveyard containing " + count + " existing messages"
: "empty dead letter graveyard"));
}
public void SetupSuite()
{
Globals = GetNewMongoDbPopulatorOptions();
_mongoTestClient = MongoClientHelpers.GetMongoClient(Globals.MongoDatabases.DicomStoreOptions, "MongoDbPopulatorTests");
_mongoTestClient.DropDatabase(TestDbName);
TestDatabase = _mongoTestClient.GetDatabase(TestDbName);
Globals.MongoDbPopulatorOptions.SeriesQueueConsumerOptions = new ConsumerOptions()
{
QueueName = "TEST.SeriesQueue",
QoSPrefetchCount = 5,
AutoAck = false
};
Globals.MongoDbPopulatorOptions.ImageQueueConsumerOptions = new ConsumerOptions()
{
QueueName = "TEST.MongoImageQueue",
QoSPrefetchCount = 50,
AutoAck = false
};
var dataset = new DicomDataset
{
new DicomUniqueIdentifier(DicomTag.SOPInstanceUID, "1.2.3.4"),
new DicomCodeString(DicomTag.Modality, "SR")
};
string serialized = DicomTypeTranslater.SerializeDatasetToJson(dataset);
TestImageMessage = new DicomFileMessage
{
DicomFilePath = "Path/To/File",
NationalPACSAccessionNumber = "123",
SeriesInstanceUID = "TestSeriesInstanceUID",
StudyInstanceUID = "TestStudyInstanceUID",
SOPInstanceUID = "TestSOPInstanceUID",
DicomDataset = serialized
};
TestSeriesMessage = new SeriesMessage
{
DirectoryPath = "Path/To/Series",
ImagesInSeries = 123,
NationalPACSAccessionNumber = "123",
SeriesInstanceUID = "TestSeriesInstanceUID",
StudyInstanceUID = "TestStudyInstanceUID",
DicomDataset = serialized
};
}
public void OneTimeSetUp()
{
try
{
_testHelper.SetUpSuite();
}
catch (OperationInterruptedException)
{
// NOTE(rkm 2020-07-23) Temp fix for RabbitMQ Travis failures
Assert.Inconclusive();
}
MongoClient mongoClient = MongoClientHelpers
.GetMongoClient(_testHelper.GlobalOptions.MongoDatabases.DeadLetterStoreOptions, "DeadLetterReprocessorHostTests");
_database = mongoClient.GetDatabase(_testHelper.GlobalOptions.MongoDatabases.DeadLetterStoreOptions.DatabaseName);
_database.DropCollection(MongoDeadLetterStore.DeadLetterStoreBaseCollectionName);
_database.DropCollection(MongoDeadLetterStore.DeadLetterGraveyardBaseCollectionName);
_deadLetterCollection = _database.GetCollection <MongoDeadLetterDocument>(MongoDeadLetterStore.DeadLetterStoreBaseCollectionName);
_deadLetterGraveyard = _database.GetCollection <MongoDeadLetterGraveyardDocument>(MongoDeadLetterStore.DeadLetterGraveyardBaseCollectionName);
}
public MongoDbReader(MongoDbOptions mongoOptions, DicomReprocessorCliOptions reprocessorOptions, string appId)
{
_logger = LogManager.GetLogger(GetType().Name);
MongoClient mongoClient = MongoClientHelpers.GetMongoClient(mongoOptions, appId);
if (string.IsNullOrWhiteSpace(reprocessorOptions.SourceCollection))
{
throw new ArgumentException("SourceCollection");
}
_collection = mongoClient.GetDatabase(mongoOptions.DatabaseName).GetCollection <BsonDocument>(reprocessorOptions.SourceCollection);
_collNamespace = mongoOptions.DatabaseName + "." + reprocessorOptions.SourceCollection;
// if specified, batch size must be gt 1:
// https://docs.mongodb.com/manual/reference/method/cursor.batchSize/
if (reprocessorOptions.MongoDbBatchSize > 1)
{
_findOptionsBase.BatchSize = reprocessorOptions.MongoDbBatchSize;
}
_autoRun = reprocessorOptions.AutoRun;
}
private void VerifyReports(GlobalOptions globals, PathFixtures pf, ReportFormat reportFormat, IEnumerable <Tuple <ConsumerOptions, IMessage> > toSend)
{
globals.FileSystemOptions.ExtractRoot = pf.ExtractRootAbsolute;
globals.CohortPackagerOptions.JobWatcherTimeoutInSeconds = 5;
globals.CohortPackagerOptions.ReporterType = "FileReporter";
globals.CohortPackagerOptions.ReportFormat = reportFormat.ToString();
MongoClient client = MongoClientHelpers.GetMongoClient(globals.MongoDatabases.ExtractionStoreOptions, "test", true);
globals.MongoDatabases.ExtractionStoreOptions.DatabaseName += "-" + Guid.NewGuid().ToString().Split('-')[0];
client.DropDatabase(globals.MongoDatabases.ExtractionStoreOptions.DatabaseName);
using (var tester = new MicroserviceTester(
globals.RabbitOptions,
globals.CohortPackagerOptions.ExtractRequestInfoOptions,
globals.CohortPackagerOptions.FileCollectionInfoOptions,
globals.CohortPackagerOptions.NoVerifyStatusOptions,
globals.CohortPackagerOptions.VerificationStatusOptions))
{
foreach ((ConsumerOptions consumerOptions, IMessage message) in toSend)
{
tester.SendMessage(consumerOptions, new MessageHeader(), message);
}
var host = new CohortPackagerHost(globals);
host.Start();
var timeoutSecs = 10;
while (!HaveFiles(pf) && timeoutSecs > 0)
{
--timeoutSecs;
Thread.Sleep(TimeSpan.FromSeconds(1));
}
host.Stop("Test end");
}
var firstLine = $"# SMI extraction validation report for testProj1/{pf.ExtractName}";
switch (reportFormat)
{
case ReportFormat.Combined:
{
string reportContent = File.ReadAllText(Path.Combine(pf.ProjReportsDirAbsolute, $"{pf.ExtractName}_report.txt"));
Assert.True(reportContent.StartsWith(firstLine));
break;
}
case ReportFormat.Split:
{
string extractReportsDirAbsolute = Path.Combine(pf.ProjReportsDirAbsolute, pf.ExtractName);
Assert.AreEqual(6, Directory.GetFiles(extractReportsDirAbsolute).Length);
string reportContent = File.ReadAllText(Path.Combine(extractReportsDirAbsolute, "README.md"));
Assert.True(reportContent.StartsWith(firstLine));
break;
}
default:
Assert.Fail($"No case for ReportFormat {reportFormat}");
break;
}
}
/// <summary>
/// Default constructor for CohortPackagerHost
/// </summary>
/// <param name="globals"></param>
/// <param name="jobStore"></param>
/// <param name="fileSystem"></param>
/// <param name="reporter">
/// Pass to override the default IJobReporter that will be created from
/// Globals.CohortPackagerOptions.ReportFormat. That value should not be set if a reporter is passed.
/// </param>
/// <param name="notifier"></param>
/// <param name="rabbitMqAdapter"></param>
/// <param name="dateTimeProvider"></param>
public CohortPackagerHost(
[NotNull] GlobalOptions globals,
[CanBeNull] ExtractJobStore jobStore = null,
[CanBeNull] IFileSystem fileSystem = null,
[CanBeNull] IJobReporter reporter = null,
[CanBeNull] IJobCompleteNotifier notifier = null,
[CanBeNull] IRabbitMqAdapter rabbitMqAdapter = null,
[CanBeNull] DateTimeProvider dateTimeProvider = null
)
: base(globals, rabbitMqAdapter)
{
if (jobStore == null)
{
MongoDbOptions mongoDbOptions = Globals.MongoDatabases.ExtractionStoreOptions;
jobStore = new MongoExtractJobStore(
MongoClientHelpers.GetMongoClient(mongoDbOptions, HostProcessName),
mongoDbOptions.DatabaseName,
dateTimeProvider
);
}
else if (dateTimeProvider != null)
{
throw new ArgumentException("jobStore and dateTimeProvider are mutually exclusive arguments");
}
// If not passed a reporter or notifier, try and construct one from the given options
string reportFormatStr = Globals.CohortPackagerOptions.ReportFormat;
if (reporter == null)
{
reporter = JobReporterFactory.GetReporter(
Globals.CohortPackagerOptions.ReporterType,
jobStore,
fileSystem ?? new FileSystem(),
Globals.FileSystemOptions.ExtractRoot,
reportFormatStr,
Globals.CohortPackagerOptions.ReportNewLine
);
}
else
{
if (!string.IsNullOrWhiteSpace(reportFormatStr))
{
throw new ArgumentException($"Passed an IJobReporter, but this conflicts with the ReportFormat of '{reportFormatStr}' in the given options");
}
if (fileSystem != null)
{
throw new ArgumentException("Passed a fileSystem, but this will be unused as also passed an existing IJobReporter");
}
}
notifier ??= JobCompleteNotifierFactory.GetNotifier(
Globals.CohortPackagerOptions.NotifierType
);
_jobWatcher = new ExtractJobWatcher(
globals.CohortPackagerOptions,
jobStore,
ExceptionCallback,
notifier,
reporter
);
AddControlHandler(new CohortPackagerControlMessageHandler(_jobWatcher));
// Setup our consumers
_requestInfoMessageConsumer = new ExtractionRequestInfoMessageConsumer(jobStore);
_fileCollectionMessageConsumer = new ExtractFileCollectionMessageConsumer(jobStore);
_anonFailedMessageConsumer = new AnonFailedMessageConsumer(jobStore);
_anonVerificationMessageConsumer = new AnonVerificationMessageConsumer(jobStore);
}
