protected override void PreInitializeImpl(IExtractCommand request, IDataLoadEventListener listener)
{
if (_request is ExtractGlobalsCommand)
{
listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Request is for the extraction of Globals."));
OutputFile = _request.GetExtractionDirectory().FullName;
return;
}
switch (FlatFileType)
{
case ExecuteExtractionToFlatFileType.CSV:
OutputFile = Path.Combine(DirectoryPopulated.FullName, GetFilename() + ".csv");
if (request.Configuration != null)
{
_output = new CSVOutputFormat(OutputFile, request.Configuration.Separator, DateFormat);
}
else
{
_output = new CSVOutputFormat(OutputFile, ",", DateFormat);
}
break;
default:
throw new ArgumentOutOfRangeException();
}
listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Setup data extraction destination as " + OutputFile + " (will not exist yet)"));
}
public DirectoryInfo GetDirectoryFor(IExtractCommand request)
{
var cmd = request as IExtractDatasetCommand;
if (string.IsNullOrWhiteSpace(ExtractionSubdirectoryPattern) || cmd == null)
{
return(request.GetExtractionDirectory());
}
var cata = cmd.SelectedDataSets.ExtractableDataSet.Catalogue;
if (ExtractionSubdirectoryPattern.Contains("$a") && string.IsNullOrWhiteSpace(cata.Acronym))
{
throw new Exception($"Catalogue {cata} does not have an Acronym and ExtractionSubdirectoryPattern contains $a");
}
var path = Path.Combine(cmd.Project.ExtractionDirectory,
ExtractionSubdirectoryPattern
.Replace("$c", QuerySyntaxHelper.MakeHeaderNameSensible(cmd.Configuration.Name))
.Replace("$i", cmd.Configuration.ID.ToString())
.Replace("$d", QuerySyntaxHelper.MakeHeaderNameSensible(cata.Name))
.Replace("$a", QuerySyntaxHelper.MakeHeaderNameSensible(cata.Acronym))
.Replace("$n", cata.ID.ToString())
);
var dir = new DirectoryInfo(path);
if (!dir.Exists)
{
dir.Create();
}
return(dir);
}
public RedisListenerAgentStartup(RedisClient redisClient, IOptions <RedisListenerOptions> redisListenerOptions, IExtractCommand extractCommand)
{
_redisClient = redisClient;
_redisListenerOptions = redisListenerOptions.Value;
_extractCommand = extractCommand;
RedisListenerKey = $"Bucket.Listener.{_redisListenerOptions.ListenerKey}";
}
public ZookeeperListenerAgentStartup(IOptions <ZookeeperListenerOptions> options, IExtractCommand extractCommand)
{
_extractCommand = extractCommand;
_options = options.Value;
_client = new ZookeeperClient(_options);
ListenerPath = $"/Bucket.Listener/{_options.ListenerKey}";
}
public void PreInitialize(IExtractCommand value, IDataLoadEventListener listener)
{
if (value is ExtractDatasetCommand)
{
Initialize(value as ExtractDatasetCommand);
}
if (value is ExtractGlobalsCommand)
{
Initialize(value as ExtractGlobalsCommand);
}
}
public RedisListenerAgentStartup(RedisClient redisClient, IOptionsMonitor <RedisListenerOptions> redisListenerOptions, IExtractCommand extractCommand)
{
_redisClient = redisClient;
_redisListenerOptions = redisListenerOptions.CurrentValue;
_extractCommand = extractCommand;
RedisListenerKey = $"Bucket.Listener.{_redisListenerOptions.ListenerKey}";
redisListenerOptions.OnChange(async(options) => {
_redisListenerOptions = options;
RedisListenerKey = $"Bucket.Listener.{_redisListenerOptions.ListenerKey}";
await SubscribeAsync();
});
}
public void PreInitialize(IExtractCommand value, IDataLoadEventListener listener)
{
_request = value;
DirectoryPopulated = _request.GetExtractionDirectory();
if (CleanExtractionFolderBeforeExtraction && value is ExtractDatasetCommand)
{
DirectoryPopulated.Delete(true);
DirectoryPopulated.Create();
}
}
protected override void SetUp()
{
base.SetUp();
_projectStub = Mock.Of <IProject>();
_projectStub.ProjectNumber = -123;
var cfg = Mock.Of <IExtractionConfiguration>();
_commandStub = Mock.Of <IExtractCommand>(cmd => cmd.Configuration == cfg);
Database = GetCleanedServer(DatabaseType.MicrosoftSQLServer);
}
public ExtractionPipelineUseCase(IProject project, IExtractCommand extractCommand, IPipeline pipeline, DataLoadInfo dataLoadInfo)
{
_dataLoadInfo = dataLoadInfo;
ExtractCommand = extractCommand;
_pipeline = pipeline;
extractCommand.ElevateState(ExtractCommandState.NotLaunched);
AddInitializationObject(ExtractCommand);
AddInitializationObject(project);
AddInitializationObject(_dataLoadInfo);
AddInitializationObject(project.DataExportRepository.CatalogueRepository);
GenerateContext();
}
public void CleanupOnStart()
{
_projectStub = Mock.Of <IProject>();
_projectStub.ProjectNumber = -123;
var cfg = Mock.Of <IExtractionConfiguration>();
_commandStub = Mock.Of <IExtractCommand>(cmd => cmd.Configuration == cfg);
var db = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase("FictionalDatabase");
if (db.Exists())
{
db.Drop();
}
}
public void PreInitialize(IExtractCommand request, IDataLoadEventListener listener)
{
_request = request;
if (_request == ExtractDatasetCommand.EmptyCommand)
{
listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Request is ExtractDatasetCommand.EmptyCommand, checking will not be carried out"));
return;
}
LinesWritten = 0;
DirectoryPopulated = request.GetExtractionDirectory();
PreInitializeImpl(request, listener);
}
public void PreInitialize(IExtractCommand value, IDataLoadEventListener listener)
{
_extractCommand = value as IExtractDatasetCommand;
}
protected override void PreInitializeImpl(IExtractCommand value, IDataLoadEventListener listener)
{
}
public bool Contains(IExtractCommand cmd)
{
return(CommandStates.ContainsKey(cmd));
}
public void PreInitialize(IExtractCommand value, IDataLoadEventListener listener)
{
_request = value;
}
protected abstract void PreInitializeImpl(IExtractCommand request, IDataLoadEventListener listener);
public void TestAnonymisingDataset(Type putterType, bool keepDates)
{
var uidMapDb = GetCleanedServer(DatabaseType.MicrosoftSQLServer, "TESTUIDMapp");
MasterDatabaseScriptExecutor e = new MasterDatabaseScriptExecutor(uidMapDb);
var patcher = new SMIDatabasePatcher();
e.CreateAndPatchDatabase(patcher, new AcceptAllCheckNotifier());
var eds = new ExternalDatabaseServer(CatalogueRepository, "eds", patcher);
eds.SetProperties(uidMapDb);
Dictionary <DicomTag, string> thingThatShouldDisappear = new Dictionary <DicomTag, string>
{
//Things we would want to disappear
{ DicomTag.PatientName, "Moscow" },
{ DicomTag.PatientBirthDate, "20010101" },
{ DicomTag.StudyDescription, "Frank has lots of problems, he lives at 60 Pancake road" },
{ DicomTag.SeriesDescription, "Coconuts" },
{ DicomTag.StudyDate, "20020101" },
};
Dictionary <DicomTag, string> thingsThatShouldRemain = new Dictionary <DicomTag, string>
{
//Things we would want to remain
//{DicomTag.SmokingStatus,"YES"},
};
var dicom = new DicomDataset
{
{ DicomTag.SOPInstanceUID, "123.4.4" },
{ DicomTag.SeriesInstanceUID, "123.4.5" },
{ DicomTag.StudyInstanceUID, "123.4.6" },
{ DicomTag.SOPClassUID, "1" },
};
foreach (var(key, value) in thingThatShouldDisappear)
{
dicom.AddOrUpdate(key, value);
}
foreach (var(key, value) in thingsThatShouldRemain)
{
dicom.AddOrUpdate(key, value);
}
dicom.AddOrUpdate(DicomTag.StudyDate, new DateTime(2002, 01, 01));
var fi = new FileInfo(Path.Combine(TestContext.CurrentContext.WorkDirectory, "madness.dcm"));
DicomFile df = new DicomFile(dicom);
df.Save(fi.FullName);
var dt = new DataTable();
dt.Columns.Add("Filepath");
dt.Columns.Add("SOPInstanceUID");
dt.Columns.Add("SeriesInstanceUID");
dt.Columns.Add("StudyInstanceUID");
dt.Columns.Add("Pat");
//note we don't have series
dt.Rows.Add(fi.Name, "123.4.4", "123.4.5", "123.4.6", "Hank");
var anonymiser = new FoDicomAnonymiser();
IExtractCommand cmd = MockExtractionCommand();
//give the mock to anonymiser
anonymiser.PreInitialize(cmd, new ThrowImmediatelyDataLoadEventListener());
anonymiser.PutterType = putterType;
anonymiser.ArchiveRootIfAny = TestContext.CurrentContext.WorkDirectory;
anonymiser.RelativeArchiveColumnName = "Filepath";
anonymiser.UIDMappingServer = eds;
anonymiser.RetainDates = keepDates;
var anoDt = anonymiser.ProcessPipelineData(dt, new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken());
Assert.AreEqual(1, anoDt.Rows.Count);
//Data table should contain new UIDs
Assert.AreNotEqual("123.4.4", anoDt.Rows[0]["SOPInstanceUID"]);
Assert.AreEqual(56, anoDt.Rows[0]["SOPInstanceUID"].ToString().Length);
Assert.AreNotEqual("123.4.6", anoDt.Rows[0]["StudyInstanceUID"]);
Assert.AreEqual(56, anoDt.Rows[0]["StudyInstanceUID"].ToString().Length);
FileInfo expectedFile = null;
if (putterType == typeof(PutInRoot))
{
expectedFile = new FileInfo(Path.Combine(TestContext.CurrentContext.WorkDirectory, "Images", anoDt.Rows[0]["SOPInstanceUID"] + ".dcm"));
}
if (putterType == typeof(PutInReleaseIdentifierSubfolders))
{
expectedFile = new FileInfo(Path.Combine(TestContext.CurrentContext.WorkDirectory, "Images", "Hank", anoDt.Rows[0]["SOPInstanceUID"] + ".dcm"));
}
if (putterType == typeof(PutInUidSeriesFolders))
{
expectedFile = new FileInfo(Path.Combine(TestContext.CurrentContext.WorkDirectory, "Images", "Hank", anoDt.Rows[0]["SeriesInstanceUID"].ToString(), anoDt.Rows[0]["SOPInstanceUID"] + ".dcm"));
}
if (putterType == typeof(PutInUidStudySeriesFolders))
{
expectedFile = new FileInfo(Path.Combine(TestContext.CurrentContext.WorkDirectory, "Images", "Hank", anoDt.Rows[0]["StudyInstanceUID"].ToString(), anoDt.Rows[0]["SeriesInstanceUID"].ToString(), anoDt.Rows[0]["SOPInstanceUID"] + ".dcm"));
}
Assert.IsTrue(expectedFile.Exists);
var anoDicom = DicomFile.Open(expectedFile.FullName);
Assert.AreEqual("Hank", anoDicom.Dataset.GetValue <string>(DicomTag.PatientID, 0));
Assert.AreEqual(anoDt.Rows[0]["SOPInstanceUID"], anoDicom.Dataset.GetValue <string>(DicomTag.SOPInstanceUID, 0));
Assert.AreEqual(56, anoDicom.Dataset.GetValue <string>(DicomTag.SeriesInstanceUID, 0).Length);
Assert.AreEqual(anoDt.Rows[0]["StudyInstanceUID"], anoDicom.Dataset.GetValue <string>(DicomTag.StudyInstanceUID, 0));
foreach (var(key, _) in thingThatShouldDisappear)
{
//if it chopped out the entire tag
if (!anoDicom.Dataset.Contains(key))
{
continue;
}
if (anoDicom.Dataset.GetValueCount(key) == 0)
{
continue;
}
var value = anoDicom.Dataset.GetSingleValue <string>(key);
switch (value)
{
//allowed values
case "ANONYMOUS": continue;
//anonymous date
case "00010101": Assert.IsFalse(keepDates);
continue;
case "20020101": Assert.IsTrue(keepDates);
continue;
default: Assert.Fail("Unexpected value for " + key + ":" + value);
break;
}
}
foreach (var(key, value) in thingsThatShouldRemain)
{
Assert.AreEqual(value, anoDicom.Dataset.GetValue <string>(key, 0));
}
}
public void PreInitialize(IExtractCommand value, IDataLoadEventListener listener)
{
// We only want to extract the files once so lets do it as part of extracting globals
_command = value as ExtractGlobalsCommand;
}