public KVPAttacherPipelineUseCase(KVPAttacher kvpAttacher, FlatFileToLoad file)
{
ExplicitDestination = kvpAttacher;
AddInitializationObject(file);
GenerateContext();
}
public void PreInitialize(FlatFileToLoad value, IDataLoadEventListener listener)
{
//we have been given a new file we no longer know the headers.
Headers = null;
_fileToLoad = value;
_listener = listener;
}
protected DataTable RunGetChunk(FlatFileToLoad file, BadDataHandlingStrategy strategy, bool throwOnEmpty)
{
return(RunGetChunk(file, s =>
{
s.BadDataHandlingStrategy = strategy;
s.ThrowOnEmptyFiles = throwOnEmpty;
}));
}
public FlatFileColumnCollection(FlatFileToLoad toLoad, bool makeHeaderNamesSane, ExplicitTypingCollection explicitlyTypedColumns, string forceHeaders, bool forceHeadersReplacesFirstLineInFile, string ignoreColumns)
{
_toLoad = toLoad;
_makeHeaderNamesSane = makeHeaderNamesSane;
_explicitlyTypedColumns = explicitlyTypedColumns;
_forceHeaders = forceHeaders;
_forceHeadersReplacesFirstLineInFile = forceHeadersReplacesFirstLineInFile;
_ignoreColumns = ignoreColumns;
}
public FlatFileToDataTablePusher(FlatFileToLoad fileToLoad, FlatFileColumnCollection headers, Func <string, object> hackValuesFunc, bool attemptToResolveNewlinesInRecords, CultureInfo culture)
{
_fileToLoad = fileToLoad;
_headers = headers;
_hackValuesFunc = hackValuesFunc;
_attemptToResolveNewlinesInRecords = attemptToResolveNewlinesInRecords;
_culture = culture ?? CultureInfo.CurrentCulture;
typeDeciderFactory = new TypeDeciderFactory(_culture);
}
public FlatFileEventHandlers(FlatFileToLoad fileToLoad, FlatFileToDataTablePusher dataPusher, bool throwOnEmptyFiles, BadDataHandlingStrategy strategy, IDataLoadEventListener listener, int maximumErrorsToReport, bool ignoreBadDataEvents)
{
_fileToLoad = fileToLoad;
_dataPusher = dataPusher;
_throwOnEmptyFiles = throwOnEmptyFiles;
_strategy = strategy;
_listener = listener;
_maximumErrorsToReport = maximumErrorsToReport;
_ignoreBadDataEvents = ignoreBadDataEvents;
}
public FlatFileToDataTablePusher(FlatFileToLoad fileToLoad, FlatFileColumnCollection headers, Func <string, object> hackValuesFunc, bool attemptToResolveNewlinesInRecords, CultureInfo culture)
{
_fileToLoad = fileToLoad;
_headers = headers;
_hackValuesFunc = hackValuesFunc;
_attemptToResolveNewlinesInRecords = attemptToResolveNewlinesInRecords;
_culture = culture;
_dateTimeParser = new DateTimeTypeDecider();
if (culture != null)
{
_dateTimeParser.Culture = culture;
}
}
protected override void OpenFile(FileInfo fileToLoad, IDataLoadEventListener listener, GracefulCancellationToken cancellationToken)
{
if (BatchesReadyForProcessing.Any())
{
throw new NotSupportedException("There are still batches awaiting dispatch to RAW, we cannot open a new file at this time");
}
var flatFileToLoad = new FlatFileToLoad(fileToLoad);
//stamp out the pipeline into an instance
var dataFlow = new KVPAttacherPipelineUseCase(this, flatFileToLoad).GetEngine(PipelineForReadingFromFlatFile, listener);
//will result in the opening and processing of the file and the passing of DataTables through the Pipeline finally arriving at the destination (us) in ProcessPipelineData
dataFlow.ExecutePipeline(cancellationToken);
}
public FlatFileToDataTablePusher(FlatFileToLoad fileToLoad, FlatFileColumnCollection headers, Func <string, object> hackValuesFunc, bool attemptToResolveNewlinesInRecords, CultureInfo culture, string explicitDateTimeFormat)
{
_fileToLoad = fileToLoad;
_headers = headers;
_hackValuesFunc = hackValuesFunc;
_attemptToResolveNewlinesInRecords = attemptToResolveNewlinesInRecords;
_culture = culture ?? CultureInfo.CurrentCulture;
_explicitDateTimeFormat = explicitDateTimeFormat;
typeDeciderFactory = new TypeDeciderFactory(_culture);
if (!string.IsNullOrWhiteSpace(explicitDateTimeFormat))
{
typeDeciderFactory.Settings.ExplicitDateFormats = new [] { explicitDateTimeFormat }
}
;
}
public FlatFileToLoadDicomFileWorklist(FlatFileToLoad file)
{
_file = file;
if (file.File == null)
{
return;
}
//input is a textual list of files/zips
if (file.File.Extension == ".txt")
{
if (_lines == null)
{
_lines = File.ReadAllLines(file.File.FullName).Where(l => !string.IsNullOrWhiteSpace(l)).ToArray();
_linesCurrent = 0;
}
}
}
public void Test_SingleFile(bool expressRelative)
{
var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer);
var source = new DicomFileCollectionSource {
FilenameField = "RelativeFileArchiveURI"
};
if (expressRelative)
{
source.ArchiveRoot = TestContext.CurrentContext.TestDirectory;
}
var f = new FlatFileToLoad(new FileInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, @"TestData/IM-0001-0013.dcm")));
source.PreInitialize(new FlatFileToLoadDicomFileWorklist(f), new ThrowImmediatelyDataLoadEventListener());
var tbl = source.GetChunk(new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken());
var destination = new DataTableUploadDestination();
destination.PreInitialize(db, new ThrowImmediatelyDataLoadEventListener());
destination.AllowResizingColumnsAtUploadTime = true;
destination.ProcessPipelineData(tbl, new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken());
destination.Dispose(new ThrowImmediatelyDataLoadEventListener(), null);
var finalTable = db.ExpectTable(destination.TargetTableName);
using (var dt = finalTable.GetDataTable())
{
//should be 1 row in the final table
Assert.AreEqual(1, dt.Rows.Count);
//the path referenced should be the file read in relative/absolute format
Assert.AreEqual(expressRelative ? "./TestData/IM-0001-0013.dcm":
f.File.FullName.Replace('\\', '/')
, dt.Rows[0]["RelativeFileArchiveURI"]);
}
Assert.IsTrue(finalTable.Exists());
finalTable.Drop();
}
public override void Execute()
{
base.Execute();
FlatFileToLoad flatFile;
//if no explicit file has been chosen
if (_file == null)
{
var file = BasicActivator.SelectFile("Cohort file");
//get user to pick one
if (file == null)
{
return;
}
flatFile = new FlatFileToLoad(file);
}
else
{
flatFile = new FlatFileToLoad(_file);
}
var auditLogBuilder = new ExtractableCohortAuditLogBuilder();
var request = GetCohortCreationRequest(auditLogBuilder.GetDescription(flatFile.File));
//user choose to cancel the cohort creation request dialogue
if (request == null)
{
return;
}
request.FileToLoad = flatFile;
var configureAndExecuteDialog = GetConfigureAndExecuteControl(request, "Uploading File " + flatFile.File.Name);
//add the flat file to the dialog with an appropriate description of what they are trying to achieve
configureAndExecuteDialog.Run(BasicActivator.RepositoryLocator, null, null, null);
}
public override void Execute()
{
base.Execute();
FlatFileToLoad flatFile;
//if no explicit file has been chosen
if (_file == null)
{
//get user to pick one
OpenFileDialog ofd = new OpenFileDialog();
if (ofd.ShowDialog() != DialogResult.OK)
{
return;
}
flatFile = new FlatFileToLoad(new FileInfo(ofd.FileName));
}
else
{
flatFile = new FlatFileToLoad(_file);
}
var request = GetCohortCreationRequest("Patient identifiers in file '" + flatFile.File.FullName + "'");
//user choose to cancel the cohort creation request dialogue
if (request == null)
{
return;
}
request.FileToLoad = flatFile;
var configureAndExecuteDialog = GetConfigureAndExecuteControl(request, "Uploading File " + flatFile.File.Name);
//add the flat file to the dialog with an appropriate description of what they are trying to achieve
Activator.ShowWindow(configureAndExecuteDialog, true);
}
public void PipelineTest()
{
var source = new DicomFileCollectionSource();
source.FilenameField = "RelativeFileArchiveURI";
var f = new FlatFileToLoad(new FileInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, @"TestData\IM-0001-0013.dcm")));
source.PreInitialize(new FlatFileToLoadDicomFileWorklist(f), new ThrowImmediatelyDataLoadEventListener());
var tbl = source.GetChunk(new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken());
var destination = new DataTableUploadDestination();
destination.PreInitialize(DiscoveredDatabaseICanCreateRandomTablesIn, new ThrowImmediatelyDataLoadEventListener());
destination.AllowResizingColumnsAtUploadTime = true;
destination.ProcessPipelineData(tbl, new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken());
destination.Dispose(new ThrowImmediatelyDataLoadEventListener(), null);
var finalTable = DiscoveredDatabaseICanCreateRandomTablesIn.ExpectTable(destination.TargetTableName);
Assert.IsTrue(finalTable.Exists());
finalTable.Drop();
}
protected DataTable RunGetChunk(FlatFileToLoad file, Action <DelimitedFlatFileDataFlowSource> adjust = null)
{
DelimitedFlatFileDataFlowSource source = new DelimitedFlatFileDataFlowSource();
source.PreInitialize(file, new ThrowImmediatelyDataLoadEventListener());
source.Separator = ",";
source.StronglyTypeInput = true; //makes the source interpret the file types properly
source.StronglyTypeInputBatchSize = 100;
source.AttemptToResolveNewLinesInRecords = true; //maximise potential for conflicts
if (adjust != null)
{
adjust(source);
}
try
{
return(source.GetChunk(new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken()));
}
finally
{
source.Dispose(new ThrowImmediatelyDataLoadEventListener(), null);
}
}
public void Test_ZipFileNotation(bool expressRelative)
{
//get a clean database to upload to
var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer);
//create a folder in which to generate some dicoms
var dirToLoad = new DirectoryInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(Test_ZipFileNotation)));
if (dirToLoad.Exists)
{
dirToLoad.Delete(true);
}
dirToLoad.Create();
//generate some random dicoms
var r = new Random(999);
DicomDataGenerator generator = new DicomDataGenerator(r, dirToLoad, "CT")
{
MaximumImages = 5
};
var people = new PersonCollection();
people.GeneratePeople(1, r);
generator.GenerateTestDataFile(people, new FileInfo("./inventory.csv"), 1);
//This generates
// Test_ZipFile
// 2015
// 3
// 18
// 751140 2.25.166922918107154891877498685128076062226.dcm
// 751140 2.25.179610809676265137473873365625829826423.dcm
// 751140 2.25.201969634959506849065133495434871450465.dcm
// 751140 2.25.237492679533001779093365416814254319890.dcm
// 751140 2.25.316241631782653383510844072713132248731.dcm
var yearDir = dirToLoad.GetDirectories().Single();
StringAssert.IsMatch("\\d{4}", yearDir.Name);
//should be 5 images in the zip file
var dicomFiles = yearDir.GetFiles("*.dcm", SearchOption.AllDirectories);
Assert.AreEqual(5, dicomFiles.Length);
//e.g. \2015\3\18\2.25.223398837779449245317520567111874824918.dcm
//e.g. \2015\3\18\2.25.179610809676265137473873365625829826423.dcm
var relativePathWithinZip1 = dicomFiles[0].FullName.Substring(dirToLoad.FullName.Length);
var relativePathWithinZip2 = dicomFiles[1].FullName.Substring(dirToLoad.FullName.Length);
//zip them up
FileInfo zip = new FileInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(Test_ZipFile) + ".zip")); Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(Test_ZipFile) + ".zip");
if (zip.Exists)
{
zip.Delete();
}
ZipFile.CreateFromDirectory(dirToLoad.FullName, zip.FullName);
//e.g. E:\RdmpDicom\Rdmp.Dicom.Tests\bin\Debug\netcoreapp2.2\Test_ZipFile.zip!\2015\3\18\2.25.223398837779449245317520567111874824918.dcm
string pathToLoad1 = zip.FullName + "!" + relativePathWithinZip1;
string pathToLoad2 = zip.FullName + "!" + relativePathWithinZip2;
var loadMeTextFile = new FileInfo(Path.Combine(dirToLoad.FullName, "LoadMe.txt"));
//tell the source to load the zip
File.WriteAllText(loadMeTextFile.FullName, string.Join(Environment.NewLine, pathToLoad1, pathToLoad2));
var f = new FlatFileToLoad(loadMeTextFile);
//Setup source
var source = new DicomFileCollectionSource {
FilenameField = "RelativeFileArchiveURI"
};
if (expressRelative)
{
source.ArchiveRoot = TestContext.CurrentContext.TestDirectory;
}
var worklist = new FlatFileToLoadDicomFileWorklist(f);
//Setup destination
var destination = new DataTableUploadDestination {
AllowResizingColumnsAtUploadTime = true
};
//setup pipeline
var contextFactory = new DataFlowPipelineContextFactory <DataTable>();
var context = contextFactory.Create(PipelineUsage.FixedDestination | PipelineUsage.FixedDestination);
//run pipeline
var pipe = new DataFlowPipelineEngine <DataTable>(context, source, destination, new ThrowImmediatelyDataLoadEventListener());
pipe.Initialize(db, worklist);
pipe.ExecutePipeline(new GracefulCancellationToken());
var finalTable = db.ExpectTable(destination.TargetTableName);
using (var dt = finalTable.GetDataTable())
{
//should be 2 rows (since we told it to only load 2 files out of the zip)
Assert.AreEqual(2, dt.Rows.Count);
string pathInDbToDicomFile = (string)dt.Rows[0]["RelativeFileArchiveURI"];
//We expect either something like:
// E:/RdmpDicom/Rdmp.Dicom.Tests/bin/Debug/netcoreapp2.2/Test_ZipFile.zip!2015/3/18/2.25.160787663560951826149226183314694084702.dcm
// ./Test_ZipFile.zip!2015/3/18/2.25.105592977437473375573190160334447272386.dcm
//the path referenced should be the file read in relative/absolute format
StringAssert.IsMatch(
expressRelative ? $@"./{zip.Name}![\d./]*.dcm":
$@"{Regex.Escape(zip.FullName.Replace('\\','/'))}![\d./]*.dcm",
pathInDbToDicomFile);
StringAssert.Contains(yearDir.Name, pathInDbToDicomFile, "Expected zip file to have subdirectories and for them to be loaded correctly");
//confirm we can read that out again
using (var pool = new ZipPool())
{
var path = new AmbiguousFilePath(TestContext.CurrentContext.TestDirectory, pathInDbToDicomFile);
Assert.IsNotNull(path.GetDataset(pool));
}
}
Assert.IsTrue(finalTable.Exists());
finalTable.Drop();
}
public void PreInitialize(FlatFileToLoad value, IDataLoadEventListener listener)
{
_fileToLoad = value;
}
public void Test_ZipFile(bool expressRelative)
{
//get a clean database to upload to
var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer);
//create a folder in which to generate some dicoms
var dirToLoad = new DirectoryInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(Test_ZipFile)));
if (dirToLoad.Exists)
{
dirToLoad.Delete(true);
}
dirToLoad.Create();
//generate some random dicoms
var r = new Random(999);
DicomDataGenerator generator = new DicomDataGenerator(r, dirToLoad, "CT")
{
MaximumImages = 5
};
var people = new PersonCollection();
people.GeneratePeople(1, r);
generator.GenerateTestDataFile(people, new FileInfo("./inventory.csv"), 1);
//This generates
// Test_ZipFile
// 2015
// 3
// 18
// 751140 2.25.166922918107154891877498685128076062226.dcm
// 751140 2.25.179610809676265137473873365625829826423.dcm
// 751140 2.25.201969634959506849065133495434871450465.dcm
// 751140 2.25.237492679533001779093365416814254319890.dcm
// 751140 2.25.316241631782653383510844072713132248731.dcm
var yearDir = dirToLoad.GetDirectories().Single();
StringAssert.IsMatch("\\d{4}", yearDir.Name);
//zip them up
FileInfo zip = new FileInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(Test_ZipFile) + ".zip")); Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(Test_ZipFile) + ".zip");
if (zip.Exists)
{
zip.Delete();
}
ZipFile.CreateFromDirectory(dirToLoad.FullName, zip.FullName);
//tell the source to load the zip
var f = new FlatFileToLoad(zip);
var source = new DicomFileCollectionSource {
FilenameField = "RelativeFileArchiveURI"
};
if (expressRelative)
{
source.ArchiveRoot = TestContext.CurrentContext.TestDirectory;
}
source.PreInitialize(new FlatFileToLoadDicomFileWorklist(f), new ThrowImmediatelyDataLoadEventListener());
var tbl = source.GetChunk(new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken());
var destination = new DataTableUploadDestination();
destination.PreInitialize(db, new ThrowImmediatelyDataLoadEventListener());
destination.AllowResizingColumnsAtUploadTime = true;
destination.ProcessPipelineData(tbl, new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken());
destination.Dispose(new ThrowImmediatelyDataLoadEventListener(), null);
var finalTable = db.ExpectTable(destination.TargetTableName);
using (var dt = finalTable.GetDataTable())
{
//should be 5 rows in the final table (5 images)
Assert.AreEqual(5, dt.Rows.Count);
string pathInDbToDicomFile = (string)dt.Rows[0]["RelativeFileArchiveURI"];
//We expect either something like:
// E:/RdmpDicom/Rdmp.Dicom.Tests/bin/Debug/netcoreapp2.2/Test_ZipFile.zip!2015/3/18/2.25.160787663560951826149226183314694084702.dcm
// ./Test_ZipFile.zip!2015/3/18/2.25.105592977437473375573190160334447272386.dcm
//the path referenced should be the file read in relative/absolute format
StringAssert.IsMatch(
expressRelative ? $@"./{zip.Name}![\d./]*.dcm":
$@"{Regex.Escape(zip.FullName.Replace('\\','/'))}![\d./]*.dcm",
pathInDbToDicomFile);
StringAssert.Contains(yearDir.Name, pathInDbToDicomFile, "Expected zip file to have subdirectories and for them to be loaded correctly");
//confirm we can read that out again
using (var pool = new ZipPool())
{
var path = new AmbiguousFilePath(TestContext.CurrentContext.TestDirectory, pathInDbToDicomFile);
Assert.IsNotNull(path.GetDataset(pool));
}
}
Assert.IsTrue(finalTable.Exists());
finalTable.Drop();
}
