public KVPAttacherPipelineUseCase(KVPAttacher kvpAttacher, FlatFileToLoad file) { ExplicitDestination = kvpAttacher; AddInitializationObject(file); GenerateContext(); }
public void PreInitialize(FlatFileToLoad value, IDataLoadEventListener listener) { //we have been given a new file we no longer know the headers. Headers = null; _fileToLoad = value; _listener = listener; }
protected DataTable RunGetChunk(FlatFileToLoad file, BadDataHandlingStrategy strategy, bool throwOnEmpty) { return(RunGetChunk(file, s => { s.BadDataHandlingStrategy = strategy; s.ThrowOnEmptyFiles = throwOnEmpty; })); }
public FlatFileColumnCollection(FlatFileToLoad toLoad, bool makeHeaderNamesSane, ExplicitTypingCollection explicitlyTypedColumns, string forceHeaders, bool forceHeadersReplacesFirstLineInFile, string ignoreColumns) { _toLoad = toLoad; _makeHeaderNamesSane = makeHeaderNamesSane; _explicitlyTypedColumns = explicitlyTypedColumns; _forceHeaders = forceHeaders; _forceHeadersReplacesFirstLineInFile = forceHeadersReplacesFirstLineInFile; _ignoreColumns = ignoreColumns; }
public FlatFileToDataTablePusher(FlatFileToLoad fileToLoad, FlatFileColumnCollection headers, Func <string, object> hackValuesFunc, bool attemptToResolveNewlinesInRecords, CultureInfo culture) { _fileToLoad = fileToLoad; _headers = headers; _hackValuesFunc = hackValuesFunc; _attemptToResolveNewlinesInRecords = attemptToResolveNewlinesInRecords; _culture = culture ?? CultureInfo.CurrentCulture; typeDeciderFactory = new TypeDeciderFactory(_culture); }
public FlatFileEventHandlers(FlatFileToLoad fileToLoad, FlatFileToDataTablePusher dataPusher, bool throwOnEmptyFiles, BadDataHandlingStrategy strategy, IDataLoadEventListener listener, int maximumErrorsToReport, bool ignoreBadDataEvents) { _fileToLoad = fileToLoad; _dataPusher = dataPusher; _throwOnEmptyFiles = throwOnEmptyFiles; _strategy = strategy; _listener = listener; _maximumErrorsToReport = maximumErrorsToReport; _ignoreBadDataEvents = ignoreBadDataEvents; }
public FlatFileToDataTablePusher(FlatFileToLoad fileToLoad, FlatFileColumnCollection headers, Func <string, object> hackValuesFunc, bool attemptToResolveNewlinesInRecords, CultureInfo culture) { _fileToLoad = fileToLoad; _headers = headers; _hackValuesFunc = hackValuesFunc; _attemptToResolveNewlinesInRecords = attemptToResolveNewlinesInRecords; _culture = culture; _dateTimeParser = new DateTimeTypeDecider(); if (culture != null) { _dateTimeParser.Culture = culture; } }
protected override void OpenFile(FileInfo fileToLoad, IDataLoadEventListener listener, GracefulCancellationToken cancellationToken) { if (BatchesReadyForProcessing.Any()) { throw new NotSupportedException("There are still batches awaiting dispatch to RAW, we cannot open a new file at this time"); } var flatFileToLoad = new FlatFileToLoad(fileToLoad); //stamp out the pipeline into an instance var dataFlow = new KVPAttacherPipelineUseCase(this, flatFileToLoad).GetEngine(PipelineForReadingFromFlatFile, listener); //will result in the opening and processing of the file and the passing of DataTables through the Pipeline finally arriving at the destination (us) in ProcessPipelineData dataFlow.ExecutePipeline(cancellationToken); }
public FlatFileToDataTablePusher(FlatFileToLoad fileToLoad, FlatFileColumnCollection headers, Func <string, object> hackValuesFunc, bool attemptToResolveNewlinesInRecords, CultureInfo culture, string explicitDateTimeFormat) { _fileToLoad = fileToLoad; _headers = headers; _hackValuesFunc = hackValuesFunc; _attemptToResolveNewlinesInRecords = attemptToResolveNewlinesInRecords; _culture = culture ?? CultureInfo.CurrentCulture; _explicitDateTimeFormat = explicitDateTimeFormat; typeDeciderFactory = new TypeDeciderFactory(_culture); if (!string.IsNullOrWhiteSpace(explicitDateTimeFormat)) { typeDeciderFactory.Settings.ExplicitDateFormats = new [] { explicitDateTimeFormat } } ; }
public FlatFileToLoadDicomFileWorklist(FlatFileToLoad file) { _file = file; if (file.File == null) { return; } //input is a textual list of files/zips if (file.File.Extension == ".txt") { if (_lines == null) { _lines = File.ReadAllLines(file.File.FullName).Where(l => !string.IsNullOrWhiteSpace(l)).ToArray(); _linesCurrent = 0; } } }
public void Test_SingleFile(bool expressRelative) { var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); var source = new DicomFileCollectionSource { FilenameField = "RelativeFileArchiveURI" }; if (expressRelative) { source.ArchiveRoot = TestContext.CurrentContext.TestDirectory; } var f = new FlatFileToLoad(new FileInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, @"TestData/IM-0001-0013.dcm"))); source.PreInitialize(new FlatFileToLoadDicomFileWorklist(f), new ThrowImmediatelyDataLoadEventListener()); var tbl = source.GetChunk(new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken()); var destination = new DataTableUploadDestination(); destination.PreInitialize(db, new ThrowImmediatelyDataLoadEventListener()); destination.AllowResizingColumnsAtUploadTime = true; destination.ProcessPipelineData(tbl, new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken()); destination.Dispose(new ThrowImmediatelyDataLoadEventListener(), null); var finalTable = db.ExpectTable(destination.TargetTableName); using (var dt = finalTable.GetDataTable()) { //should be 1 row in the final table Assert.AreEqual(1, dt.Rows.Count); //the path referenced should be the file read in relative/absolute format Assert.AreEqual(expressRelative ? "./TestData/IM-0001-0013.dcm": f.File.FullName.Replace('\\', '/') , dt.Rows[0]["RelativeFileArchiveURI"]); } Assert.IsTrue(finalTable.Exists()); finalTable.Drop(); }
public override void Execute() { base.Execute(); FlatFileToLoad flatFile; //if no explicit file has been chosen if (_file == null) { var file = BasicActivator.SelectFile("Cohort file"); //get user to pick one if (file == null) { return; } flatFile = new FlatFileToLoad(file); } else { flatFile = new FlatFileToLoad(_file); } var auditLogBuilder = new ExtractableCohortAuditLogBuilder(); var request = GetCohortCreationRequest(auditLogBuilder.GetDescription(flatFile.File)); //user choose to cancel the cohort creation request dialogue if (request == null) { return; } request.FileToLoad = flatFile; var configureAndExecuteDialog = GetConfigureAndExecuteControl(request, "Uploading File " + flatFile.File.Name); //add the flat file to the dialog with an appropriate description of what they are trying to achieve configureAndExecuteDialog.Run(BasicActivator.RepositoryLocator, null, null, null); }
public override void Execute() { base.Execute(); FlatFileToLoad flatFile; //if no explicit file has been chosen if (_file == null) { //get user to pick one OpenFileDialog ofd = new OpenFileDialog(); if (ofd.ShowDialog() != DialogResult.OK) { return; } flatFile = new FlatFileToLoad(new FileInfo(ofd.FileName)); } else { flatFile = new FlatFileToLoad(_file); } var request = GetCohortCreationRequest("Patient identifiers in file '" + flatFile.File.FullName + "'"); //user choose to cancel the cohort creation request dialogue if (request == null) { return; } request.FileToLoad = flatFile; var configureAndExecuteDialog = GetConfigureAndExecuteControl(request, "Uploading File " + flatFile.File.Name); //add the flat file to the dialog with an appropriate description of what they are trying to achieve Activator.ShowWindow(configureAndExecuteDialog, true); }
public void PipelineTest() { var source = new DicomFileCollectionSource(); source.FilenameField = "RelativeFileArchiveURI"; var f = new FlatFileToLoad(new FileInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, @"TestData\IM-0001-0013.dcm"))); source.PreInitialize(new FlatFileToLoadDicomFileWorklist(f), new ThrowImmediatelyDataLoadEventListener()); var tbl = source.GetChunk(new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken()); var destination = new DataTableUploadDestination(); destination.PreInitialize(DiscoveredDatabaseICanCreateRandomTablesIn, new ThrowImmediatelyDataLoadEventListener()); destination.AllowResizingColumnsAtUploadTime = true; destination.ProcessPipelineData(tbl, new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken()); destination.Dispose(new ThrowImmediatelyDataLoadEventListener(), null); var finalTable = DiscoveredDatabaseICanCreateRandomTablesIn.ExpectTable(destination.TargetTableName); Assert.IsTrue(finalTable.Exists()); finalTable.Drop(); }
protected DataTable RunGetChunk(FlatFileToLoad file, Action <DelimitedFlatFileDataFlowSource> adjust = null) { DelimitedFlatFileDataFlowSource source = new DelimitedFlatFileDataFlowSource(); source.PreInitialize(file, new ThrowImmediatelyDataLoadEventListener()); source.Separator = ","; source.StronglyTypeInput = true; //makes the source interpret the file types properly source.StronglyTypeInputBatchSize = 100; source.AttemptToResolveNewLinesInRecords = true; //maximise potential for conflicts if (adjust != null) { adjust(source); } try { return(source.GetChunk(new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken())); } finally { source.Dispose(new ThrowImmediatelyDataLoadEventListener(), null); } }
public void Test_ZipFileNotation(bool expressRelative) { //get a clean database to upload to var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); //create a folder in which to generate some dicoms var dirToLoad = new DirectoryInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(Test_ZipFileNotation))); if (dirToLoad.Exists) { dirToLoad.Delete(true); } dirToLoad.Create(); //generate some random dicoms var r = new Random(999); DicomDataGenerator generator = new DicomDataGenerator(r, dirToLoad, "CT") { MaximumImages = 5 }; var people = new PersonCollection(); people.GeneratePeople(1, r); generator.GenerateTestDataFile(people, new FileInfo("./inventory.csv"), 1); //This generates // Test_ZipFile // 2015 // 3 // 18 // 751140 2.25.166922918107154891877498685128076062226.dcm // 751140 2.25.179610809676265137473873365625829826423.dcm // 751140 2.25.201969634959506849065133495434871450465.dcm // 751140 2.25.237492679533001779093365416814254319890.dcm // 751140 2.25.316241631782653383510844072713132248731.dcm var yearDir = dirToLoad.GetDirectories().Single(); StringAssert.IsMatch("\\d{4}", yearDir.Name); //should be 5 images in the zip file var dicomFiles = yearDir.GetFiles("*.dcm", SearchOption.AllDirectories); Assert.AreEqual(5, dicomFiles.Length); //e.g. \2015\3\18\2.25.223398837779449245317520567111874824918.dcm //e.g. \2015\3\18\2.25.179610809676265137473873365625829826423.dcm var relativePathWithinZip1 = dicomFiles[0].FullName.Substring(dirToLoad.FullName.Length); var relativePathWithinZip2 = dicomFiles[1].FullName.Substring(dirToLoad.FullName.Length); //zip them up FileInfo zip = new FileInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(Test_ZipFile) + ".zip")); Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(Test_ZipFile) + ".zip"); if (zip.Exists) { zip.Delete(); } ZipFile.CreateFromDirectory(dirToLoad.FullName, zip.FullName); //e.g. E:\RdmpDicom\Rdmp.Dicom.Tests\bin\Debug\netcoreapp2.2\Test_ZipFile.zip!\2015\3\18\2.25.223398837779449245317520567111874824918.dcm string pathToLoad1 = zip.FullName + "!" + relativePathWithinZip1; string pathToLoad2 = zip.FullName + "!" + relativePathWithinZip2; var loadMeTextFile = new FileInfo(Path.Combine(dirToLoad.FullName, "LoadMe.txt")); //tell the source to load the zip File.WriteAllText(loadMeTextFile.FullName, string.Join(Environment.NewLine, pathToLoad1, pathToLoad2)); var f = new FlatFileToLoad(loadMeTextFile); //Setup source var source = new DicomFileCollectionSource { FilenameField = "RelativeFileArchiveURI" }; if (expressRelative) { source.ArchiveRoot = TestContext.CurrentContext.TestDirectory; } var worklist = new FlatFileToLoadDicomFileWorklist(f); //Setup destination var destination = new DataTableUploadDestination { AllowResizingColumnsAtUploadTime = true }; //setup pipeline var contextFactory = new DataFlowPipelineContextFactory <DataTable>(); var context = contextFactory.Create(PipelineUsage.FixedDestination | PipelineUsage.FixedDestination); //run pipeline var pipe = new DataFlowPipelineEngine <DataTable>(context, source, destination, new ThrowImmediatelyDataLoadEventListener()); pipe.Initialize(db, worklist); pipe.ExecutePipeline(new GracefulCancellationToken()); var finalTable = db.ExpectTable(destination.TargetTableName); using (var dt = finalTable.GetDataTable()) { //should be 2 rows (since we told it to only load 2 files out of the zip) Assert.AreEqual(2, dt.Rows.Count); string pathInDbToDicomFile = (string)dt.Rows[0]["RelativeFileArchiveURI"]; //We expect either something like: // E:/RdmpDicom/Rdmp.Dicom.Tests/bin/Debug/netcoreapp2.2/Test_ZipFile.zip!2015/3/18/2.25.160787663560951826149226183314694084702.dcm // ./Test_ZipFile.zip!2015/3/18/2.25.105592977437473375573190160334447272386.dcm //the path referenced should be the file read in relative/absolute format StringAssert.IsMatch( expressRelative ? $@"./{zip.Name}![\d./]*.dcm": $@"{Regex.Escape(zip.FullName.Replace('\\','/'))}![\d./]*.dcm", pathInDbToDicomFile); StringAssert.Contains(yearDir.Name, pathInDbToDicomFile, "Expected zip file to have subdirectories and for them to be loaded correctly"); //confirm we can read that out again using (var pool = new ZipPool()) { var path = new AmbiguousFilePath(TestContext.CurrentContext.TestDirectory, pathInDbToDicomFile); Assert.IsNotNull(path.GetDataset(pool)); } } Assert.IsTrue(finalTable.Exists()); finalTable.Drop(); }
public void PreInitialize(FlatFileToLoad value, IDataLoadEventListener listener) { _fileToLoad = value; }
public void Test_ZipFile(bool expressRelative) { //get a clean database to upload to var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); //create a folder in which to generate some dicoms var dirToLoad = new DirectoryInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(Test_ZipFile))); if (dirToLoad.Exists) { dirToLoad.Delete(true); } dirToLoad.Create(); //generate some random dicoms var r = new Random(999); DicomDataGenerator generator = new DicomDataGenerator(r, dirToLoad, "CT") { MaximumImages = 5 }; var people = new PersonCollection(); people.GeneratePeople(1, r); generator.GenerateTestDataFile(people, new FileInfo("./inventory.csv"), 1); //This generates // Test_ZipFile // 2015 // 3 // 18 // 751140 2.25.166922918107154891877498685128076062226.dcm // 751140 2.25.179610809676265137473873365625829826423.dcm // 751140 2.25.201969634959506849065133495434871450465.dcm // 751140 2.25.237492679533001779093365416814254319890.dcm // 751140 2.25.316241631782653383510844072713132248731.dcm var yearDir = dirToLoad.GetDirectories().Single(); StringAssert.IsMatch("\\d{4}", yearDir.Name); //zip them up FileInfo zip = new FileInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(Test_ZipFile) + ".zip")); Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(Test_ZipFile) + ".zip"); if (zip.Exists) { zip.Delete(); } ZipFile.CreateFromDirectory(dirToLoad.FullName, zip.FullName); //tell the source to load the zip var f = new FlatFileToLoad(zip); var source = new DicomFileCollectionSource { FilenameField = "RelativeFileArchiveURI" }; if (expressRelative) { source.ArchiveRoot = TestContext.CurrentContext.TestDirectory; } source.PreInitialize(new FlatFileToLoadDicomFileWorklist(f), new ThrowImmediatelyDataLoadEventListener()); var tbl = source.GetChunk(new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken()); var destination = new DataTableUploadDestination(); destination.PreInitialize(db, new ThrowImmediatelyDataLoadEventListener()); destination.AllowResizingColumnsAtUploadTime = true; destination.ProcessPipelineData(tbl, new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken()); destination.Dispose(new ThrowImmediatelyDataLoadEventListener(), null); var finalTable = db.ExpectTable(destination.TargetTableName); using (var dt = finalTable.GetDataTable()) { //should be 5 rows in the final table (5 images) Assert.AreEqual(5, dt.Rows.Count); string pathInDbToDicomFile = (string)dt.Rows[0]["RelativeFileArchiveURI"]; //We expect either something like: // E:/RdmpDicom/Rdmp.Dicom.Tests/bin/Debug/netcoreapp2.2/Test_ZipFile.zip!2015/3/18/2.25.160787663560951826149226183314694084702.dcm // ./Test_ZipFile.zip!2015/3/18/2.25.105592977437473375573190160334447272386.dcm //the path referenced should be the file read in relative/absolute format StringAssert.IsMatch( expressRelative ? $@"./{zip.Name}![\d./]*.dcm": $@"{Regex.Escape(zip.FullName.Replace('\\','/'))}![\d./]*.dcm", pathInDbToDicomFile); StringAssert.Contains(yearDir.Name, pathInDbToDicomFile, "Expected zip file to have subdirectories and for them to be loaded correctly"); //confirm we can read that out again using (var pool = new ZipPool()) { var path = new AmbiguousFilePath(TestContext.CurrentContext.TestDirectory, pathInDbToDicomFile); Assert.IsNotNull(path.GetDataset(pool)); } } Assert.IsTrue(finalTable.Exists()); finalTable.Drop(); }