public bool GetNextFileOrDirectoryToProcess(out DirectoryInfo directory, out AmbiguousFilePath file) { directory = null; file = null; if (index >= _filesAndOrDirectories.Length) { return(false); } if (File.Exists(_filesAndOrDirectories[index])) { file = new AmbiguousFilePath(_filesAndOrDirectories[index]); index++; return(true); } if (Directory.Exists(_filesAndOrDirectories[index])) { directory = new DirectoryInfo(_filesAndOrDirectories[index]); index++; return(true); } throw new Exception("Array element " + index + " of filesAndOrDirectories was not a File or Directory (or the referenced file did not exist). Array element is '" + _filesAndOrDirectories[index] + "'"); }
public void GetDatasetFromZipFileTest() { FileInfo fzip = new FileInfo(Path.Combine(TestContext.CurrentContext.WorkDirectory, "omgzip.zip")); if (fzip.Exists) { fzip.Delete(); } using (var z = ZipFile.Open(fzip.FullName, ZipArchiveMode.Create)) { var entry = z.CreateEntry("test.dcm"); using (Stream s = entry.Open()) s.Write(TestDicomFiles.IM_0001_0013, 0, TestDicomFiles.IM_0001_0013.Length); } Assert.Throws <AmbiguousFilePathResolutionException>(() => new AmbiguousFilePath(Path.Combine(TestContext.CurrentContext.WorkDirectory, "omgzip.zip")).GetDataset()); Assert.Throws <AmbiguousFilePathResolutionException>(() => new AmbiguousFilePath(Path.Combine(TestContext.CurrentContext.WorkDirectory, "omgzip.zip!lol")).GetDataset()); var a = new AmbiguousFilePath(Path.Combine(TestContext.CurrentContext.WorkDirectory, "omgzip.zip!test.dcm")); var ds = a.GetDataset(); Assert.NotNull(ds.Dataset.GetValue <string>(DicomTag.SOPInstanceUID, 0)); fzip.Delete(); }
public void GetDatasetFromFileTest() { FileInfo f = new FileInfo(Path.Combine(TestContext.CurrentContext.WorkDirectory, "test.dcm")); File.WriteAllBytes(f.FullName, TestDicomFiles.IM_0001_0013); var a = new AmbiguousFilePath(f.FullName); var ds = a.GetDataset(); Assert.NotNull(ds.Dataset.GetValue <string>(DicomTag.SOPInstanceUID, 0)); f.Delete(); }
public bool GetNextFileOrDirectoryToProcess(out DirectoryInfo directory, out AmbiguousFilePath file) { file = null; directory = null; if (_dataExhausted) { return(false); } //input is a single dicom file/zip if (_lines == null) { _dataExhausted = true; file = new AmbiguousFilePath(_file.File.FullName); return(true); } //input was a text file full of other things to load if (_linesCurrent < _lines.Length) { var line = _lines[_linesCurrent]; if (File.Exists(line.Trim())) { _linesCurrent++; file = new AmbiguousFilePath(new FileInfo(line.Trim()).FullName); return(true); } if (Directory.Exists(line.Trim())) { _linesCurrent++; directory = new DirectoryInfo(line); return(true); } if (!AmbiguousFilePath.IsZipReference(line)) { throw new Exception( $"Text file '{_file.File.Name}' contained a line that was neither a File or a Directory:'{line}'"); } _linesCurrent++; file = new AmbiguousFilePath(line); return(true); } _dataExhausted = true; return(false); }
private void ProcessZipArchive(DataTable dt, IDataLoadEventListener listener, string zipFileName) { var skippedEntries = 0; var corruptedEntries = 0; try { using (var archive = ZipFile.Open(zipFileName, ZipArchiveMode.Read)) foreach (var f in archive.Entries) { //it's not a dicom file! if (!AmbiguousFilePath.IsDicomReference(f.FullName)) { skippedEntries++; continue; } try { var buffer = ByteStreamHelper.ReadFully(f.Open()); using (var memoryStream = new MemoryStream(buffer)) ProcessFile(memoryStream, dt, zipFileName + "!" + f.FullName, listener); } catch (Exception e) { corruptedEntries++; RecordError("Zip entry '" + f.FullName + "'", e); if (corruptedEntries <= 3) { continue; } listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, "Skipping the rest of '" + f.FullName + "'", e)); break; } } } catch (InvalidDataException e) { listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, "Error processing zip file '" + zipFileName + "'", e)); } if (skippedEntries > 0) { listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, "Skipped '" + skippedEntries + "' in zip archive '" + zipFileName + "' because they did not have .dcm extensions")); } UpdateProgressListeners(); }
public void BasicPathsTest() { var a = new AmbiguousFilePath(@"c:\temp\my.dcm"); Assert.AreEqual(@"c:\temp\my.dcm", a.FullPath); a = new AmbiguousFilePath(@"c:\temp", @"c:\temp\my.dcm"); Assert.AreEqual(@"c:\temp\my.dcm", a.FullPath); a = new AmbiguousFilePath(@"c:\temp", @"c:\temp\myzip.zip!my.dcm"); Assert.AreEqual(@"c:\temp\myzip.zip!my.dcm", a.FullPath); a = new AmbiguousFilePath(@"c:\temp", @"\myzip.zip!my.dcm"); Assert.AreEqual(@"c:\temp\myzip.zip!my.dcm", a.FullPath); }
public void GetDatasetFromZipFile_WithPooling_Test() { FileInfo fzip = new FileInfo(Path.Combine(TestContext.CurrentContext.WorkDirectory, "omgzip.zip")); if (fzip.Exists) { fzip.Delete(); } var bytes = File.ReadAllBytes(Path.Combine(TestContext.CurrentContext.TestDirectory, "TestData", "IM-0001-0024.dcm")); //Create a zip file with lots of entries using (var z = ZipFile.Open(fzip.FullName, ZipArchiveMode.Create)) for (int i = 0; i < 1500; i++) { var entry = z.CreateEntry("test" + i + ".dcm"); using Stream s = entry.Open(); s.Write(bytes, 0, bytes.Length); } //we want to read one out of the middle var a = new AmbiguousFilePath(Path.Combine(TestContext.CurrentContext.WorkDirectory, "omgzip.zip!test750.dcm")); //read the same entry lots of times without pooling var sw = Stopwatch.StartNew(); for (int i = 0; i < 1000; i++) { a.GetDataset(); } Console.WriteLine("No Caching:" + sw.ElapsedMilliseconds + "ms"); //read the same entry lots of times with pooling sw = Stopwatch.StartNew(); using (var pool = new ZipPool()) { for (int i = 0; i < 1000; i++) { a.GetDataset(pool); } Console.WriteLine("With Caching:" + sw.ElapsedMilliseconds + "ms"); Assert.AreEqual(999, pool.CacheHits); Assert.AreEqual(1, pool.CacheMisses); } }
public void BasicPathsTest() { bool isLinux = EnvironmentInfo.IsLinux; if (isLinux) { //in linux this looks like a relative path var ex = Assert.Throws <ArgumentException>(() => new AmbiguousFilePath(@"c:\temp\my.dcm")); StringAssert.StartsWith("Relative path was encountered without specifying a root", ex.Message); ex = Assert.Throws <ArgumentException>(() => new AmbiguousFilePath(@"c:\temp", @"c:\temp\my.dcm")); StringAssert.IsMatch("Specified root path '.*' was not IsAbsolute", ex.Message); } else { var a = new AmbiguousFilePath(@"c:\temp\my.dcm"); Assert.AreEqual(@"c:\temp\my.dcm", a.FullPath); a = new AmbiguousFilePath(@"c:\temp", @"c:\temp\my.dcm"); Assert.AreEqual(@"c:\temp\my.dcm", a.FullPath); a = new AmbiguousFilePath(@"c:\temp", @"c:\temp\myzip.zip!my.dcm"); Assert.AreEqual(@"c:\temp\myzip.zip!my.dcm", a.FullPath); a = new AmbiguousFilePath(@"c:\temp", @"myzip.zip!my.dcm"); Assert.AreEqual(@"c:\temp\myzip.zip!my.dcm", a.FullPath); } //give it some linux style paths var b = new AmbiguousFilePath(@"/temp/my.dcm"); Assert.AreEqual(@"/temp/my.dcm", b.FullPath); b = new AmbiguousFilePath(@"/temp", @"/temp/my.dcm"); Assert.AreEqual(@"/temp/my.dcm", b.FullPath); b = new AmbiguousFilePath(@"/temp", @"/temp/myzip.zip!my.dcm"); Assert.AreEqual(@"/temp/myzip.zip!my.dcm", b.FullPath); b = new AmbiguousFilePath(@"/temp/", @"./myzip.zip!my.dcm"); Assert.AreEqual(@"/temp/./myzip.zip!my.dcm", b.FullPath); }
public void TestZipEntry_Exists() { var zipFile = new FileInfo(Path.Combine(TestContext.CurrentContext.WorkDirectory, "my.zip")); var rootDir = Directory.CreateDirectory(Path.Combine(TestContext.CurrentContext.WorkDirectory, nameof(TestZipEntry_Exists))); var subDirectory = rootDir.CreateSubdirectory("subdir"); var sourceFile = new FileInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, @"TestData/IM-0001-0013.dcm")); sourceFile.CopyTo(Path.Combine(rootDir.FullName, "file1.dcm"), true); sourceFile.CopyTo(Path.Combine(subDirectory.FullName, "file2.dcm"), true); if (zipFile.Exists) { zipFile.Delete(); } ZipFile.CreateFromDirectory(rootDir.FullName, zipFile.FullName); FileAssert.Exists(zipFile.FullName); var exists = new AmbiguousFilePath(zipFile.FullName + "!file1.dcm"); Assert.IsNotNull(exists.GetDataset()); var notexists = new AmbiguousFilePath(zipFile.FullName + "!file2.dcm"); var ex = Assert.Throws <AmbiguousFilePathResolutionException>(() => notexists.GetDataset()); StringAssert.Contains("Could not find path 'file2.dcm' within zip archive", ex.Message); var existsRelative = new AmbiguousFilePath(zipFile.DirectoryName, "my.zip!file1.dcm"); Assert.IsNotNull(existsRelative.GetDataset()); var existsRelativeWithLeadingSlash = new AmbiguousFilePath(zipFile.DirectoryName, "my.zip!/file1.dcm"); Assert.IsNotNull(existsRelativeWithLeadingSlash.GetDataset()); var existsRelativeWithLeadingSlashInSubdir = new AmbiguousFilePath(zipFile.DirectoryName, "my.zip!/subdir/file2.dcm"); Assert.IsNotNull(existsRelativeWithLeadingSlashInSubdir.GetDataset()); var existsRelativeWithLeadingBackSlashInSubdir = new AmbiguousFilePath(zipFile.DirectoryName, "my.zip!\\subdir\\file2.dcm"); Assert.IsNotNull(existsRelativeWithLeadingBackSlashInSubdir.GetDataset()); }
private void ProcessDirectory(DataTable dt, DirectoryInfo directoryInfo, IDataLoadEventListener listener) { listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Started Directory '" + directoryInfo.FullName + "' on Thread " + Thread.CurrentThread.ManagedThreadId)); FileInfo[] dicomFiles; FileInfo[] zipFiles; try { dicomFiles = directoryInfo.EnumerateFiles().Where(f => AmbiguousFilePath.IsDicomReference(f.FullName)).ToArray(); zipFiles = directoryInfo.EnumerateFiles("*.zip").ToArray(); } catch (Exception e) { RecordError(directoryInfo.FullName, e); return; } //process all dcm files in current directory foreach (var dcmFile in dicomFiles) { try { using (var fs = dcmFile.OpenRead()) ProcessFile(fs, dt, dcmFile.FullName, listener); } catch (Exception e) { RecordError(dcmFile.FullName, e); } } foreach (var zipFile in zipFiles) { ProcessZipArchive(dt, listener, zipFile.FullName); } UpdateProgressListeners(); }
public void TestIsDicomReference(string input, bool expected) { Assert.AreEqual(expected, AmbiguousFilePath.IsDicomReference(input)); }
public void Test_ZipFile(bool expressRelative) { //get a clean database to upload to var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); //create a folder in which to generate some dicoms var dirToLoad = new DirectoryInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(Test_ZipFile))); if (dirToLoad.Exists) { dirToLoad.Delete(true); } dirToLoad.Create(); //generate some random dicoms var r = new Random(999); DicomDataGenerator generator = new DicomDataGenerator(r, dirToLoad, "CT") { MaximumImages = 5 }; var people = new PersonCollection(); people.GeneratePeople(1, r); generator.GenerateTestDataFile(people, new FileInfo("./inventory.csv"), 1); //This generates // Test_ZipFile // 2015 // 3 // 18 // 751140 2.25.166922918107154891877498685128076062226.dcm // 751140 2.25.179610809676265137473873365625829826423.dcm // 751140 2.25.201969634959506849065133495434871450465.dcm // 751140 2.25.237492679533001779093365416814254319890.dcm // 751140 2.25.316241631782653383510844072713132248731.dcm var yearDir = dirToLoad.GetDirectories().Single(); StringAssert.IsMatch("\\d{4}", yearDir.Name); //zip them up FileInfo zip = new FileInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(Test_ZipFile) + ".zip")); Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(Test_ZipFile) + ".zip"); if (zip.Exists) { zip.Delete(); } ZipFile.CreateFromDirectory(dirToLoad.FullName, zip.FullName); //tell the source to load the zip var f = new FlatFileToLoad(zip); var source = new DicomFileCollectionSource { FilenameField = "RelativeFileArchiveURI" }; if (expressRelative) { source.ArchiveRoot = TestContext.CurrentContext.TestDirectory; } source.PreInitialize(new FlatFileToLoadDicomFileWorklist(f), new ThrowImmediatelyDataLoadEventListener()); var tbl = source.GetChunk(new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken()); var destination = new DataTableUploadDestination(); destination.PreInitialize(db, new ThrowImmediatelyDataLoadEventListener()); destination.AllowResizingColumnsAtUploadTime = true; destination.ProcessPipelineData(tbl, new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken()); destination.Dispose(new ThrowImmediatelyDataLoadEventListener(), null); var finalTable = db.ExpectTable(destination.TargetTableName); using (var dt = finalTable.GetDataTable()) { //should be 5 rows in the final table (5 images) Assert.AreEqual(5, dt.Rows.Count); string pathInDbToDicomFile = (string)dt.Rows[0]["RelativeFileArchiveURI"]; //We expect either something like: // E:/RdmpDicom/Rdmp.Dicom.Tests/bin/Debug/netcoreapp2.2/Test_ZipFile.zip!2015/3/18/2.25.160787663560951826149226183314694084702.dcm // ./Test_ZipFile.zip!2015/3/18/2.25.105592977437473375573190160334447272386.dcm //the path referenced should be the file read in relative/absolute format StringAssert.IsMatch( expressRelative ? $@"./{zip.Name}![\d./]*.dcm": $@"{Regex.Escape(zip.FullName.Replace('\\','/'))}![\d./]*.dcm", pathInDbToDicomFile); StringAssert.Contains(yearDir.Name, pathInDbToDicomFile, "Expected zip file to have subdirectories and for them to be loaded correctly"); //confirm we can read that out again using (var pool = new ZipPool()) { var path = new AmbiguousFilePath(TestContext.CurrentContext.TestDirectory, pathInDbToDicomFile); Assert.IsNotNull(path.GetDataset(pool)); } } Assert.IsTrue(finalTable.Exists()); finalTable.Drop(); }
public void Test_ZipFileNotation(bool expressRelative) { //get a clean database to upload to var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); //create a folder in which to generate some dicoms var dirToLoad = new DirectoryInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(Test_ZipFileNotation))); if (dirToLoad.Exists) { dirToLoad.Delete(true); } dirToLoad.Create(); //generate some random dicoms var r = new Random(999); DicomDataGenerator generator = new DicomDataGenerator(r, dirToLoad, "CT") { MaximumImages = 5 }; var people = new PersonCollection(); people.GeneratePeople(1, r); generator.GenerateTestDataFile(people, new FileInfo("./inventory.csv"), 1); //This generates // Test_ZipFile // 2015 // 3 // 18 // 751140 2.25.166922918107154891877498685128076062226.dcm // 751140 2.25.179610809676265137473873365625829826423.dcm // 751140 2.25.201969634959506849065133495434871450465.dcm // 751140 2.25.237492679533001779093365416814254319890.dcm // 751140 2.25.316241631782653383510844072713132248731.dcm var yearDir = dirToLoad.GetDirectories().Single(); StringAssert.IsMatch("\\d{4}", yearDir.Name); //should be 5 images in the zip file var dicomFiles = yearDir.GetFiles("*.dcm", SearchOption.AllDirectories); Assert.AreEqual(5, dicomFiles.Length); //e.g. \2015\3\18\2.25.223398837779449245317520567111874824918.dcm //e.g. \2015\3\18\2.25.179610809676265137473873365625829826423.dcm var relativePathWithinZip1 = dicomFiles[0].FullName.Substring(dirToLoad.FullName.Length); var relativePathWithinZip2 = dicomFiles[1].FullName.Substring(dirToLoad.FullName.Length); //zip them up FileInfo zip = new FileInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(Test_ZipFile) + ".zip")); Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(Test_ZipFile) + ".zip"); if (zip.Exists) { zip.Delete(); } ZipFile.CreateFromDirectory(dirToLoad.FullName, zip.FullName); //e.g. E:\RdmpDicom\Rdmp.Dicom.Tests\bin\Debug\netcoreapp2.2\Test_ZipFile.zip!\2015\3\18\2.25.223398837779449245317520567111874824918.dcm string pathToLoad1 = zip.FullName + "!" + relativePathWithinZip1; string pathToLoad2 = zip.FullName + "!" + relativePathWithinZip2; var loadMeTextFile = new FileInfo(Path.Combine(dirToLoad.FullName, "LoadMe.txt")); //tell the source to load the zip File.WriteAllText(loadMeTextFile.FullName, string.Join(Environment.NewLine, pathToLoad1, pathToLoad2)); var f = new FlatFileToLoad(loadMeTextFile); //Setup source var source = new DicomFileCollectionSource { FilenameField = "RelativeFileArchiveURI" }; if (expressRelative) { source.ArchiveRoot = TestContext.CurrentContext.TestDirectory; } var worklist = new FlatFileToLoadDicomFileWorklist(f); //Setup destination var destination = new DataTableUploadDestination { AllowResizingColumnsAtUploadTime = true }; //setup pipeline var contextFactory = new DataFlowPipelineContextFactory <DataTable>(); var context = contextFactory.Create(PipelineUsage.FixedDestination | PipelineUsage.FixedDestination); //run pipeline var pipe = new DataFlowPipelineEngine <DataTable>(context, source, destination, new ThrowImmediatelyDataLoadEventListener()); pipe.Initialize(db, worklist); pipe.ExecutePipeline(new GracefulCancellationToken()); var finalTable = db.ExpectTable(destination.TargetTableName); using (var dt = finalTable.GetDataTable()) { //should be 2 rows (since we told it to only load 2 files out of the zip) Assert.AreEqual(2, dt.Rows.Count); string pathInDbToDicomFile = (string)dt.Rows[0]["RelativeFileArchiveURI"]; //We expect either something like: // E:/RdmpDicom/Rdmp.Dicom.Tests/bin/Debug/netcoreapp2.2/Test_ZipFile.zip!2015/3/18/2.25.160787663560951826149226183314694084702.dcm // ./Test_ZipFile.zip!2015/3/18/2.25.105592977437473375573190160334447272386.dcm //the path referenced should be the file read in relative/absolute format StringAssert.IsMatch( expressRelative ? $@"./{zip.Name}![\d./]*.dcm": $@"{Regex.Escape(zip.FullName.Replace('\\','/'))}![\d./]*.dcm", pathInDbToDicomFile); StringAssert.Contains(yearDir.Name, pathInDbToDicomFile, "Expected zip file to have subdirectories and for them to be loaded correctly"); //confirm we can read that out again using (var pool = new ZipPool()) { var path = new AmbiguousFilePath(TestContext.CurrentContext.TestDirectory, pathInDbToDicomFile); Assert.IsNotNull(path.GetDataset(pool)); } } Assert.IsTrue(finalTable.Exists()); finalTable.Drop(); }