public bool GetNextFileOrDirectoryToProcess(out DirectoryInfo directory, out AmbiguousFilePath file)
{
directory = null;
file = null;
if (index >= _filesAndOrDirectories.Length)
{
return(false);
}
if (File.Exists(_filesAndOrDirectories[index]))
{
file = new AmbiguousFilePath(_filesAndOrDirectories[index]);
index++;
return(true);
}
if (Directory.Exists(_filesAndOrDirectories[index]))
{
directory = new DirectoryInfo(_filesAndOrDirectories[index]);
index++;
return(true);
}
throw new Exception("Array element " + index + " of filesAndOrDirectories was not a File or Directory (or the referenced file did not exist). Array element is '" + _filesAndOrDirectories[index] + "'");
}
public void GetDatasetFromZipFileTest()
{
FileInfo fzip = new FileInfo(Path.Combine(TestContext.CurrentContext.WorkDirectory, "omgzip.zip"));
if (fzip.Exists)
{
fzip.Delete();
}
using (var z = ZipFile.Open(fzip.FullName, ZipArchiveMode.Create))
{
var entry = z.CreateEntry("test.dcm");
using (Stream s = entry.Open())
s.Write(TestDicomFiles.IM_0001_0013, 0, TestDicomFiles.IM_0001_0013.Length);
}
Assert.Throws <AmbiguousFilePathResolutionException>(() => new AmbiguousFilePath(Path.Combine(TestContext.CurrentContext.WorkDirectory, "omgzip.zip")).GetDataset());
Assert.Throws <AmbiguousFilePathResolutionException>(() => new AmbiguousFilePath(Path.Combine(TestContext.CurrentContext.WorkDirectory, "omgzip.zip!lol")).GetDataset());
var a = new AmbiguousFilePath(Path.Combine(TestContext.CurrentContext.WorkDirectory, "omgzip.zip!test.dcm"));
var ds = a.GetDataset();
Assert.NotNull(ds.Dataset.GetValue <string>(DicomTag.SOPInstanceUID, 0));
fzip.Delete();
}
public void GetDatasetFromFileTest()
{
FileInfo f = new FileInfo(Path.Combine(TestContext.CurrentContext.WorkDirectory, "test.dcm"));
File.WriteAllBytes(f.FullName, TestDicomFiles.IM_0001_0013);
var a = new AmbiguousFilePath(f.FullName);
var ds = a.GetDataset();
Assert.NotNull(ds.Dataset.GetValue <string>(DicomTag.SOPInstanceUID, 0));
f.Delete();
}
public bool GetNextFileOrDirectoryToProcess(out DirectoryInfo directory, out AmbiguousFilePath file)
{
file = null;
directory = null;
if (_dataExhausted)
{
return(false);
}
//input is a single dicom file/zip
if (_lines == null)
{
_dataExhausted = true;
file = new AmbiguousFilePath(_file.File.FullName);
return(true);
}
//input was a text file full of other things to load
if (_linesCurrent < _lines.Length)
{
var line = _lines[_linesCurrent];
if (File.Exists(line.Trim()))
{
_linesCurrent++;
file = new AmbiguousFilePath(new FileInfo(line.Trim()).FullName);
return(true);
}
if (Directory.Exists(line.Trim()))
{
_linesCurrent++;
directory = new DirectoryInfo(line);
return(true);
}
if (!AmbiguousFilePath.IsZipReference(line))
{
throw new Exception(
$"Text file '{_file.File.Name}' contained a line that was neither a File or a Directory:'{line}'");
}
_linesCurrent++;
file = new AmbiguousFilePath(line);
return(true);
}
_dataExhausted = true;
return(false);
}
private void ProcessZipArchive(DataTable dt, IDataLoadEventListener listener, string zipFileName)
{
var skippedEntries = 0;
var corruptedEntries = 0;
try
{
using (var archive = ZipFile.Open(zipFileName, ZipArchiveMode.Read))
foreach (var f in archive.Entries)
{
//it's not a dicom file!
if (!AmbiguousFilePath.IsDicomReference(f.FullName))
{
skippedEntries++;
continue;
}
try
{
var buffer = ByteStreamHelper.ReadFully(f.Open());
using (var memoryStream = new MemoryStream(buffer))
ProcessFile(memoryStream, dt, zipFileName + "!" + f.FullName, listener);
}
catch (Exception e)
{
corruptedEntries++;
RecordError("Zip entry '" + f.FullName + "'", e);
if (corruptedEntries <= 3)
{
continue;
}
listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, "Skipping the rest of '" + f.FullName + "'", e));
break;
}
}
}
catch (InvalidDataException e)
{
listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, "Error processing zip file '" + zipFileName + "'", e));
}
if (skippedEntries > 0)
{
listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, "Skipped '" + skippedEntries + "' in zip archive '" + zipFileName + "' because they did not have .dcm extensions"));
}
UpdateProgressListeners();
}
public void BasicPathsTest()
{
var a = new AmbiguousFilePath(@"c:\temp\my.dcm");
Assert.AreEqual(@"c:\temp\my.dcm", a.FullPath);
a = new AmbiguousFilePath(@"c:\temp", @"c:\temp\my.dcm");
Assert.AreEqual(@"c:\temp\my.dcm", a.FullPath);
a = new AmbiguousFilePath(@"c:\temp", @"c:\temp\myzip.zip!my.dcm");
Assert.AreEqual(@"c:\temp\myzip.zip!my.dcm", a.FullPath);
a = new AmbiguousFilePath(@"c:\temp", @"\myzip.zip!my.dcm");
Assert.AreEqual(@"c:\temp\myzip.zip!my.dcm", a.FullPath);
}
public void GetDatasetFromZipFile_WithPooling_Test()
{
FileInfo fzip = new FileInfo(Path.Combine(TestContext.CurrentContext.WorkDirectory, "omgzip.zip"));
if (fzip.Exists)
{
fzip.Delete();
}
var bytes = File.ReadAllBytes(Path.Combine(TestContext.CurrentContext.TestDirectory, "TestData", "IM-0001-0024.dcm"));
//Create a zip file with lots of entries
using (var z = ZipFile.Open(fzip.FullName, ZipArchiveMode.Create))
for (int i = 0; i < 1500; i++)
{
var entry = z.CreateEntry("test" + i + ".dcm");
using Stream s = entry.Open();
s.Write(bytes, 0, bytes.Length);
}
//we want to read one out of the middle
var a = new AmbiguousFilePath(Path.Combine(TestContext.CurrentContext.WorkDirectory, "omgzip.zip!test750.dcm"));
//read the same entry lots of times without pooling
var sw = Stopwatch.StartNew();
for (int i = 0; i < 1000; i++)
{
a.GetDataset();
}
Console.WriteLine("No Caching:" + sw.ElapsedMilliseconds + "ms");
//read the same entry lots of times with pooling
sw = Stopwatch.StartNew();
using (var pool = new ZipPool())
{
for (int i = 0; i < 1000; i++)
{
a.GetDataset(pool);
}
Console.WriteLine("With Caching:" + sw.ElapsedMilliseconds + "ms");
Assert.AreEqual(999, pool.CacheHits);
Assert.AreEqual(1, pool.CacheMisses);
}
}
public void BasicPathsTest()
{
bool isLinux = EnvironmentInfo.IsLinux;
if (isLinux)
{
//in linux this looks like a relative path
var ex = Assert.Throws <ArgumentException>(() => new AmbiguousFilePath(@"c:\temp\my.dcm"));
StringAssert.StartsWith("Relative path was encountered without specifying a root", ex.Message);
ex = Assert.Throws <ArgumentException>(() => new AmbiguousFilePath(@"c:\temp", @"c:\temp\my.dcm"));
StringAssert.IsMatch("Specified root path '.*' was not IsAbsolute", ex.Message);
}
else
{
var a = new AmbiguousFilePath(@"c:\temp\my.dcm");
Assert.AreEqual(@"c:\temp\my.dcm", a.FullPath);
a = new AmbiguousFilePath(@"c:\temp", @"c:\temp\my.dcm");
Assert.AreEqual(@"c:\temp\my.dcm", a.FullPath);
a = new AmbiguousFilePath(@"c:\temp", @"c:\temp\myzip.zip!my.dcm");
Assert.AreEqual(@"c:\temp\myzip.zip!my.dcm", a.FullPath);
a = new AmbiguousFilePath(@"c:\temp", @"myzip.zip!my.dcm");
Assert.AreEqual(@"c:\temp\myzip.zip!my.dcm", a.FullPath);
}
//give it some linux style paths
var b = new AmbiguousFilePath(@"/temp/my.dcm");
Assert.AreEqual(@"/temp/my.dcm", b.FullPath);
b = new AmbiguousFilePath(@"/temp", @"/temp/my.dcm");
Assert.AreEqual(@"/temp/my.dcm", b.FullPath);
b = new AmbiguousFilePath(@"/temp", @"/temp/myzip.zip!my.dcm");
Assert.AreEqual(@"/temp/myzip.zip!my.dcm", b.FullPath);
b = new AmbiguousFilePath(@"/temp/", @"./myzip.zip!my.dcm");
Assert.AreEqual(@"/temp/./myzip.zip!my.dcm", b.FullPath);
}
public void TestZipEntry_Exists()
{
var zipFile = new FileInfo(Path.Combine(TestContext.CurrentContext.WorkDirectory, "my.zip"));
var rootDir = Directory.CreateDirectory(Path.Combine(TestContext.CurrentContext.WorkDirectory, nameof(TestZipEntry_Exists)));
var subDirectory = rootDir.CreateSubdirectory("subdir");
var sourceFile = new FileInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, @"TestData/IM-0001-0013.dcm"));
sourceFile.CopyTo(Path.Combine(rootDir.FullName, "file1.dcm"), true);
sourceFile.CopyTo(Path.Combine(subDirectory.FullName, "file2.dcm"), true);
if (zipFile.Exists)
{
zipFile.Delete();
}
ZipFile.CreateFromDirectory(rootDir.FullName, zipFile.FullName);
FileAssert.Exists(zipFile.FullName);
var exists = new AmbiguousFilePath(zipFile.FullName + "!file1.dcm");
Assert.IsNotNull(exists.GetDataset());
var notexists = new AmbiguousFilePath(zipFile.FullName + "!file2.dcm");
var ex = Assert.Throws <AmbiguousFilePathResolutionException>(() => notexists.GetDataset());
StringAssert.Contains("Could not find path 'file2.dcm' within zip archive", ex.Message);
var existsRelative = new AmbiguousFilePath(zipFile.DirectoryName, "my.zip!file1.dcm");
Assert.IsNotNull(existsRelative.GetDataset());
var existsRelativeWithLeadingSlash = new AmbiguousFilePath(zipFile.DirectoryName, "my.zip!/file1.dcm");
Assert.IsNotNull(existsRelativeWithLeadingSlash.GetDataset());
var existsRelativeWithLeadingSlashInSubdir = new AmbiguousFilePath(zipFile.DirectoryName, "my.zip!/subdir/file2.dcm");
Assert.IsNotNull(existsRelativeWithLeadingSlashInSubdir.GetDataset());
var existsRelativeWithLeadingBackSlashInSubdir = new AmbiguousFilePath(zipFile.DirectoryName, "my.zip!\\subdir\\file2.dcm");
Assert.IsNotNull(existsRelativeWithLeadingBackSlashInSubdir.GetDataset());
}
private void ProcessDirectory(DataTable dt, DirectoryInfo directoryInfo, IDataLoadEventListener listener)
{
listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Started Directory '" + directoryInfo.FullName + "' on Thread " + Thread.CurrentThread.ManagedThreadId));
FileInfo[] dicomFiles;
FileInfo[] zipFiles;
try
{
dicomFiles = directoryInfo.EnumerateFiles().Where(f => AmbiguousFilePath.IsDicomReference(f.FullName)).ToArray();
zipFiles = directoryInfo.EnumerateFiles("*.zip").ToArray();
}
catch (Exception e)
{
RecordError(directoryInfo.FullName, e);
return;
}
//process all dcm files in current directory
foreach (var dcmFile in dicomFiles)
{
try
{
using (var fs = dcmFile.OpenRead())
ProcessFile(fs, dt, dcmFile.FullName, listener);
}
catch (Exception e)
{
RecordError(dcmFile.FullName, e);
}
}
foreach (var zipFile in zipFiles)
{
ProcessZipArchive(dt, listener, zipFile.FullName);
}
UpdateProgressListeners();
}
public void TestIsDicomReference(string input, bool expected)
{
Assert.AreEqual(expected, AmbiguousFilePath.IsDicomReference(input));
}
public void Test_ZipFile(bool expressRelative)
{
//get a clean database to upload to
var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer);
//create a folder in which to generate some dicoms
var dirToLoad = new DirectoryInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(Test_ZipFile)));
if (dirToLoad.Exists)
{
dirToLoad.Delete(true);
}
dirToLoad.Create();
//generate some random dicoms
var r = new Random(999);
DicomDataGenerator generator = new DicomDataGenerator(r, dirToLoad, "CT")
{
MaximumImages = 5
};
var people = new PersonCollection();
people.GeneratePeople(1, r);
generator.GenerateTestDataFile(people, new FileInfo("./inventory.csv"), 1);
//This generates
// Test_ZipFile
// 2015
// 3
// 18
// 751140 2.25.166922918107154891877498685128076062226.dcm
// 751140 2.25.179610809676265137473873365625829826423.dcm
// 751140 2.25.201969634959506849065133495434871450465.dcm
// 751140 2.25.237492679533001779093365416814254319890.dcm
// 751140 2.25.316241631782653383510844072713132248731.dcm
var yearDir = dirToLoad.GetDirectories().Single();
StringAssert.IsMatch("\\d{4}", yearDir.Name);
//zip them up
FileInfo zip = new FileInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(Test_ZipFile) + ".zip")); Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(Test_ZipFile) + ".zip");
if (zip.Exists)
{
zip.Delete();
}
ZipFile.CreateFromDirectory(dirToLoad.FullName, zip.FullName);
//tell the source to load the zip
var f = new FlatFileToLoad(zip);
var source = new DicomFileCollectionSource {
FilenameField = "RelativeFileArchiveURI"
};
if (expressRelative)
{
source.ArchiveRoot = TestContext.CurrentContext.TestDirectory;
}
source.PreInitialize(new FlatFileToLoadDicomFileWorklist(f), new ThrowImmediatelyDataLoadEventListener());
var tbl = source.GetChunk(new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken());
var destination = new DataTableUploadDestination();
destination.PreInitialize(db, new ThrowImmediatelyDataLoadEventListener());
destination.AllowResizingColumnsAtUploadTime = true;
destination.ProcessPipelineData(tbl, new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken());
destination.Dispose(new ThrowImmediatelyDataLoadEventListener(), null);
var finalTable = db.ExpectTable(destination.TargetTableName);
using (var dt = finalTable.GetDataTable())
{
//should be 5 rows in the final table (5 images)
Assert.AreEqual(5, dt.Rows.Count);
string pathInDbToDicomFile = (string)dt.Rows[0]["RelativeFileArchiveURI"];
//We expect either something like:
// E:/RdmpDicom/Rdmp.Dicom.Tests/bin/Debug/netcoreapp2.2/Test_ZipFile.zip!2015/3/18/2.25.160787663560951826149226183314694084702.dcm
// ./Test_ZipFile.zip!2015/3/18/2.25.105592977437473375573190160334447272386.dcm
//the path referenced should be the file read in relative/absolute format
StringAssert.IsMatch(
expressRelative ? $@"./{zip.Name}![\d./]*.dcm":
$@"{Regex.Escape(zip.FullName.Replace('\\','/'))}![\d./]*.dcm",
pathInDbToDicomFile);
StringAssert.Contains(yearDir.Name, pathInDbToDicomFile, "Expected zip file to have subdirectories and for them to be loaded correctly");
//confirm we can read that out again
using (var pool = new ZipPool())
{
var path = new AmbiguousFilePath(TestContext.CurrentContext.TestDirectory, pathInDbToDicomFile);
Assert.IsNotNull(path.GetDataset(pool));
}
}
Assert.IsTrue(finalTable.Exists());
finalTable.Drop();
}
public void Test_ZipFileNotation(bool expressRelative)
{
//get a clean database to upload to
var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer);
//create a folder in which to generate some dicoms
var dirToLoad = new DirectoryInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(Test_ZipFileNotation)));
if (dirToLoad.Exists)
{
dirToLoad.Delete(true);
}
dirToLoad.Create();
//generate some random dicoms
var r = new Random(999);
DicomDataGenerator generator = new DicomDataGenerator(r, dirToLoad, "CT")
{
MaximumImages = 5
};
var people = new PersonCollection();
people.GeneratePeople(1, r);
generator.GenerateTestDataFile(people, new FileInfo("./inventory.csv"), 1);
//This generates
// Test_ZipFile
// 2015
// 3
// 18
// 751140 2.25.166922918107154891877498685128076062226.dcm
// 751140 2.25.179610809676265137473873365625829826423.dcm
// 751140 2.25.201969634959506849065133495434871450465.dcm
// 751140 2.25.237492679533001779093365416814254319890.dcm
// 751140 2.25.316241631782653383510844072713132248731.dcm
var yearDir = dirToLoad.GetDirectories().Single();
StringAssert.IsMatch("\\d{4}", yearDir.Name);
//should be 5 images in the zip file
var dicomFiles = yearDir.GetFiles("*.dcm", SearchOption.AllDirectories);
Assert.AreEqual(5, dicomFiles.Length);
//e.g. \2015\3\18\2.25.223398837779449245317520567111874824918.dcm
//e.g. \2015\3\18\2.25.179610809676265137473873365625829826423.dcm
var relativePathWithinZip1 = dicomFiles[0].FullName.Substring(dirToLoad.FullName.Length);
var relativePathWithinZip2 = dicomFiles[1].FullName.Substring(dirToLoad.FullName.Length);
//zip them up
FileInfo zip = new FileInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(Test_ZipFile) + ".zip")); Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(Test_ZipFile) + ".zip");
if (zip.Exists)
{
zip.Delete();
}
ZipFile.CreateFromDirectory(dirToLoad.FullName, zip.FullName);
//e.g. E:\RdmpDicom\Rdmp.Dicom.Tests\bin\Debug\netcoreapp2.2\Test_ZipFile.zip!\2015\3\18\2.25.223398837779449245317520567111874824918.dcm
string pathToLoad1 = zip.FullName + "!" + relativePathWithinZip1;
string pathToLoad2 = zip.FullName + "!" + relativePathWithinZip2;
var loadMeTextFile = new FileInfo(Path.Combine(dirToLoad.FullName, "LoadMe.txt"));
//tell the source to load the zip
File.WriteAllText(loadMeTextFile.FullName, string.Join(Environment.NewLine, pathToLoad1, pathToLoad2));
var f = new FlatFileToLoad(loadMeTextFile);
//Setup source
var source = new DicomFileCollectionSource {
FilenameField = "RelativeFileArchiveURI"
};
if (expressRelative)
{
source.ArchiveRoot = TestContext.CurrentContext.TestDirectory;
}
var worklist = new FlatFileToLoadDicomFileWorklist(f);
//Setup destination
var destination = new DataTableUploadDestination {
AllowResizingColumnsAtUploadTime = true
};
//setup pipeline
var contextFactory = new DataFlowPipelineContextFactory <DataTable>();
var context = contextFactory.Create(PipelineUsage.FixedDestination | PipelineUsage.FixedDestination);
//run pipeline
var pipe = new DataFlowPipelineEngine <DataTable>(context, source, destination, new ThrowImmediatelyDataLoadEventListener());
pipe.Initialize(db, worklist);
pipe.ExecutePipeline(new GracefulCancellationToken());
var finalTable = db.ExpectTable(destination.TargetTableName);
using (var dt = finalTable.GetDataTable())
{
//should be 2 rows (since we told it to only load 2 files out of the zip)
Assert.AreEqual(2, dt.Rows.Count);
string pathInDbToDicomFile = (string)dt.Rows[0]["RelativeFileArchiveURI"];
//We expect either something like:
// E:/RdmpDicom/Rdmp.Dicom.Tests/bin/Debug/netcoreapp2.2/Test_ZipFile.zip!2015/3/18/2.25.160787663560951826149226183314694084702.dcm
// ./Test_ZipFile.zip!2015/3/18/2.25.105592977437473375573190160334447272386.dcm
//the path referenced should be the file read in relative/absolute format
StringAssert.IsMatch(
expressRelative ? $@"./{zip.Name}![\d./]*.dcm":
$@"{Regex.Escape(zip.FullName.Replace('\\','/'))}![\d./]*.dcm",
pathInDbToDicomFile);
StringAssert.Contains(yearDir.Name, pathInDbToDicomFile, "Expected zip file to have subdirectories and for them to be loaded correctly");
//confirm we can read that out again
using (var pool = new ZipPool())
{
var path = new AmbiguousFilePath(TestContext.CurrentContext.TestDirectory, pathInDbToDicomFile);
Assert.IsNotNull(path.GetDataset(pool));
}
}
Assert.IsTrue(finalTable.Exists());
finalTable.Drop();
}
