public DataTable ProcessPipelineData(DataTable toProcess, IDataLoadEventListener listener, GracefulCancellationToken cancellationToken)
{
//Things we ignore, Lookups, SupportingSql etc
if (_extractCommand == null)
{
listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Ignoring non dataset command "));
return(toProcess);
}
//if it isn't a dicom dataset don't process it
if (!toProcess.Columns.Contains(RelativeArchiveColumnName))
{
listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, "Dataset " + _extractCommand.DatasetBundle.DataSet + " did not contain field '" + RelativeArchiveColumnName + "' so we will not attempt to extract images"));
return(toProcess);
}
if (_putter == null)
{
_putter = (IPutDicomFilesInExtractionDirectories) new ObjectConstructor().Construct(PutterType);
}
var projectNumber = _extractCommand.Configuration.Project.ProjectNumber.Value;
var mappingServer = new MappingRepository(UIDMappingServer);
var destinationDirectory = new DirectoryInfo(Path.Combine(_extractCommand.GetExtractionDirectory().FullName, "Images"));
var releaseCol = _extractCommand.QueryBuilder.SelectColumns.Select(c => c.IColumn).Single(c => c.IsExtractionIdentifier);
// See: ftp://medical.nema.org/medical/dicom/2011/11_15pu.pdf
var flags = DicomAnonymizer.SecurityProfileOptions.BasicProfile |
DicomAnonymizer.SecurityProfileOptions.CleanStructdCont |
DicomAnonymizer.SecurityProfileOptions.CleanDesc |
DicomAnonymizer.SecurityProfileOptions.RetainUIDs;
if (RetainDates)
{
flags |= DicomAnonymizer.SecurityProfileOptions.RetainLongFullDates;
}
var profile = DicomAnonymizer.SecurityProfile.LoadProfile(null, flags);
var anonymiser = new DicomAnonymizer(profile);
using (var pool = new ZipPool())
{
_sw.Start();
foreach (DataRow row in toProcess.Rows)
{
if (_errors > 0 && _errors > ErrorThreshold)
{
throw new Exception($"Number of errors reported ({_errors}) reached the threshold ({ErrorThreshold})");
}
cancellationToken.ThrowIfAbortRequested();
var path = new AmbiguousFilePath(ArchiveRootIfAny, (string)row[RelativeArchiveColumnName]);
DicomFile dicomFile;
try
{
dicomFile = path.GetDataset(pool);
}
catch (Exception e)
{
listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Error, $"Failed to get image at path '{path.FullPath}'", e));
_errors++;
continue;
}
//get the new patient ID
var releaseId = row[releaseCol.GetRuntimeName()].ToString();
DicomDataset ds;
try
{
ds = anonymiser.Anonymize(dicomFile.Dataset);
}
catch (Exception e)
{
listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Error, $"Failed to anonymize image at path '{path.FullPath}'", e));
_errors++;
continue;
}
//now we want to explicitly use our own release Id regardless of what FoDicom said
ds.AddOrUpdate(DicomTag.PatientID, releaseId);
//rewrite the UIDs
foreach (var kvp in UIDMapping.SupportedTags)
{
if (!ds.Contains(kvp.Key))
{
continue;
}
var value = ds.GetValue <string>(kvp.Key, 0);
//if it has a value for this UID
if (value == null)
{
continue;
}
var releaseValue = mappingServer.GetOrAllocateMapping(value, projectNumber, kvp.Value);
//change value in dataset
ds.AddOrUpdate(kvp.Key, releaseValue);
//and change value in DataTable
if (toProcess.Columns.Contains(kvp.Key.DictionaryEntry.Keyword))
{
row[kvp.Key.DictionaryEntry.Keyword] = releaseValue;
}
}
var newPath = _putter.WriteOutDataset(destinationDirectory, releaseId, ds);
row[RelativeArchiveColumnName] = newPath;
_anonymisedImagesCount++;
listener.OnProgress(this, new ProgressEventArgs("Writing ANO images", new ProgressMeasurement(_anonymisedImagesCount, ProgressType.Records), _sw.Elapsed));
}
_sw.Stop();
}
return(toProcess);
}
/// <summary>
///
/// </summary>
/// <returns></returns>
/// <exception cref="OperationCanceledException"></exception>
public ExitCodeType Run(IDataLoadJob job, GracefulCancellationToken cancellationToken)
{
job.StartLogging();
job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Starting load for " + job.LoadMetadata.Name));
try
{
foreach (var component in Components)
{
cancellationToken.ThrowIfAbortRequested();
try
{
//schedule the component for disposal
job.PushForDisposal(component);
//run current component
var exitCodeType = component.Run(job, cancellationToken);
//current component failed so jump out, either because load not nessesary or crash
if (exitCodeType == ExitCodeType.OperationNotRequired)
{
TryDispose(exitCodeType, job);
//load not nessesary so abort entire DLE process but also cleanup still
return(exitCodeType);
}
if (exitCodeType != ExitCodeType.Success)
{
throw new Exception("Component " + component.Description + " returned result " + exitCodeType);
}
}
catch (OperationCanceledException e)
{
job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, component.Description + " has been cancelled by the user", e));
throw;
}
catch (Exception e)
{
job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Error, component.Description + " crashed while running Job ", e));
job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Error, "Job crashed", e));
TryDispose(ExitCodeType.Error, job);
return(ExitCodeType.Error);
}
}
TryDispose(ExitCodeType.Success, job);
job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Completed job " + job.JobID));
return(ExitCodeType.Success);
}
catch (OperationCanceledException)
{
if (cancellationToken.IsAbortRequested)
{
job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Job " + job.JobID + "cancelled in pipeline"));
}
TryDispose(cancellationToken.IsAbortRequested ? ExitCodeType.Abort : ExitCodeType.Success, job);
throw;
}
}