protected override void OnExecute(CommandProcessor theProcessor)
{
long fileSize = 0;
if (File.Exists(_file.Filename))
{
var finfo = new FileInfo(_file.Filename);
fileSize = finfo.Length;
}
String seriesInstanceUid = _file.DataSet[DicomTags.SeriesInstanceUid].GetString(0, string.Empty);
String sopInstanceUid = _file.DataSet[DicomTags.SopInstanceUid].GetString(0, string.Empty);
if (_studyXml.Contains(seriesInstanceUid, sopInstanceUid))
{
_duplicate = true;
}
// Setup the insert parameters
if (false == _studyXml.AddFile(_file, fileSize, _settings))
{
Platform.Log(LogLevel.Error, "Unexpected error adding SOP to XML Study Descriptor for file {0}",
_file.Filename);
throw new ApplicationException("Unexpected error adding SOP to XML Study Descriptor for SOP: " +
_file.MediaStorageSopInstanceUid);
}
if (_writeFile)
{
// Write it back out. We flush it out with every added image so that if a failure happens,
// we can recover properly.
bool fileCreated;
_studyStorageLocation.SaveStudyXml(_studyXml, out fileCreated);
}
}
public void StudyXmlTest()
{
string directory;
using (var processor = new TestCommandProcessor())
{
var studyXml = new StudyXml(DicomUid.GenerateUid().UID);
var images = SetupMRSeries(2, 5, studyXml.StudyInstanceUid);
directory = Path.Combine(processor.ProcessorContext.TempDirectory, "StudyXmlTest");
DirectoryUtility.DeleteIfExists(directory);
processor.AddCommand(new CreateDirectoryCommand(directory));
foreach (var i in images)
{
string file = Path.Combine(directory, i[DicomTags.SopInstanceUid] + ".dcm");
processor.AddCommand(new SaveDicomFileCommand(file, new DicomFile(file, new DicomAttributeCollection(), i), false));
processor.AddCommand(new InsertInstanceXmlCommand(studyXml, file));
}
Assert.IsTrue(processor.Execute(), processor.FailureReason);
Assert.IsTrue((images.Count * 2 + 1) == processor.TestContext.CommandsExecuted);
foreach (var i in images)
{
Assert.IsTrue(studyXml.Contains(i[DicomTags.SeriesInstanceUid], i[DicomTags.SopInstanceUid]));
}
DirectoryUtility.DeleteIfExists(directory);
}
}
private bool ProcessWorkQueueUid(Model.WorkQueue item, WorkQueueUid sop, StudyXml studyXml, IDicomCodecFactory theCodecFactory)
{
Platform.CheckForNullReference(item, "item");
Platform.CheckForNullReference(sop, "sop");
Platform.CheckForNullReference(studyXml, "studyXml");
if (!studyXml.Contains(sop.SeriesInstanceUid, sop.SopInstanceUid))
{
// Uid was inserted but not in the study xml.
// Auto-recovery might have detect problem with that file and remove it from the study.
// Assume the study xml has been corrected and ignore the uid.
Platform.Log(LogLevel.Warn, "Skipping SOP {0} in series {1}. It is no longer part of the study.", sop.SopInstanceUid, sop.SeriesInstanceUid);
// Delete it out of the queue
DeleteWorkQueueUid(sop);
return true;
}
string basePath = Path.Combine(StorageLocation.GetStudyPath(), sop.SeriesInstanceUid);
basePath = Path.Combine(basePath, sop.SopInstanceUid);
string path;
if (sop.Extension != null)
path = basePath + "." + sop.Extension;
else
path = basePath + ServerPlatform.DicomFileExtension;
try
{
ProcessFile(item, sop, path, studyXml, theCodecFactory);
// WorkQueueUid has been deleted out by the processor
return true;
}
catch (Exception e)
{
if (e.InnerException != null && e.InnerException is DicomCodecUnsupportedSopException)
{
Platform.Log(LogLevel.Warn, e, "Instance not supported for compressor: {0}. Deleting WorkQueue entry for SOP {1}", e.Message, sop.SopInstanceUid);
item.FailureDescription = e.InnerException != null ? e.InnerException.Message : e.Message;
// Delete it out of the queue
DeleteWorkQueueUid(sop);
return false;
}
Platform.Log(LogLevel.Error, e, "Unexpected exception when compressing file: {0} SOP Instance: {1}", path, sop.SopInstanceUid);
item.FailureDescription = e.InnerException != null ? e.InnerException.Message : e.Message;
sop.FailureCount++;
UpdateWorkQueueUid(sop);
return false;
}
}
protected override void OnExecute(CommandProcessor theProcessor)
{
// backup
if (_studyXml.Contains(_seriesUid))
{
Platform.Log(LogLevel.Info, "Removing series {0} from StudyXML for study {1}", _seriesUid, _studyInstanceUid);
_oldSeriesXml = _studyXml[_seriesUid];
if (!_studyXml.RemoveSeries(_seriesUid))
{
throw new ApplicationException(String.Format("Could not remove series {0} from study {1}", _seriesUid, _studyInstanceUid));
}
}
}
public void StudyXmlTestFailure()
{
string directory;
using (var processor = new TestCommandProcessor())
{
var studyXml = new StudyXml(DicomUid.GenerateUid().UID);
var images = SetupMRSeries(2, 5, studyXml.StudyInstanceUid);
directory = Path.Combine(processor.ProcessorContext.TempDirectory, "StudyXmlTest2");
DirectoryUtility.DeleteIfExists(directory);
processor.AddCommand(new CreateDirectoryCommand(directory));
foreach (var i in images)
{
string file = Path.Combine(directory, i[DicomTags.SopInstanceUid] + ".dcm");
processor.AddCommand(new SaveDicomFileCommand(file, new DicomFile(file, new DicomAttributeCollection(), i), false));
processor.AddCommand(new InsertInstanceXmlCommand(studyXml, file));
}
string file2 = Path.Combine(directory, "Test.dcm");
processor.AddCommand(new SaveDicomFileCommand(file2, _dicomFile, true));
processor.AddCommand(new SaveDicomFileCommand(file2, _dicomFile, true));
Assert.IsFalse(processor.Execute(), processor.FailureReason);
Assert.IsTrue((images.Count * 2 + 3) == processor.TestContext.CommandsExecuted);
foreach (var i in images)
{
Assert.IsFalse(studyXml.Contains(i[DicomTags.SeriesInstanceUid], i[DicomTags.SopInstanceUid]));
}
// Directory should be deleted too
Assert.IsFalse(Directory.Exists(directory));
}
}
private void ProcessSeriesLevelDelete(Model.WorkQueue item)
{
// ensure the Study is loaded.
Study study = StorageLocation.Study;
Platform.CheckForNullReference(study, "Study record doesn't exist");
Platform.Log(LogLevel.Info, "Processing Series Level Deletion for Study {0}, A#: {1}",
study.StudyInstanceUid, study.AccessionNumber);
_seriesToDelete = new List <Series>();
bool completed = false;
try
{
// Load the list of Series to be deleted from the WorkQueueUid
LoadUids(item);
// Go through the list of series and add commands
// to delete each of them. It's all or nothing.
using (ServerCommandProcessor processor = new ServerCommandProcessor(String.Format("Deleting Series from study {0}, A#:{1}, Patient: {2}, ID:{3}", study.StudyInstanceUid, study.AccessionNumber, study.PatientsName, study.PatientId)))
{
StudyXml studyXml = StorageLocation.LoadStudyXml();
IDictionary <string, Series> existingSeries = StorageLocation.Study.Series;
// Add commands to delete the folders and update the xml
foreach (WorkQueueUid uid in WorkQueueUidList)
{
// Delete from study XML
if (studyXml.Contains(uid.SeriesInstanceUid))
{
//Note: DeleteDirectoryCommand doesn't throw exception if the folder doesn't exist
var xmlUpdate = new RemoveSeriesFromStudyXml(studyXml, uid.SeriesInstanceUid);
processor.AddCommand(xmlUpdate);
}
// Delete from filesystem
string path = StorageLocation.GetSeriesPath(uid.SeriesInstanceUid);
if (Directory.Exists(path))
{
var delDir = new DeleteDirectoryCommand(path, true);
processor.AddCommand(delDir);
}
}
// flush the updated xml to disk
processor.AddCommand(new SaveXmlCommand(studyXml, StorageLocation));
// Update the db.. NOTE: these commands are executed at the end.
foreach (WorkQueueUid uid in WorkQueueUidList)
{
// Delete from DB
WorkQueueUid queueUid = uid;
Series theSeries = existingSeries[queueUid.SeriesInstanceUid];
if (theSeries != null)
{
_seriesToDelete.Add(theSeries);
var delSeries = new DeleteSeriesFromDBCommand(StorageLocation, theSeries);
processor.AddCommand(delSeries);
delSeries.Executing += DeleteSeriesFromDbExecuting;
}
else
{
// Series doesn't exist
Platform.Log(LogLevel.Info, "Series {0} is invalid or no longer exists", uid.SeriesInstanceUid);
}
// The WorkQueueUid must be cleared before the entry can be removed from the queue
var deleteUid = new DeleteWorkQueueUidCommand(uid);
processor.AddCommand(deleteUid);
// Force a re-archival if necessary
processor.AddCommand(new InsertArchiveQueueCommand(item.ServerPartitionKey, item.StudyStorageKey));
}
if (!processor.Execute())
{
throw new ApplicationException(
String.Format("Error occurred when series from Study {0}, A#: {1}",
study.StudyInstanceUid, study.AccessionNumber), processor.FailureException);
}
else
{
foreach (Series series in _seriesToDelete)
{
OnSeriesDeleted(series);
}
}
}
completed = true;
}
finally
{
if (completed)
{
OnCompleted();
PostProcessing(item, WorkQueueProcessorStatus.Complete, WorkQueueProcessorDatabaseUpdate.ResetQueueState);
}
else
{
PostProcessing(item, WorkQueueProcessorStatus.Pending, WorkQueueProcessorDatabaseUpdate.None);
}
}
}
protected override void ProcessItem(Model.WorkQueue item)
{
Platform.CheckForNullReference(item, "item");
Platform.CheckForNullReference(item.StudyStorageKey, "item.StudyStorageKey");
var context = new StudyProcessorContext(StorageLocation);
// TODO: Should we enforce the patient's name rule?
// If we do, the Study record will have the new patient's name
// but how should we handle the name in the Patient record?
bool enforceNameRules = false;
var processor = new SopInstanceProcessor(context)
{
EnforceNameRules = enforceNameRules
};
var seriesMap = new Dictionary <string, List <string> >();
bool successful = true;
string failureDescription = null;
// The processor stores its state in the Data column
ReadQueueData(item);
if (_queueData.State == null || !_queueData.State.ExecuteAtLeastOnce)
{
// Added for ticket #9673:
// If the study folder does not exist and the study has been archived, trigger a restore and we're done
if (!Directory.Exists(StorageLocation.GetStudyPath()))
{
if (StorageLocation.ArchiveLocations.Count > 0)
{
Platform.Log(LogLevel.Info,
"Reprocessing archived study {0} for Patient {1} (PatientId:{2} A#:{3}) on Partition {4} without study data on the filesystem. Inserting Restore Request.",
Study.StudyInstanceUid, Study.PatientsName, Study.PatientId,
Study.AccessionNumber, ServerPartition.Description);
PostProcessing(item, WorkQueueProcessorStatus.Complete, WorkQueueProcessorDatabaseUpdate.ResetQueueState);
// Post process had to be done first so the study is unlocked so the RestoreRequest can be inserted.
ServerHelper.InsertRestoreRequest(StorageLocation);
RaiseAlert(WorkQueueItem, AlertLevel.Warning,
string.Format(
"Found study {0} for Patient {1} (A#:{2})on Partition {3} without storage folder, restoring study.",
Study.StudyInstanceUid, Study.PatientsName, Study.AccessionNumber, ServerPartition.Description));
return;
}
}
if (Study == null)
{
Platform.Log(LogLevel.Info,
"Reprocessing study {0} on Partition {1}", StorageLocation.StudyInstanceUid,
ServerPartition.Description);
}
else
{
Platform.Log(LogLevel.Info,
"Reprocessing study {0} for Patient {1} (PatientId:{2} A#:{3}) on Partition {4}",
Study.StudyInstanceUid, Study.PatientsName, Study.PatientId,
Study.AccessionNumber, ServerPartition.Description);
}
CleanupDatabase();
}
else
{
if (_queueData.State.Completed)
{
#region SAFE-GUARD CODE: PREVENT INFINITE LOOP
// The processor indicated it had completed reprocessing in previous run. The entry should have been removed and this block of code should never be called.
// However, we have seen ReprocessStudy entries that mysterously contain rows in the WorkQueueUid table.
// The rows prevent the entry from being removed from the database and the ReprocessStudy keeps repeating itself.
// update the state first, increment the CompleteAttemptCount
_queueData.State.ExecuteAtLeastOnce = true;
_queueData.State.Completed = true;
_queueData.State.CompleteAttemptCount++;
SaveState(item, _queueData);
if (_queueData.State.CompleteAttemptCount < 10)
{
// maybe there was db error in previous attempt to remove the entry. Let's try again.
Platform.Log(LogLevel.Info, "Resuming Reprocessing study {0} but it was already completed!!!", StorageLocation.StudyInstanceUid);
PostProcessing(item, WorkQueueProcessorStatus.Complete, WorkQueueProcessorDatabaseUpdate.ResetQueueState);
}
else
{
// we are definitely stuck.
Platform.Log(LogLevel.Error, "ReprocessStudy {0} for study {1} appears stuck. Aborting it.", item.Key, StorageLocation.StudyInstanceUid);
item.FailureDescription = "This entry had completed but could not be removed.";
PostProcessingFailure(item, WorkQueueProcessorFailureType.Fatal);
}
return;
#endregion
}
if (Study == null)
{
Platform.Log(LogLevel.Info,
"Resuming Reprocessing study {0} on Partition {1}", StorageLocation.StudyInstanceUid,
ServerPartition.Description);
}
else
{
Platform.Log(LogLevel.Info,
"Resuming Reprocessing study {0} for Patient {1} (PatientId:{2} A#:{3}) on Partition {4}",
Study.StudyInstanceUid, Study.PatientsName, Study.PatientId,
Study.AccessionNumber, ServerPartition.Description);
}
}
StudyXml studyXml = LoadStudyXml();
int reprocessedCounter = 0;
var removedFiles = new List <FileInfo>();
try
{
// Traverse the directories, process 500 files at a time
FileProcessor.Process(StorageLocation.GetStudyPath(), "*.*",
delegate(string path, out bool cancel)
{
#region Reprocess File
var file = new FileInfo(path);
// ignore all files except those ending ".dcm"
// ignore "bad(0).dcm" files too
if (Regex.IsMatch(file.Name.ToUpper(), "[0-9]+\\.DCM$"))
{
try
{
var dicomFile = new DicomFile(path);
dicomFile.Load(DicomReadOptions.StorePixelDataReferences | DicomReadOptions.Default);
string seriesUid = dicomFile.DataSet[DicomTags.SeriesInstanceUid].GetString(0, string.Empty);
string instanceUid = dicomFile.DataSet[DicomTags.SopInstanceUid].GetString(0, string.Empty);
if (studyXml.Contains(seriesUid, instanceUid))
{
if (!seriesMap.ContainsKey(seriesUid))
{
seriesMap.Add(seriesUid, new List <string>());
}
if (!seriesMap[seriesUid].Contains(instanceUid))
{
seriesMap[seriesUid].Add(instanceUid);
}
else
{
Platform.Log(LogLevel.Warn, "SOP Instance UID in {0} appears more than once in the study.", path);
}
}
else
{
Platform.Log(ServerPlatform.InstanceLogLevel, "Reprocessing SOP {0} for study {1}", instanceUid, StorageLocation.StudyInstanceUid);
string groupId = ServerHelper.GetUidGroup(dicomFile, StorageLocation.ServerPartition, WorkQueueItem.InsertTime);
ProcessingResult result = processor.ProcessFile(groupId, dicomFile, studyXml, true, false, null, null);
switch (result.Status)
{
case ProcessingStatus.Success:
reprocessedCounter++;
if (!seriesMap.ContainsKey(seriesUid))
{
seriesMap.Add(seriesUid, new List <string>());
}
if (!seriesMap[seriesUid].Contains(instanceUid))
{
seriesMap[seriesUid].Add(instanceUid);
}
else
{
Platform.Log(LogLevel.Warn, "SOP Instance UID in {0} appears more than once in the study.", path);
}
break;
case ProcessingStatus.Reconciled:
Platform.Log(LogLevel.Warn, "SOP was unexpectedly reconciled on reprocess SOP {0} for study {1}. It will be removed from the folder.", instanceUid, StorageLocation.StudyInstanceUid);
failureDescription = String.Format("SOP Was reconciled: {0}", instanceUid);
// Added for #10620 (Previously we didn't do anything here)
// Because we are reprocessing files in the study folder, when file needs to be reconciled it is copied to the reconcile folder
// Therefore, we need to delete the one in the study folder. Otherwise, there will be problem when the SIQ entry is reconciled.
// InstanceAlreadyExistsException will also be thrown by the SOpInstanceProcessor if this ReprocessStudy WQI
// resumes and reprocesses the same file again.
// Note: we are sure that the file has been copied to the Reconcile folder and there's no way back.
// We must get rid of this file in the study folder.
FileUtils.Delete(path);
// Special handling: if the file is one which we're supposed to reprocess at the end (see ProcessAdditionalFiles), we must remove the file from the list
if (_additionalFilesToProcess != null && _additionalFilesToProcess.Contains(path))
{
_additionalFilesToProcess.Remove(path);
}
break;
}
}
}
catch (DicomException ex)
{
// TODO : should we fail the reprocess instead? Deleting an dicom file can lead to incomplete study.
removedFiles.Add(file);
Platform.Log(LogLevel.Warn, "Skip reprocessing and delete {0}: Not readable.", path);
FileUtils.Delete(path);
failureDescription = ex.Message;
}
}
else if (!file.Extension.Equals(".xml") && !file.Extension.Equals(".gz"))
{
// not a ".dcm" or header file, delete it
removedFiles.Add(file);
FileUtils.Delete(path);
}
#endregion
cancel = reprocessedCounter >= 500;
}, true);
if (studyXml != null)
{
EnsureConsistentObjectCount(studyXml, seriesMap);
SaveStudyXml(studyXml);
}
// Completed if either all files have been reprocessed
// or no more dicom files left that can be reprocessed.
_completed = reprocessedCounter == 0;
}
catch (Exception e)
{
successful = false;
failureDescription = e.Message;
Platform.Log(LogLevel.Error, e, "Unexpected exception when reprocessing study: {0}", StorageLocation.StudyInstanceUid);
Platform.Log(LogLevel.Error, "Study may be in invalid unprocessed state. Study location: {0}", StorageLocation.GetStudyPath());
throw;
}
finally
{
LogRemovedFiles(removedFiles);
// Update the state
_queueData.State.ExecuteAtLeastOnce = true;
_queueData.State.Completed = _completed;
_queueData.State.CompleteAttemptCount++;
SaveState(item, _queueData);
if (!successful)
{
FailQueueItem(item, failureDescription);
}
else
{
if (!_completed)
{
// Put it back to Pending
PostProcessing(item, WorkQueueProcessorStatus.Pending, WorkQueueProcessorDatabaseUpdate.None);
}
else
{
LogHistory();
// Run Study / Series Rules Engine.
var engine = new StudyRulesEngine(StorageLocation, ServerPartition);
engine.Apply(ServerRuleApplyTimeEnum.StudyProcessed);
// Log the FilesystemQueue related entries
StorageLocation.LogFilesystemQueue();
PostProcessing(item, WorkQueueProcessorStatus.Complete, WorkQueueProcessorDatabaseUpdate.ResetQueueState);
Platform.Log(LogLevel.Info, "Completed reprocessing of study {0} on partition {1}", StorageLocation.StudyInstanceUid, ServerPartition.Description);
}
}
}
}
private bool ProcessWorkQueueUid(Model.WorkQueue item, WorkQueueUid sop, StudyXml studyXml, IDicomCodecFactory theCodecFactory)
{
Platform.CheckForNullReference(item, "item");
Platform.CheckForNullReference(sop, "sop");
Platform.CheckForNullReference(studyXml, "studyXml");
if (!studyXml.Contains(sop.SeriesInstanceUid, sop.SopInstanceUid))
{
// Uid was inserted but not in the study xml.
// Auto-recovery might have detect problem with that file and remove it from the study.
// Assume the study xml has been corrected and ignore the uid.
Platform.Log(LogLevel.Warn, "Skipping SOP {0} in series {1}. It is no longer part of the study.", sop.SopInstanceUid, sop.SeriesInstanceUid);
// Delete it out of the queue
DeleteWorkQueueUid(sop);
return(true);
}
string basePath = Path.Combine(StorageLocation.GetStudyPath(), sop.SeriesInstanceUid);
basePath = Path.Combine(basePath, sop.SopInstanceUid);
string path;
if (sop.Extension != null)
{
path = basePath + "." + sop.Extension;
}
else
{
path = basePath + ServerPlatform.DicomFileExtension;
}
try
{
ProcessFile(item, sop, path, studyXml, theCodecFactory);
// WorkQueueUid has been deleted out by the processor
return(true);
}
catch (Exception e)
{
if (e.InnerException != null && e.InnerException is DicomCodecUnsupportedSopException)
{
Platform.Log(LogLevel.Warn, e, "Instance not supported for compressor: {0}. Deleting WorkQueue entry for SOP {1}", e.Message, sop.SopInstanceUid);
item.FailureDescription = e.InnerException != null ? e.InnerException.Message : e.Message;
// Delete it out of the queue
DeleteWorkQueueUid(sop);
return(false);
}
Platform.Log(LogLevel.Error, e, "Unexpected exception when compressing file: {0} SOP Instance: {1}", path, sop.SopInstanceUid);
item.FailureDescription = e.InnerException != null ? e.InnerException.Message : e.Message;
sop.FailureCount++;
UpdateWorkQueueUid(sop);
return(false);
}
}
