public SopInstanceProcessor(StudyProcessorContext context)
{
Platform.CheckForNullReference(context, "context");
_context = context;
_patientNameRules = new PatientNameRules(context.Study);
}
protected override void ProcessItem(Model.WorkQueue item)
{
Platform.CheckForNullReference(item, "item");
Platform.CheckForNullReference(item.StudyStorageKey, "item.StudyStorageKey");
var context = new StudyProcessorContext(StorageLocation, WorkQueueItem);
// TODO: Should we enforce the patient's name rule?
// If we do, the Study record will have the new patient's name
// but how should we handle the name in the Patient record?
const bool enforceNameRules = false;
var processor = new SopInstanceProcessor(context) { EnforceNameRules = enforceNameRules};
var seriesMap = new Dictionary<string, List<string>>();
bool successful = true;
string failureDescription = null;
// The processor stores its state in the Data column
ReadQueueData(item);
if (_queueData.State == null || !_queueData.State.ExecuteAtLeastOnce)
{
// Added for ticket #9673:
// If the study folder does not exist and the study has been archived, trigger a restore and we're done
if (!Directory.Exists(StorageLocation.GetStudyPath()))
{
if (StorageLocation.ArchiveLocations.Count > 0)
{
Platform.Log(LogLevel.Info,
"Reprocessing archived study {0} for Patient {1} (PatientId:{2} A#:{3}) on Partition {4} without study data on the filesystem. Inserting Restore Request.",
Study.StudyInstanceUid, Study.PatientsName, Study.PatientId,
Study.AccessionNumber, ServerPartition.Description);
PostProcessing(item, WorkQueueProcessorStatus.Complete, WorkQueueProcessorDatabaseUpdate.ResetQueueState);
// Post process had to be done first so the study is unlocked so the RestoreRequest can be inserted.
ServerHelper.InsertRestoreRequest(StorageLocation);
RaiseAlert(WorkQueueItem, AlertLevel.Warning,
string.Format(
"Found study {0} for Patient {1} (A#:{2})on Partition {3} without storage folder, restoring study.",
Study.StudyInstanceUid, Study.PatientsName, Study.AccessionNumber, ServerPartition.Description));
return;
}
}
if (Study == null)
Platform.Log(LogLevel.Info,
"Reprocessing study {0} on Partition {1}", StorageLocation.StudyInstanceUid,
ServerPartition.Description);
else
Platform.Log(LogLevel.Info,
"Reprocessing study {0} for Patient {1} (PatientId:{2} A#:{3}) on Partition {4}",
Study.StudyInstanceUid, Study.PatientsName, Study.PatientId,
Study.AccessionNumber, ServerPartition.Description);
CleanupDatabase();
}
else
{
if (_queueData.State.Completed)
{
#region SAFE-GUARD CODE: PREVENT INFINITE LOOP
// The processor indicated it had completed reprocessing in previous run. The entry should have been removed and this block of code should never be called.
// However, we have seen ReprocessStudy entries that mysterously contain rows in the WorkQueueUid table.
// The rows prevent the entry from being removed from the database and the ReprocessStudy keeps repeating itself.
// update the state first, increment the CompleteAttemptCount
_queueData.State.ExecuteAtLeastOnce = true;
_queueData.State.Completed = true;
_queueData.State.CompleteAttemptCount++;
SaveState(item, _queueData);
if (_queueData.State.CompleteAttemptCount < 10)
{
// maybe there was db error in previous attempt to remove the entry. Let's try again.
Platform.Log(LogLevel.Info, "Resuming Reprocessing study {0} but it was already completed!!!", StorageLocation.StudyInstanceUid);
PostProcessing(item, WorkQueueProcessorStatus.Complete, WorkQueueProcessorDatabaseUpdate.ResetQueueState);
}
else
{
// we are definitely stuck.
Platform.Log(LogLevel.Error, "ReprocessStudy {0} for study {1} appears stuck. Aborting it.", item.Key, StorageLocation.StudyInstanceUid);
item.FailureDescription = "This entry had completed but could not be removed.";
PostProcessingFailure(item, WorkQueueProcessorFailureType.Fatal);
}
return;
#endregion
}
if (Study == null)
Platform.Log(LogLevel.Info,
"Resuming Reprocessing study {0} on Partition {1}", StorageLocation.StudyInstanceUid,
ServerPartition.Description);
else
Platform.Log(LogLevel.Info,
"Resuming Reprocessing study {0} for Patient {1} (PatientId:{2} A#:{3}) on Partition {4}",
Study.StudyInstanceUid, Study.PatientsName, Study.PatientId,
Study.AccessionNumber, ServerPartition.Description);
}
StudyXml studyXml = LoadStudyXml();
int reprocessedCounter = 0;
var removedFiles = new List<FileInfo>();
try
{
// Traverse the directories, process 500 files at a time
FileProcessor.Process(StorageLocation.GetStudyPath(), "*.*",
delegate(string path, out bool cancel)
{
#region Reprocess File
var file = new FileInfo(path);
// ignore all files except those ending ".dcm"
// ignore "bad(0).dcm" files too
if (Regex.IsMatch(file.Name.ToUpper(), "[0-9]+\\.DCM$"))
{
try
{
var dicomFile = new DicomFile(path);
dicomFile.Load(DicomReadOptions.StorePixelDataReferences | DicomReadOptions.Default);
string seriesUid = dicomFile.DataSet[DicomTags.SeriesInstanceUid].GetString(0, string.Empty);
string instanceUid =dicomFile.DataSet[DicomTags.SopInstanceUid].GetString(0,string.Empty);
if (studyXml.Contains(seriesUid, instanceUid))
{
if (!seriesMap.ContainsKey(seriesUid))
{
seriesMap.Add(seriesUid, new List<string>());
}
if (!seriesMap[seriesUid].Contains(instanceUid))
seriesMap[seriesUid].Add(instanceUid);
else
{
Platform.Log(LogLevel.Warn, "SOP Instance UID in {0} appears more than once in the study.", path);
}
}
else
{
Platform.Log(ServerPlatform.InstanceLogLevel, "Reprocessing SOP {0} for study {1}",instanceUid, StorageLocation.StudyInstanceUid);
string groupId = ServerHelper.GetUidGroup(dicomFile, StorageLocation.ServerPartition, WorkQueueItem.InsertTime);
ProcessingResult result = processor.ProcessFile(groupId, dicomFile, studyXml, true, false, null, null, SopInstanceProcessorSopType.ReprocessedSop);
switch (result.Status)
{
case ProcessingStatus.Success:
reprocessedCounter++;
if (!seriesMap.ContainsKey(seriesUid))
{
seriesMap.Add(seriesUid, new List<string>());
}
if (!seriesMap[seriesUid].Contains(instanceUid))
seriesMap[seriesUid].Add(instanceUid);
else
{
Platform.Log(LogLevel.Warn, "SOP Instance UID in {0} appears more than once in the study.", path);
}
break;
case ProcessingStatus.Reconciled:
Platform.Log(LogLevel.Warn, "SOP was unexpectedly reconciled on reprocess SOP {0} for study {1}. It will be removed from the folder.", instanceUid, StorageLocation.StudyInstanceUid);
failureDescription = String.Format("SOP Was reconciled: {0}", instanceUid);
// Added for #10620 (Previously we didn't do anything here)
// Because we are reprocessing files in the study folder, when file needs to be reconciled it is copied to the reconcile folder
// Therefore, we need to delete the one in the study folder. Otherwise, there will be problem when the SIQ entry is reconciled.
// InstanceAlreadyExistsException will also be thrown by the SOpInstanceProcessor if this ReprocessStudy WQI
// resumes and reprocesses the same file again.
// Note: we are sure that the file has been copied to the Reconcile folder and there's no way back.
// We must get rid of this file in the study folder.
FileUtils.Delete(path);
// Special handling: if the file is one which we're supposed to reprocess at the end (see ProcessAdditionalFiles), we must remove the file from the list
if (_additionalFilesToProcess != null && _additionalFilesToProcess.Contains(path))
{
_additionalFilesToProcess.Remove(path);
}
break;
}
}
}
catch (DicomException ex)
{
// TODO : should we fail the reprocess instead? Deleting an dicom file can lead to incomplete study.
removedFiles.Add(file);
Platform.Log(LogLevel.Warn, "Skip reprocessing and delete {0}: Not readable.", path);
FileUtils.Delete(path);
failureDescription = ex.Message;
}
}
else if (!file.Extension.Equals(".xml") && !file.Extension.Equals(".gz"))
{
// not a ".dcm" or header file, delete it
removedFiles.Add(file);
FileUtils.Delete(path);
}
#endregion
if (reprocessedCounter>0 && reprocessedCounter % 200 == 0)
{
Platform.Log(LogLevel.Info, "Reprocessed {0} files for study {1}", reprocessedCounter, StorageLocation.StudyInstanceUid);
}
cancel = reprocessedCounter >= 5000;
}, true);
if (studyXml != null)
{
EnsureConsistentObjectCount(studyXml, seriesMap);
SaveStudyXml(studyXml);
}
// Completed if either all files have been reprocessed
// or no more dicom files left that can be reprocessed.
_completed = reprocessedCounter == 0;
}
catch (Exception e)
{
successful = false;
failureDescription = e.Message;
Platform.Log(LogLevel.Error, e, "Unexpected exception when reprocessing study: {0}", StorageLocation.StudyInstanceUid);
Platform.Log(LogLevel.Error, "Study may be in invalid unprocessed state. Study location: {0}", StorageLocation.GetStudyPath());
throw;
}
finally
{
LogRemovedFiles(removedFiles);
// Update the state
_queueData.State.ExecuteAtLeastOnce = true;
_queueData.State.Completed = _completed;
_queueData.State.CompleteAttemptCount++;
SaveState(item, _queueData);
if (!successful)
{
FailQueueItem(item, failureDescription);
}
else
{
if (!_completed)
{
// Put it back to Pending
PostProcessing(item, WorkQueueProcessorStatus.Pending, WorkQueueProcessorDatabaseUpdate.None);
}
else
{
LogHistory();
// Run Study / Series Rules Engine.
var engine = new StudyRulesEngine(StorageLocation, ServerPartition);
engine.Apply(ServerRuleApplyTimeEnum.StudyProcessed);
// Log the FilesystemQueue related entries
StorageLocation.LogFilesystemQueue();
PostProcessing(item, WorkQueueProcessorStatus.Complete, WorkQueueProcessorDatabaseUpdate.ResetQueueState);
Platform.Log(LogLevel.Info, "Completed reprocessing of study {0} on partition {1}", StorageLocation.StudyInstanceUid, ServerPartition.Description);
}
}
}
}
protected override void ProcessItem(Model.WorkQueue item)
{
Platform.CheckForNullReference(item, "item");
Platform.CheckForNullReference(StorageLocation, "StorageLocation");
// Verify the study is not lossy online and lossless in the archive.
// This could happen if the images were received WHILE the study was being lossy compressed.
// The study state would not be set until the compression was completed or partially completed.
CheckIfStudyIsLossy();
Statistics.TotalProcessTime.Start();
bool successful;
bool idle = false;
//Load the specific UIDs that need to be processed.
LoadUids(item);
int totalUidCount = WorkQueueUidList.Count;
if (totalUidCount == 0)
{
successful = true;
idle = true;
}
else
{
try
{
Context = new StudyProcessorContext(StorageLocation, WorkQueueItem);
// Load the rules engine
_sopProcessedRulesEngine = new ServerRulesEngine(ServerRuleApplyTimeEnum.SopProcessed, item.ServerPartitionKey);
_sopProcessedRulesEngine.AddOmittedType(ServerRuleTypeEnum.SopCompress);
_sopProcessedRulesEngine.Load();
Statistics.SopProcessedEngineLoadTime.Add(_sopProcessedRulesEngine.Statistics.LoadTime);
Context.SopProcessedRulesEngine = _sopProcessedRulesEngine;
if (Study != null)
{
Platform.Log(LogLevel.Info, "Processing study {0} for Patient {1} (PatientId:{2} A#:{3}), {4} objects",
Study.StudyInstanceUid, Study.PatientsName, Study.PatientId,
Study.AccessionNumber, WorkQueueUidList.Count);
}
else
{
Platform.Log(LogLevel.Info, "Processing study {0}, {1} objects",
StorageLocation.StudyInstanceUid, WorkQueueUidList.Count);
}
// ProcessSavedFile the images in the list
successful = ProcessUidList(item) > 0;
}
catch (StudyIsNearlineException ex)
{
// delay until the target is restored
// NOTE: If the study could not be restored after certain period of time, this entry will be failed.
if (ex.RestoreRequested)
{
PostponeItem(string.Format("Unable to auto-reconcile at this time: the target study {0} is not online yet. Restore has been requested.", ex.StudyInstanceUid));
return;
}
// fail right away
FailQueueItem(item, string.Format("Unable to auto-reconcile at this time: the target study {0} is not nearline and could not be restored.", ex.StudyInstanceUid));
return;
}
}
Statistics.TotalProcessTime.End();
if (successful)
{
if (idle && item.ExpirationTime <= Platform.Time)
{
// Run Study / Series Rules Engine.
var engine = new StudyRulesEngine(StorageLocation, ServerPartition);
engine.Apply(ServerRuleApplyTimeEnum.StudyProcessed);
// Log the FilesystemQueue related entries
StorageLocation.LogFilesystemQueue();
// Delete the queue entry.
PostProcessing(item,
WorkQueueProcessorStatus.Complete,
WorkQueueProcessorDatabaseUpdate.ResetQueueState);
}
else if (idle)
PostProcessing(item,
WorkQueueProcessorStatus.IdleNoDelete, // Don't delete, so we ensure the rules engine is run later.
WorkQueueProcessorDatabaseUpdate.ResetQueueState);
else
PostProcessing(item,
WorkQueueProcessorStatus.Pending,
WorkQueueProcessorDatabaseUpdate.ResetQueueState);
}
else
{
bool allFailedDuplicate = CollectionUtils.TrueForAll(WorkQueueUidList, uid => uid.Duplicate && uid.Failed);
if (allFailedDuplicate)
{
Platform.Log(LogLevel.Error, "All entries are duplicates");
PostProcessingFailure(item, WorkQueueProcessorFailureType.Fatal);
return;
}
PostProcessingFailure(item, WorkQueueProcessorFailureType.NonFatal);
}
}
public SopInstanceProcessor(StudyProcessorContext context)
{
Platform.CheckForNullReference(context, "context");
_context = context;
_patientNameRules = new PatientNameRules(context.Study);
}
private void AddDuplicateToStudy(DicomFile duplicateDicomFile, WorkQueueUid uid, ProcessDuplicateAction action)
{
var context = new StudyProcessorContext(StorageLocation, WorkQueueItem);
var sopInstanceProcessor = new SopInstanceProcessor(context) { EnforceNameRules = true };
string group = uid.GroupID ?? ServerHelper.GetUidGroup(duplicateDicomFile, ServerPartition, WorkQueueItem.InsertTime);
StudyXml studyXml = StorageLocation.LoadStudyXml();
int originalInstanceCount = studyXml.NumberOfStudyRelatedInstances;
bool compare = action != ProcessDuplicateAction.OverwriteAsIs;
// NOTE: "compare" has no effect for OverwriteUseExisting or OverwriteUseDuplicate
// because in both cases, the study and the duplicates are modified to be the same.
ProcessingResult result = sopInstanceProcessor.ProcessFile(group, duplicateDicomFile, studyXml, compare, true, uid, duplicateDicomFile.Filename, SopInstanceProcessorSopType.UpdatedSop);
if (result.Status == ProcessingStatus.Reconciled)
{
throw new ApplicationException("Unexpected status of Reconciled image in duplicate handling!");
}
Debug.Assert(studyXml.NumberOfStudyRelatedInstances == originalInstanceCount + 1);
Debug.Assert(File.Exists(StorageLocation.GetSopInstancePath(uid.SeriesInstanceUid, uid.SopInstanceUid)));
}