protected override void ProcessFile(Model.WorkQueueUid queueUid, DicomFile file, ClearCanvas.Dicom.Utilities.Xml.StudyXml stream, bool compare)
{
Platform.CheckFalse(compare, "compare");
SopInstanceProcessor processor = new SopInstanceProcessor(Context);
FileInfo fileInfo = new FileInfo(file.Filename);
long fileSize = fileInfo.Length;
processor.InstanceStats.FileSize = (ulong)fileSize;
string sopInstanceUid = file.DataSet[DicomTags.SopInstanceUid].GetString(0, "File:" + fileInfo.Name);
processor.InstanceStats.Description = sopInstanceUid;
if (Study != null)
{
StudyComparer comparer = new StudyComparer();
DifferenceCollection list = comparer.Compare(file, Study, ServerPartition.GetComparisonOptions());
if (list != null && list.Count > 0)
{
Platform.Log(LogLevel.Warn, "Dicom file contains information inconsistent with the study in the system");
}
}
string groupID = ServerHelper.GetUidGroup(file, StorageLocation.ServerPartition, WorkQueueItem.InsertTime);
processor.ProcessFile(groupID, file, stream, false, false, null, null);
Statistics.StudyInstanceUid = StorageLocation.StudyInstanceUid;
if (String.IsNullOrEmpty(processor.Modality) == false)
Statistics.Modality = processor.Modality;
// Update the statistics
Statistics.NumInstances++;
}
protected override void ProcessItem(Model.WorkQueue item)
{
Platform.CheckForNullReference(item, "item");
Platform.CheckForNullReference(item.StudyStorageKey, "item.StudyStorageKey");
var context = new StudyProcessorContext(StorageLocation, WorkQueueItem);
// TODO: Should we enforce the patient's name rule?
// If we do, the Study record will have the new patient's name
// but how should we handle the name in the Patient record?
const bool enforceNameRules = false;
var processor = new SopInstanceProcessor(context) { EnforceNameRules = enforceNameRules};
var seriesMap = new Dictionary<string, List<string>>();
bool successful = true;
string failureDescription = null;
// The processor stores its state in the Data column
ReadQueueData(item);
if (_queueData.State == null || !_queueData.State.ExecuteAtLeastOnce)
{
// Added for ticket #9673:
// If the study folder does not exist and the study has been archived, trigger a restore and we're done
if (!Directory.Exists(StorageLocation.GetStudyPath()))
{
if (StorageLocation.ArchiveLocations.Count > 0)
{
Platform.Log(LogLevel.Info,
"Reprocessing archived study {0} for Patient {1} (PatientId:{2} A#:{3}) on Partition {4} without study data on the filesystem. Inserting Restore Request.",
Study.StudyInstanceUid, Study.PatientsName, Study.PatientId,
Study.AccessionNumber, ServerPartition.Description);
PostProcessing(item, WorkQueueProcessorStatus.Complete, WorkQueueProcessorDatabaseUpdate.ResetQueueState);
// Post process had to be done first so the study is unlocked so the RestoreRequest can be inserted.
ServerHelper.InsertRestoreRequest(StorageLocation);
RaiseAlert(WorkQueueItem, AlertLevel.Warning,
string.Format(
"Found study {0} for Patient {1} (A#:{2})on Partition {3} without storage folder, restoring study.",
Study.StudyInstanceUid, Study.PatientsName, Study.AccessionNumber, ServerPartition.Description));
return;
}
}
if (Study == null)
Platform.Log(LogLevel.Info,
"Reprocessing study {0} on Partition {1}", StorageLocation.StudyInstanceUid,
ServerPartition.Description);
else
Platform.Log(LogLevel.Info,
"Reprocessing study {0} for Patient {1} (PatientId:{2} A#:{3}) on Partition {4}",
Study.StudyInstanceUid, Study.PatientsName, Study.PatientId,
Study.AccessionNumber, ServerPartition.Description);
CleanupDatabase();
}
else
{
if (_queueData.State.Completed)
{
#region SAFE-GUARD CODE: PREVENT INFINITE LOOP
// The processor indicated it had completed reprocessing in previous run. The entry should have been removed and this block of code should never be called.
// However, we have seen ReprocessStudy entries that mysterously contain rows in the WorkQueueUid table.
// The rows prevent the entry from being removed from the database and the ReprocessStudy keeps repeating itself.
// update the state first, increment the CompleteAttemptCount
_queueData.State.ExecuteAtLeastOnce = true;
_queueData.State.Completed = true;
_queueData.State.CompleteAttemptCount++;
SaveState(item, _queueData);
if (_queueData.State.CompleteAttemptCount < 10)
{
// maybe there was db error in previous attempt to remove the entry. Let's try again.
Platform.Log(LogLevel.Info, "Resuming Reprocessing study {0} but it was already completed!!!", StorageLocation.StudyInstanceUid);
PostProcessing(item, WorkQueueProcessorStatus.Complete, WorkQueueProcessorDatabaseUpdate.ResetQueueState);
}
else
{
// we are definitely stuck.
Platform.Log(LogLevel.Error, "ReprocessStudy {0} for study {1} appears stuck. Aborting it.", item.Key, StorageLocation.StudyInstanceUid);
item.FailureDescription = "This entry had completed but could not be removed.";
PostProcessingFailure(item, WorkQueueProcessorFailureType.Fatal);
}
return;
#endregion
}
if (Study == null)
Platform.Log(LogLevel.Info,
"Resuming Reprocessing study {0} on Partition {1}", StorageLocation.StudyInstanceUid,
ServerPartition.Description);
else
Platform.Log(LogLevel.Info,
"Resuming Reprocessing study {0} for Patient {1} (PatientId:{2} A#:{3}) on Partition {4}",
Study.StudyInstanceUid, Study.PatientsName, Study.PatientId,
Study.AccessionNumber, ServerPartition.Description);
}
StudyXml studyXml = LoadStudyXml();
int reprocessedCounter = 0;
var removedFiles = new List<FileInfo>();
try
{
// Traverse the directories, process 500 files at a time
FileProcessor.Process(StorageLocation.GetStudyPath(), "*.*",
delegate(string path, out bool cancel)
{
#region Reprocess File
var file = new FileInfo(path);
// ignore all files except those ending ".dcm"
// ignore "bad(0).dcm" files too
if (Regex.IsMatch(file.Name.ToUpper(), "[0-9]+\\.DCM$"))
{
try
{
var dicomFile = new DicomFile(path);
dicomFile.Load(DicomReadOptions.StorePixelDataReferences | DicomReadOptions.Default);
string seriesUid = dicomFile.DataSet[DicomTags.SeriesInstanceUid].GetString(0, string.Empty);
string instanceUid =dicomFile.DataSet[DicomTags.SopInstanceUid].GetString(0,string.Empty);
if (studyXml.Contains(seriesUid, instanceUid))
{
if (!seriesMap.ContainsKey(seriesUid))
{
seriesMap.Add(seriesUid, new List<string>());
}
if (!seriesMap[seriesUid].Contains(instanceUid))
seriesMap[seriesUid].Add(instanceUid);
else
{
Platform.Log(LogLevel.Warn, "SOP Instance UID in {0} appears more than once in the study.", path);
}
}
else
{
Platform.Log(ServerPlatform.InstanceLogLevel, "Reprocessing SOP {0} for study {1}",instanceUid, StorageLocation.StudyInstanceUid);
string groupId = ServerHelper.GetUidGroup(dicomFile, StorageLocation.ServerPartition, WorkQueueItem.InsertTime);
ProcessingResult result = processor.ProcessFile(groupId, dicomFile, studyXml, true, false, null, null, SopInstanceProcessorSopType.ReprocessedSop);
switch (result.Status)
{
case ProcessingStatus.Success:
reprocessedCounter++;
if (!seriesMap.ContainsKey(seriesUid))
{
seriesMap.Add(seriesUid, new List<string>());
}
if (!seriesMap[seriesUid].Contains(instanceUid))
seriesMap[seriesUid].Add(instanceUid);
else
{
Platform.Log(LogLevel.Warn, "SOP Instance UID in {0} appears more than once in the study.", path);
}
break;
case ProcessingStatus.Reconciled:
Platform.Log(LogLevel.Warn, "SOP was unexpectedly reconciled on reprocess SOP {0} for study {1}. It will be removed from the folder.", instanceUid, StorageLocation.StudyInstanceUid);
failureDescription = String.Format("SOP Was reconciled: {0}", instanceUid);
// Added for #10620 (Previously we didn't do anything here)
// Because we are reprocessing files in the study folder, when file needs to be reconciled it is copied to the reconcile folder
// Therefore, we need to delete the one in the study folder. Otherwise, there will be problem when the SIQ entry is reconciled.
// InstanceAlreadyExistsException will also be thrown by the SOpInstanceProcessor if this ReprocessStudy WQI
// resumes and reprocesses the same file again.
// Note: we are sure that the file has been copied to the Reconcile folder and there's no way back.
// We must get rid of this file in the study folder.
FileUtils.Delete(path);
// Special handling: if the file is one which we're supposed to reprocess at the end (see ProcessAdditionalFiles), we must remove the file from the list
if (_additionalFilesToProcess != null && _additionalFilesToProcess.Contains(path))
{
_additionalFilesToProcess.Remove(path);
}
break;
}
}
}
catch (DicomException ex)
{
// TODO : should we fail the reprocess instead? Deleting an dicom file can lead to incomplete study.
removedFiles.Add(file);
Platform.Log(LogLevel.Warn, "Skip reprocessing and delete {0}: Not readable.", path);
FileUtils.Delete(path);
failureDescription = ex.Message;
}
}
else if (!file.Extension.Equals(".xml") && !file.Extension.Equals(".gz"))
{
// not a ".dcm" or header file, delete it
removedFiles.Add(file);
FileUtils.Delete(path);
}
#endregion
if (reprocessedCounter>0 && reprocessedCounter % 200 == 0)
{
Platform.Log(LogLevel.Info, "Reprocessed {0} files for study {1}", reprocessedCounter, StorageLocation.StudyInstanceUid);
}
cancel = reprocessedCounter >= 5000;
}, true);
if (studyXml != null)
{
EnsureConsistentObjectCount(studyXml, seriesMap);
SaveStudyXml(studyXml);
}
// Completed if either all files have been reprocessed
// or no more dicom files left that can be reprocessed.
_completed = reprocessedCounter == 0;
}
catch (Exception e)
{
successful = false;
failureDescription = e.Message;
Platform.Log(LogLevel.Error, e, "Unexpected exception when reprocessing study: {0}", StorageLocation.StudyInstanceUid);
Platform.Log(LogLevel.Error, "Study may be in invalid unprocessed state. Study location: {0}", StorageLocation.GetStudyPath());
throw;
}
finally
{
LogRemovedFiles(removedFiles);
// Update the state
_queueData.State.ExecuteAtLeastOnce = true;
_queueData.State.Completed = _completed;
_queueData.State.CompleteAttemptCount++;
SaveState(item, _queueData);
if (!successful)
{
FailQueueItem(item, failureDescription);
}
else
{
if (!_completed)
{
// Put it back to Pending
PostProcessing(item, WorkQueueProcessorStatus.Pending, WorkQueueProcessorDatabaseUpdate.None);
}
else
{
LogHistory();
// Run Study / Series Rules Engine.
var engine = new StudyRulesEngine(StorageLocation, ServerPartition);
engine.Apply(ServerRuleApplyTimeEnum.StudyProcessed);
// Log the FilesystemQueue related entries
StorageLocation.LogFilesystemQueue();
PostProcessing(item, WorkQueueProcessorStatus.Complete, WorkQueueProcessorDatabaseUpdate.ResetQueueState);
Platform.Log(LogLevel.Info, "Completed reprocessing of study {0} on partition {1}", StorageLocation.StudyInstanceUid, ServerPartition.Description);
}
}
}
}
/// <summary>
/// ProcessSavedFile a specific DICOM file related to a <see cref="WorkQueue"/> request.
/// </summary>
/// <param name="queueUid"></param>
/// <param name="stream">The <see cref="StudyXml"/> file to update with information from the file.</param>
/// <param name="file">The file being processed.</param>
/// <param name="compare">Indicates whether to compare the DICOM file against the study in the system.</param>
protected virtual void ProcessFile(WorkQueueUid queueUid, DicomFile file, StudyXml stream, bool compare)
{
var processor = new SopInstanceProcessor(Context) {EnforceNameRules = true};
var fileInfo = new FileInfo(file.Filename);
long fileSize = fileInfo.Length;
processor.InstanceStats.FileLoadTime.Start();
processor.InstanceStats.FileLoadTime.End();
processor.InstanceStats.FileSize = (ulong)fileSize;
string sopInstanceUid = file.DataSet[DicomTags.SopInstanceUid].GetString(0, "File:" + fileInfo.Name);
processor.InstanceStats.Description = sopInstanceUid;
string group = queueUid.GroupID ?? ServerHelper.GetUidGroup(file, ServerPartition, WorkQueueItem.InsertTime);
ProcessingResult result = processor.ProcessFile(group, file, stream, compare, true, queueUid, null, SopInstanceProcessorSopType.NewSop);
if (result.Status == ProcessingStatus.Reconciled)
{
// file has been saved by SopInstanceProcessor in another place for reconcilation
// Note: SopInstanceProcessor has removed the WorkQueueUid so we
// only need to delete the file here.
FileUtils.Delete(fileInfo.FullName);
}
Statistics.StudyInstanceUid = StorageLocation.StudyInstanceUid;
if (String.IsNullOrEmpty(processor.Modality) == false)
Statistics.Modality = processor.Modality;
// Update the statistics
Statistics.NumInstances++;
Statistics.AddSubStats(processor.InstanceStats);
}
private void AddDuplicateToStudy(DicomFile duplicateDicomFile, WorkQueueUid uid, ProcessDuplicateAction action)
{
var context = new StudyProcessorContext(StorageLocation, WorkQueueItem);
var sopInstanceProcessor = new SopInstanceProcessor(context) { EnforceNameRules = true };
string group = uid.GroupID ?? ServerHelper.GetUidGroup(duplicateDicomFile, ServerPartition, WorkQueueItem.InsertTime);
StudyXml studyXml = StorageLocation.LoadStudyXml();
int originalInstanceCount = studyXml.NumberOfStudyRelatedInstances;
bool compare = action != ProcessDuplicateAction.OverwriteAsIs;
// NOTE: "compare" has no effect for OverwriteUseExisting or OverwriteUseDuplicate
// because in both cases, the study and the duplicates are modified to be the same.
ProcessingResult result = sopInstanceProcessor.ProcessFile(group, duplicateDicomFile, studyXml, compare, true, uid, duplicateDicomFile.Filename, SopInstanceProcessorSopType.UpdatedSop);
if (result.Status == ProcessingStatus.Reconciled)
{
throw new ApplicationException("Unexpected status of Reconciled image in duplicate handling!");
}
Debug.Assert(studyXml.NumberOfStudyRelatedInstances == originalInstanceCount + 1);
Debug.Assert(File.Exists(StorageLocation.GetSopInstancePath(uid.SeriesInstanceUid, uid.SopInstanceUid)));
}
