protected override void OnExecute(CommandProcessor theProcessor) { long fileSize = 0; if (File.Exists(_file.Filename)) { var finfo = new FileInfo(_file.Filename); fileSize = finfo.Length; } String seriesInstanceUid = _file.DataSet[DicomTags.SeriesInstanceUid].GetString(0, string.Empty); String sopInstanceUid = _file.DataSet[DicomTags.SopInstanceUid].GetString(0, string.Empty); if (_studyXml.Contains(seriesInstanceUid, sopInstanceUid)) { _duplicate = true; } // Setup the insert parameters if (false == _studyXml.AddFile(_file, fileSize, _settings)) { Platform.Log(LogLevel.Error, "Unexpected error adding SOP to XML Study Descriptor for file {0}", _file.Filename); throw new ApplicationException("Unexpected error adding SOP to XML Study Descriptor for SOP: " + _file.MediaStorageSopInstanceUid); } if (_writeFile) { // Write it back out. We flush it out with every added image so that if a failure happens, // we can recover properly. bool fileCreated; _studyStorageLocation.SaveStudyXml(_studyXml, out fileCreated); } }
public void StudyXmlTest() { string directory; using (var processor = new TestCommandProcessor()) { var studyXml = new StudyXml(DicomUid.GenerateUid().UID); var images = SetupMRSeries(2, 5, studyXml.StudyInstanceUid); directory = Path.Combine(processor.ProcessorContext.TempDirectory, "StudyXmlTest"); DirectoryUtility.DeleteIfExists(directory); processor.AddCommand(new CreateDirectoryCommand(directory)); foreach (var i in images) { string file = Path.Combine(directory, i[DicomTags.SopInstanceUid] + ".dcm"); processor.AddCommand(new SaveDicomFileCommand(file, new DicomFile(file, new DicomAttributeCollection(), i), false)); processor.AddCommand(new InsertInstanceXmlCommand(studyXml, file)); } Assert.IsTrue(processor.Execute(), processor.FailureReason); Assert.IsTrue((images.Count * 2 + 1) == processor.TestContext.CommandsExecuted); foreach (var i in images) { Assert.IsTrue(studyXml.Contains(i[DicomTags.SeriesInstanceUid], i[DicomTags.SopInstanceUid])); } DirectoryUtility.DeleteIfExists(directory); } }
private bool ProcessWorkQueueUid(Model.WorkQueue item, WorkQueueUid sop, StudyXml studyXml, IDicomCodecFactory theCodecFactory) { Platform.CheckForNullReference(item, "item"); Platform.CheckForNullReference(sop, "sop"); Platform.CheckForNullReference(studyXml, "studyXml"); if (!studyXml.Contains(sop.SeriesInstanceUid, sop.SopInstanceUid)) { // Uid was inserted but not in the study xml. // Auto-recovery might have detect problem with that file and remove it from the study. // Assume the study xml has been corrected and ignore the uid. Platform.Log(LogLevel.Warn, "Skipping SOP {0} in series {1}. It is no longer part of the study.", sop.SopInstanceUid, sop.SeriesInstanceUid); // Delete it out of the queue DeleteWorkQueueUid(sop); return true; } string basePath = Path.Combine(StorageLocation.GetStudyPath(), sop.SeriesInstanceUid); basePath = Path.Combine(basePath, sop.SopInstanceUid); string path; if (sop.Extension != null) path = basePath + "." + sop.Extension; else path = basePath + ServerPlatform.DicomFileExtension; try { ProcessFile(item, sop, path, studyXml, theCodecFactory); // WorkQueueUid has been deleted out by the processor return true; } catch (Exception e) { if (e.InnerException != null && e.InnerException is DicomCodecUnsupportedSopException) { Platform.Log(LogLevel.Warn, e, "Instance not supported for compressor: {0}. Deleting WorkQueue entry for SOP {1}", e.Message, sop.SopInstanceUid); item.FailureDescription = e.InnerException != null ? e.InnerException.Message : e.Message; // Delete it out of the queue DeleteWorkQueueUid(sop); return false; } Platform.Log(LogLevel.Error, e, "Unexpected exception when compressing file: {0} SOP Instance: {1}", path, sop.SopInstanceUid); item.FailureDescription = e.InnerException != null ? e.InnerException.Message : e.Message; sop.FailureCount++; UpdateWorkQueueUid(sop); return false; } }
protected override void OnExecute(CommandProcessor theProcessor) { // backup if (_studyXml.Contains(_seriesUid)) { Platform.Log(LogLevel.Info, "Removing series {0} from StudyXML for study {1}", _seriesUid, _studyInstanceUid); _oldSeriesXml = _studyXml[_seriesUid]; if (!_studyXml.RemoveSeries(_seriesUid)) { throw new ApplicationException(String.Format("Could not remove series {0} from study {1}", _seriesUid, _studyInstanceUid)); } } }
public void StudyXmlTestFailure() { string directory; using (var processor = new TestCommandProcessor()) { var studyXml = new StudyXml(DicomUid.GenerateUid().UID); var images = SetupMRSeries(2, 5, studyXml.StudyInstanceUid); directory = Path.Combine(processor.ProcessorContext.TempDirectory, "StudyXmlTest2"); DirectoryUtility.DeleteIfExists(directory); processor.AddCommand(new CreateDirectoryCommand(directory)); foreach (var i in images) { string file = Path.Combine(directory, i[DicomTags.SopInstanceUid] + ".dcm"); processor.AddCommand(new SaveDicomFileCommand(file, new DicomFile(file, new DicomAttributeCollection(), i), false)); processor.AddCommand(new InsertInstanceXmlCommand(studyXml, file)); } string file2 = Path.Combine(directory, "Test.dcm"); processor.AddCommand(new SaveDicomFileCommand(file2, _dicomFile, true)); processor.AddCommand(new SaveDicomFileCommand(file2, _dicomFile, true)); Assert.IsFalse(processor.Execute(), processor.FailureReason); Assert.IsTrue((images.Count * 2 + 3) == processor.TestContext.CommandsExecuted); foreach (var i in images) { Assert.IsFalse(studyXml.Contains(i[DicomTags.SeriesInstanceUid], i[DicomTags.SopInstanceUid])); } // Directory should be deleted too Assert.IsFalse(Directory.Exists(directory)); } }
private void ProcessSeriesLevelDelete(Model.WorkQueue item) { // ensure the Study is loaded. Study study = StorageLocation.Study; Platform.CheckForNullReference(study, "Study record doesn't exist"); Platform.Log(LogLevel.Info, "Processing Series Level Deletion for Study {0}, A#: {1}", study.StudyInstanceUid, study.AccessionNumber); _seriesToDelete = new List <Series>(); bool completed = false; try { // Load the list of Series to be deleted from the WorkQueueUid LoadUids(item); // Go through the list of series and add commands // to delete each of them. It's all or nothing. using (ServerCommandProcessor processor = new ServerCommandProcessor(String.Format("Deleting Series from study {0}, A#:{1}, Patient: {2}, ID:{3}", study.StudyInstanceUid, study.AccessionNumber, study.PatientsName, study.PatientId))) { StudyXml studyXml = StorageLocation.LoadStudyXml(); IDictionary <string, Series> existingSeries = StorageLocation.Study.Series; // Add commands to delete the folders and update the xml foreach (WorkQueueUid uid in WorkQueueUidList) { // Delete from study XML if (studyXml.Contains(uid.SeriesInstanceUid)) { //Note: DeleteDirectoryCommand doesn't throw exception if the folder doesn't exist var xmlUpdate = new RemoveSeriesFromStudyXml(studyXml, uid.SeriesInstanceUid); processor.AddCommand(xmlUpdate); } // Delete from filesystem string path = StorageLocation.GetSeriesPath(uid.SeriesInstanceUid); if (Directory.Exists(path)) { var delDir = new DeleteDirectoryCommand(path, true); processor.AddCommand(delDir); } } // flush the updated xml to disk processor.AddCommand(new SaveXmlCommand(studyXml, StorageLocation)); // Update the db.. NOTE: these commands are executed at the end. foreach (WorkQueueUid uid in WorkQueueUidList) { // Delete from DB WorkQueueUid queueUid = uid; Series theSeries = existingSeries[queueUid.SeriesInstanceUid]; if (theSeries != null) { _seriesToDelete.Add(theSeries); var delSeries = new DeleteSeriesFromDBCommand(StorageLocation, theSeries); processor.AddCommand(delSeries); delSeries.Executing += DeleteSeriesFromDbExecuting; } else { // Series doesn't exist Platform.Log(LogLevel.Info, "Series {0} is invalid or no longer exists", uid.SeriesInstanceUid); } // The WorkQueueUid must be cleared before the entry can be removed from the queue var deleteUid = new DeleteWorkQueueUidCommand(uid); processor.AddCommand(deleteUid); // Force a re-archival if necessary processor.AddCommand(new InsertArchiveQueueCommand(item.ServerPartitionKey, item.StudyStorageKey)); } if (!processor.Execute()) { throw new ApplicationException( String.Format("Error occurred when series from Study {0}, A#: {1}", study.StudyInstanceUid, study.AccessionNumber), processor.FailureException); } else { foreach (Series series in _seriesToDelete) { OnSeriesDeleted(series); } } } completed = true; } finally { if (completed) { OnCompleted(); PostProcessing(item, WorkQueueProcessorStatus.Complete, WorkQueueProcessorDatabaseUpdate.ResetQueueState); } else { PostProcessing(item, WorkQueueProcessorStatus.Pending, WorkQueueProcessorDatabaseUpdate.None); } } }
protected override void ProcessItem(Model.WorkQueue item) { Platform.CheckForNullReference(item, "item"); Platform.CheckForNullReference(item.StudyStorageKey, "item.StudyStorageKey"); var context = new StudyProcessorContext(StorageLocation); // TODO: Should we enforce the patient's name rule? // If we do, the Study record will have the new patient's name // but how should we handle the name in the Patient record? bool enforceNameRules = false; var processor = new SopInstanceProcessor(context) { EnforceNameRules = enforceNameRules }; var seriesMap = new Dictionary <string, List <string> >(); bool successful = true; string failureDescription = null; // The processor stores its state in the Data column ReadQueueData(item); if (_queueData.State == null || !_queueData.State.ExecuteAtLeastOnce) { // Added for ticket #9673: // If the study folder does not exist and the study has been archived, trigger a restore and we're done if (!Directory.Exists(StorageLocation.GetStudyPath())) { if (StorageLocation.ArchiveLocations.Count > 0) { Platform.Log(LogLevel.Info, "Reprocessing archived study {0} for Patient {1} (PatientId:{2} A#:{3}) on Partition {4} without study data on the filesystem. Inserting Restore Request.", Study.StudyInstanceUid, Study.PatientsName, Study.PatientId, Study.AccessionNumber, ServerPartition.Description); PostProcessing(item, WorkQueueProcessorStatus.Complete, WorkQueueProcessorDatabaseUpdate.ResetQueueState); // Post process had to be done first so the study is unlocked so the RestoreRequest can be inserted. ServerHelper.InsertRestoreRequest(StorageLocation); RaiseAlert(WorkQueueItem, AlertLevel.Warning, string.Format( "Found study {0} for Patient {1} (A#:{2})on Partition {3} without storage folder, restoring study.", Study.StudyInstanceUid, Study.PatientsName, Study.AccessionNumber, ServerPartition.Description)); return; } } if (Study == null) { Platform.Log(LogLevel.Info, "Reprocessing study {0} on Partition {1}", StorageLocation.StudyInstanceUid, ServerPartition.Description); } else { Platform.Log(LogLevel.Info, "Reprocessing study {0} for Patient {1} (PatientId:{2} A#:{3}) on Partition {4}", Study.StudyInstanceUid, Study.PatientsName, Study.PatientId, Study.AccessionNumber, ServerPartition.Description); } CleanupDatabase(); } else { if (_queueData.State.Completed) { #region SAFE-GUARD CODE: PREVENT INFINITE LOOP // The processor indicated it had completed reprocessing in previous run. The entry should have been removed and this block of code should never be called. // However, we have seen ReprocessStudy entries that mysterously contain rows in the WorkQueueUid table. // The rows prevent the entry from being removed from the database and the ReprocessStudy keeps repeating itself. // update the state first, increment the CompleteAttemptCount _queueData.State.ExecuteAtLeastOnce = true; _queueData.State.Completed = true; _queueData.State.CompleteAttemptCount++; SaveState(item, _queueData); if (_queueData.State.CompleteAttemptCount < 10) { // maybe there was db error in previous attempt to remove the entry. Let's try again. Platform.Log(LogLevel.Info, "Resuming Reprocessing study {0} but it was already completed!!!", StorageLocation.StudyInstanceUid); PostProcessing(item, WorkQueueProcessorStatus.Complete, WorkQueueProcessorDatabaseUpdate.ResetQueueState); } else { // we are definitely stuck. Platform.Log(LogLevel.Error, "ReprocessStudy {0} for study {1} appears stuck. Aborting it.", item.Key, StorageLocation.StudyInstanceUid); item.FailureDescription = "This entry had completed but could not be removed."; PostProcessingFailure(item, WorkQueueProcessorFailureType.Fatal); } return; #endregion } if (Study == null) { Platform.Log(LogLevel.Info, "Resuming Reprocessing study {0} on Partition {1}", StorageLocation.StudyInstanceUid, ServerPartition.Description); } else { Platform.Log(LogLevel.Info, "Resuming Reprocessing study {0} for Patient {1} (PatientId:{2} A#:{3}) on Partition {4}", Study.StudyInstanceUid, Study.PatientsName, Study.PatientId, Study.AccessionNumber, ServerPartition.Description); } } StudyXml studyXml = LoadStudyXml(); int reprocessedCounter = 0; var removedFiles = new List <FileInfo>(); try { // Traverse the directories, process 500 files at a time FileProcessor.Process(StorageLocation.GetStudyPath(), "*.*", delegate(string path, out bool cancel) { #region Reprocess File var file = new FileInfo(path); // ignore all files except those ending ".dcm" // ignore "bad(0).dcm" files too if (Regex.IsMatch(file.Name.ToUpper(), "[0-9]+\\.DCM$")) { try { var dicomFile = new DicomFile(path); dicomFile.Load(DicomReadOptions.StorePixelDataReferences | DicomReadOptions.Default); string seriesUid = dicomFile.DataSet[DicomTags.SeriesInstanceUid].GetString(0, string.Empty); string instanceUid = dicomFile.DataSet[DicomTags.SopInstanceUid].GetString(0, string.Empty); if (studyXml.Contains(seriesUid, instanceUid)) { if (!seriesMap.ContainsKey(seriesUid)) { seriesMap.Add(seriesUid, new List <string>()); } if (!seriesMap[seriesUid].Contains(instanceUid)) { seriesMap[seriesUid].Add(instanceUid); } else { Platform.Log(LogLevel.Warn, "SOP Instance UID in {0} appears more than once in the study.", path); } } else { Platform.Log(ServerPlatform.InstanceLogLevel, "Reprocessing SOP {0} for study {1}", instanceUid, StorageLocation.StudyInstanceUid); string groupId = ServerHelper.GetUidGroup(dicomFile, StorageLocation.ServerPartition, WorkQueueItem.InsertTime); ProcessingResult result = processor.ProcessFile(groupId, dicomFile, studyXml, true, false, null, null); switch (result.Status) { case ProcessingStatus.Success: reprocessedCounter++; if (!seriesMap.ContainsKey(seriesUid)) { seriesMap.Add(seriesUid, new List <string>()); } if (!seriesMap[seriesUid].Contains(instanceUid)) { seriesMap[seriesUid].Add(instanceUid); } else { Platform.Log(LogLevel.Warn, "SOP Instance UID in {0} appears more than once in the study.", path); } break; case ProcessingStatus.Reconciled: Platform.Log(LogLevel.Warn, "SOP was unexpectedly reconciled on reprocess SOP {0} for study {1}. It will be removed from the folder.", instanceUid, StorageLocation.StudyInstanceUid); failureDescription = String.Format("SOP Was reconciled: {0}", instanceUid); // Added for #10620 (Previously we didn't do anything here) // Because we are reprocessing files in the study folder, when file needs to be reconciled it is copied to the reconcile folder // Therefore, we need to delete the one in the study folder. Otherwise, there will be problem when the SIQ entry is reconciled. // InstanceAlreadyExistsException will also be thrown by the SOpInstanceProcessor if this ReprocessStudy WQI // resumes and reprocesses the same file again. // Note: we are sure that the file has been copied to the Reconcile folder and there's no way back. // We must get rid of this file in the study folder. FileUtils.Delete(path); // Special handling: if the file is one which we're supposed to reprocess at the end (see ProcessAdditionalFiles), we must remove the file from the list if (_additionalFilesToProcess != null && _additionalFilesToProcess.Contains(path)) { _additionalFilesToProcess.Remove(path); } break; } } } catch (DicomException ex) { // TODO : should we fail the reprocess instead? Deleting an dicom file can lead to incomplete study. removedFiles.Add(file); Platform.Log(LogLevel.Warn, "Skip reprocessing and delete {0}: Not readable.", path); FileUtils.Delete(path); failureDescription = ex.Message; } } else if (!file.Extension.Equals(".xml") && !file.Extension.Equals(".gz")) { // not a ".dcm" or header file, delete it removedFiles.Add(file); FileUtils.Delete(path); } #endregion cancel = reprocessedCounter >= 500; }, true); if (studyXml != null) { EnsureConsistentObjectCount(studyXml, seriesMap); SaveStudyXml(studyXml); } // Completed if either all files have been reprocessed // or no more dicom files left that can be reprocessed. _completed = reprocessedCounter == 0; } catch (Exception e) { successful = false; failureDescription = e.Message; Platform.Log(LogLevel.Error, e, "Unexpected exception when reprocessing study: {0}", StorageLocation.StudyInstanceUid); Platform.Log(LogLevel.Error, "Study may be in invalid unprocessed state. Study location: {0}", StorageLocation.GetStudyPath()); throw; } finally { LogRemovedFiles(removedFiles); // Update the state _queueData.State.ExecuteAtLeastOnce = true; _queueData.State.Completed = _completed; _queueData.State.CompleteAttemptCount++; SaveState(item, _queueData); if (!successful) { FailQueueItem(item, failureDescription); } else { if (!_completed) { // Put it back to Pending PostProcessing(item, WorkQueueProcessorStatus.Pending, WorkQueueProcessorDatabaseUpdate.None); } else { LogHistory(); // Run Study / Series Rules Engine. var engine = new StudyRulesEngine(StorageLocation, ServerPartition); engine.Apply(ServerRuleApplyTimeEnum.StudyProcessed); // Log the FilesystemQueue related entries StorageLocation.LogFilesystemQueue(); PostProcessing(item, WorkQueueProcessorStatus.Complete, WorkQueueProcessorDatabaseUpdate.ResetQueueState); Platform.Log(LogLevel.Info, "Completed reprocessing of study {0} on partition {1}", StorageLocation.StudyInstanceUid, ServerPartition.Description); } } } }
private bool ProcessWorkQueueUid(Model.WorkQueue item, WorkQueueUid sop, StudyXml studyXml, IDicomCodecFactory theCodecFactory) { Platform.CheckForNullReference(item, "item"); Platform.CheckForNullReference(sop, "sop"); Platform.CheckForNullReference(studyXml, "studyXml"); if (!studyXml.Contains(sop.SeriesInstanceUid, sop.SopInstanceUid)) { // Uid was inserted but not in the study xml. // Auto-recovery might have detect problem with that file and remove it from the study. // Assume the study xml has been corrected and ignore the uid. Platform.Log(LogLevel.Warn, "Skipping SOP {0} in series {1}. It is no longer part of the study.", sop.SopInstanceUid, sop.SeriesInstanceUid); // Delete it out of the queue DeleteWorkQueueUid(sop); return(true); } string basePath = Path.Combine(StorageLocation.GetStudyPath(), sop.SeriesInstanceUid); basePath = Path.Combine(basePath, sop.SopInstanceUid); string path; if (sop.Extension != null) { path = basePath + "." + sop.Extension; } else { path = basePath + ServerPlatform.DicomFileExtension; } try { ProcessFile(item, sop, path, studyXml, theCodecFactory); // WorkQueueUid has been deleted out by the processor return(true); } catch (Exception e) { if (e.InnerException != null && e.InnerException is DicomCodecUnsupportedSopException) { Platform.Log(LogLevel.Warn, e, "Instance not supported for compressor: {0}. Deleting WorkQueue entry for SOP {1}", e.Message, sop.SopInstanceUid); item.FailureDescription = e.InnerException != null ? e.InnerException.Message : e.Message; // Delete it out of the queue DeleteWorkQueueUid(sop); return(false); } Platform.Log(LogLevel.Error, e, "Unexpected exception when compressing file: {0} SOP Instance: {1}", path, sop.SopInstanceUid); item.FailureDescription = e.InnerException != null ? e.InnerException.Message : e.Message; sop.FailureCount++; UpdateWorkQueueUid(sop); return(false); } }