/// <summary>
/// Do the actual StudyPurge
/// </summary>
/// <param name="item"></param>
/// <param name="fs">The filesystem being worked on.</param>
private void DoStudyPurge(ServerFilesystemInfo fs, Model.ServiceLock item)
{
DateTime deleteTime = Platform.Time;
FilesystemQueueTypeEnum type = FilesystemQueueTypeEnum.PurgeStudy;
while (_bytesToRemove > 0)
{
Platform.Log(LogLevel.Debug,
"{1:0.0} MBs needs to be removed from '{0}'. Querying for studies that can be purged",
fs.Filesystem.Description, _bytesToRemove / (1024 * 1024));
IList <FilesystemQueue> list =
GetFilesystemQueueCandidates(item, deleteTime, type, false);
if (list.Count > 0)
{
ProcessStudyPurgeCandidates(list);
if (CancelPending)
{
break;
}
}
else
{
// No candidates
break;
}
}
}
protected override void OnProcess(Model.ServiceLock item)
{
_store = PersistentStoreRegistry.GetDefaultStore();
using (var context = new ServerExecutionContext())
{
var broker = context.ReadContext.GetBroker <IServerPartitionEntityBroker>();
var criteria = new ServerPartitionSelectCriteria();
criteria.AeTitle.SortAsc(0);
_partitions = broker.Find(criteria);
}
ServerFilesystemInfo info = FilesystemMonitor.Instance.GetFilesystemInfo(item.FilesystemKey);
Platform.Log(LogLevel.Info, "Starting reinventory of filesystem: {0}", info.Filesystem.Description);
ReinventoryFilesystem(info.Filesystem);
item.ScheduledTime = item.ScheduledTime.AddDays(1);
if (CancelPending)
{
Platform.Log(LogLevel.Info,
"Filesystem Reinventory of {0} has been canceled, rescheduling. Note that the entire Filesystem will be reinventoried again.",
info.Filesystem.Description);
UnlockServiceLock(item, true, Platform.Time.AddMinutes(1));
}
else
{
Platform.Log(LogLevel.Info, "Filesystem Reinventory of {0} has completed.",
info.Filesystem.Description);
UnlockServiceLock(item, false, Platform.Time.AddDays(1));
}
}
private void MigrateStudies(Model.ServiceLock item, ServerFilesystemInfo fs)
{
ServerFilesystemInfo newFilesystem = FilesystemMonitor.Instance.GetLowerTierFilesystemForStorage(fs);
if (newFilesystem == null)
{
Platform.Log(LogLevel.Warn,
"No writable storage in lower tiers. Tier-migration for '{0}' is disabled at this time.",
fs.Filesystem.Description);
return;
}
Platform.Log(LogLevel.Info, "Starting Tier Migration from {0}", fs.Filesystem.Description);
try
{
DoStudyMigrate(fs, item);
}
catch (Exception e)
{
Platform.Log(LogLevel.Error, e, "Unexpected exception when scheduling tier-migration.");
}
Platform.Log(LogLevel.Info, "{0} studies have been scheduled for migration from filesystem '{1}'",
_studiesMigrated, fs.Filesystem.Description);
}
/// <summary>
/// Do the actual Study migration.
/// </summary>
/// <param name="fs">The filesystem</param>
/// <param name="item">The ServiceLock item being processed.</param>
private void DoStudyMigrate(ServerFilesystemInfo fs, Model.ServiceLock item)
{
FilesystemQueueTypeEnum type = FilesystemQueueTypeEnum.TierMigrate;
while (_bytesToRemove > 0)
{
TimeSpanStatistics queryTime = new TimeSpanStatistics("Query Time");
queryTime.Start();
IList <FilesystemQueue> list = GetFilesystemQueueCandidates(item, Platform.Time, type, false);
queryTime.End();
Platform.Log(LogLevel.Info,
"{1:0.0} MBs needs to be removed from '{0}'. Found {2} studies that can be migrated in {3} seconds",
fs.Filesystem.Description, _bytesToRemove / (1024 * 1024),
list.Count,
queryTime.Value.TotalSeconds);
if (list.Count > 0)
{
ProcessStudyMigrateCandidates(list);
if (CancelPending)
{
break;
}
}
else
{
break;
}
}
}
private static ServerFilesystemInfo EnsureFilesystemIsValid(Model.ServiceLock item)
{
ServerFilesystemInfo filesystem = null;
if (item.FilesystemKey != null)
{
filesystem = FilesystemMonitor.Instance.GetFilesystemInfo(item.FilesystemKey);
if (filesystem == null)
{
Platform.Log(LogLevel.Warn, "Filesystem for incoming folders is no longer valid. Assigning new filesystem.");
item.FilesystemKey = null;
UpdateFilesystemKey(item);
}
}
if (filesystem == null)
{
filesystem = SelectFilesystem();
if (filesystem == null)
{
UnlockServiceLock(item, true, Platform.Time.AddHours(2));
}
else
{
item.FilesystemKey = filesystem.Filesystem.Key;
UpdateFilesystemKey(item);
}
}
return(filesystem);
}
/// <summary>
/// Process the <see cref="ServiceLock"/> item.
/// </summary>
/// <param name="item"></param>
protected override void OnProcess(Model.ServiceLock item)
{
LoadRulesEngine();
ServerFilesystemInfo info = FilesystemMonitor.Instance.GetFilesystemInfo(item.FilesystemKey);
_stats.StudyRate.Start();
_stats.Filesystem = info.Filesystem.Description;
Platform.Log(LogLevel.Info, "Starting reprocess of filesystem: {0}", info.Filesystem.Description);
ReprocessFilesystem(info.Filesystem);
Platform.Log(LogLevel.Info, "Completed reprocess of filesystem: {0}", info.Filesystem.Description);
_stats.StudyRate.SetData(_stats.NumStudies);
_stats.StudyRate.End();
StatisticsLogger.Log(LogLevel.Info, _stats);
item.ScheduledTime = item.ScheduledTime.AddDays(1);
if (CancelPending)
{
Platform.Log(LogLevel.Info,
"Filesystem Reprocess of {0} has been canceled, rescheduling. Entire filesystem will be reprocessed.",
info.Filesystem.Description);
UnlockServiceLock(item, true, Platform.Time.AddMinutes(1));
}
else
{
UnlockServiceLock(item, false, Platform.Time.AddDays(1));
}
}
/// <summary>
/// Process the <see cref="ServiceLock"/> item.
/// </summary>
/// <param name="item"></param>
protected override void OnProcess(Model.ServiceLock item)
{
PersistentStoreRegistry.GetDefaultStore();
using (ServerExecutionContext context = new ServerExecutionContext())
{
IServerPartitionEntityBroker broker = context.ReadContext.GetBroker <IServerPartitionEntityBroker>();
ServerPartitionSelectCriteria criteria = new ServerPartitionSelectCriteria();
criteria.AeTitle.SortAsc(0);
_partitions = broker.Find(criteria);
}
ServerFilesystemInfo info = FilesystemMonitor.Instance.GetFilesystemInfo(item.FilesystemKey);
Platform.Log(LogLevel.Info, "Starting rebuilding of Study XML files for filesystem: {0}", info.Filesystem.Description);
TraverseFilesystemStudies(info.Filesystem);
item.ScheduledTime = item.ScheduledTime.AddDays(1);
if (CancelPending)
{
Platform.Log(LogLevel.Info,
"FilesystemRebuildXml of {0} has been canceled, rescheduling. Note that the entire Filesystem will be rebuilt again.",
info.Filesystem.Description);
UnlockServiceLock(item, true, Platform.Time.AddMinutes(1));
}
else
{
UnlockServiceLock(item, false, Platform.Time.AddDays(1));
}
Platform.Log(LogLevel.Info, "Completed rebuilding of the Study XML files for filesystem: {0}", info.Filesystem.Description);
}
protected override void OnProcess(Model.ServiceLock item)
{
DirectoryImportSettings settings = DirectoryImportSettings.Default;
ServerFilesystemInfo filesystem = EnsureFilesystemIsValid(item);
if (filesystem != null)
{
Platform.Log(LogLevel.Debug, "Start importing dicom files from {0}", filesystem.Filesystem.FilesystemPath);
foreach (ServerPartition partition in ServerPartitionMonitor.Instance)
{
DirectoryImporterParameters parms = new DirectoryImporterParameters();
String incomingFolder = partition.GetIncomingFolder(); // String.Format("{0}_{1}", partition.PartitionFolder, FilesystemMonitor.ImportDirectorySuffix);
parms.Directory = new DirectoryInfo(filesystem.Filesystem.GetAbsolutePath(incomingFolder));
parms.PartitionAE = partition.AeTitle;
parms.MaxImages = settings.MaxBatchSize;
parms.Delay = settings.ImageDelay;
parms.Filter = "*.*";
if (!parms.Directory.Exists)
{
parms.Directory.Create();
}
DirectoryImporterBackgroundProcess process = new DirectoryImporterBackgroundProcess(parms);
process.SopImported += delegate { _importedSopCounter++; };
process.RestoreTriggered += delegate { _restoreTriggered = true; };
_queue.Enqueue(process);
}
// start the processes.
for (int n = 0; n < settings.MaxConcurrency && n < _queue.Count; n++)
{
LaunchNextBackgroundProcess();
}
_allCompleted.WaitOne();
if (CancelPending)
{
Platform.Log(LogLevel.Info, "All import processes have completed gracefully.");
}
}
if (_restoreTriggered)
{
DateTime newScheduledTime = Platform.Time.AddSeconds(Math.Max(settings.RecheckDelaySeconds, 60));
Platform.Log(LogLevel.Info, "Some Study/Studies need to be restored first. File Import will resume until {0}", newScheduledTime);
UnlockServiceLock(item, true, newScheduledTime);
}
else
{
UnlockServiceLock(item, true, Platform.Time.AddSeconds(_importedSopCounter > 0? 5: settings.RecheckDelaySeconds));
}
}
protected override void ProcessItem(Model.WorkQueue item)
{
Platform.CheckForNullReference(item, "item");
Platform.Log(LogLevel.Info,
"Starting Tier Migration of study {0} for Patient {1} (PatientId:{2} A#:{3}) on Partition {4}",
Study.StudyInstanceUid, Study.PatientsName, Study.PatientId,
Study.AccessionNumber, ServerPartition.Description);
// The WorkQueue Data column contains the state of the processing. It is set to "Migrated" if the entry has been
// successfully executed once.
TierMigrationProcessingState state = GetCurrentState(item);
switch (state)
{
case TierMigrationProcessingState.Migrated:
Platform.Log(LogLevel.Info, "Study has been migrated to {0} on {1}", StorageLocation.FilesystemPath, StorageLocation.FilesystemTierEnum.Description);
PostProcessing(item, WorkQueueProcessorStatus.Complete, WorkQueueProcessorDatabaseUpdate.ResetQueueState);
break;
case TierMigrationProcessingState.NotStated:
ServerFilesystemInfo currFilesystem = FilesystemMonitor.Instance.GetFilesystemInfo(StorageLocation.FilesystemKey);
ServerFilesystemInfo newFilesystem = FilesystemMonitor.Instance.GetLowerTierFilesystemForStorage(currFilesystem);
if (newFilesystem == null)
{
// This entry shouldn't have been scheduled in the first place.
// It's possible that the folder wasn't full when the entry was scheduled.
// Another possiblity is the study was migrated but an error was encountered when updating the entry.//
// We should re-insert the filesystem queue so that if the study will be migrated if the space is freed up
// in the future.
String msg = String.Format(
"Study '{0}' cannot be migrated: no writable filesystem can be found in lower tiers for filesystem '{1}'. Reschedule migration for future.",
StorageLocation.StudyInstanceUid, currFilesystem.Filesystem.Description);
Platform.Log(LogLevel.Warn, msg);
ReinsertFilesystemQueue();
PostProcessing(item, WorkQueueProcessorStatus.Complete, WorkQueueProcessorDatabaseUpdate.ResetQueueState);
}
else
{
DoMigrateStudy(StorageLocation, newFilesystem);
// Update the state separately so that if the validation (done in the PostProcessing method) fails,
// we know the study has been migrated when we resume after auto-recovery has been completed.
UpdateState(item.Key, TierMigrationProcessingState.Migrated);
PostProcessing(item, WorkQueueProcessorStatus.Complete, WorkQueueProcessorDatabaseUpdate.ResetQueueState);
}
break;
default:
throw new NotImplementedException("Not implemented");
}
}
private void LoadAdditionalEntities()
{
Debug.Assert(ServerPartition != null);
Debug.Assert(StorageLocation != null);
using (ServerExecutionContext context = new ServerExecutionContext())
{
if (_filesystem != null)
_filesystem = FilesystemMonitor.Instance.GetFilesystemInfo(StorageLocation.FilesystemKey);
_study = StorageLocation.LoadStudy(context.ReadContext);
_patient = Patient.Load(context.ReadContext, _study.PatientKey);
}
}
private void LoadAdditionalEntities()
{
Debug.Assert(ServerPartition != null);
Debug.Assert(StorageLocation != null);
using (ServerExecutionContext context = new ServerExecutionContext())
{
if (_filesystem != null)
{
_filesystem = FilesystemMonitor.Instance.GetFilesystemInfo(StorageLocation.FilesystemKey);
}
_study = StorageLocation.LoadStudy(context.ReadContext);
_patient = Patient.Load(context.ReadContext, _study.PatientKey);
}
}
private void PurgeStudies(Model.ServiceLock item, ServerFilesystemInfo fs)
{
Platform.Log(LogLevel.Info, "Starting query for Filesystem Purge candidates on '{0}'.",
fs.Filesystem.Description);
try
{
DoStudyPurge(fs, item);
}
catch (Exception e)
{
Platform.Log(LogLevel.Error, e, "Unexpected exception when processing StudyDelete records.");
}
Platform.Log(LogLevel.Info, "{0} studies have been scheduled for purging from filesystem '{1}'",
_studiesPurged, fs.Filesystem.Description);
}
protected override void OnProcess(Model.ServiceLock item)
{
ServerFilesystemInfo fs = FilesystemMonitor.Instance.GetFilesystemInfo(item.FilesystemKey);
Platform.Log(LogLevel.Info,
"Starting check for studies to lossless compress on filesystem '{0}'.",
fs.Filesystem.Description);
int delayMinutes = ServiceLockSettings.Default.FilesystemLosslessCompressRecheckDelay;
try
{
DateTime deleteTime = Platform.Time;
FilesystemQueueTypeEnum type = FilesystemQueueTypeEnum.LosslessCompress;
IList <FilesystemQueue> list = GetFilesystemQueueCandidates(item, deleteTime, type, false);
if (list.Count > 0)
{
ProcessCompressCandidates(list, type);
}
}
catch (Exception e)
{
Platform.Log(LogLevel.Error, e, "Unexpected exception when processing LosslessCompress records.");
delayMinutes = 5;
}
DateTime scheduledTime = Platform.Time.AddMinutes(delayMinutes);
if (_studiesInserted == 0)
{
Platform.Log(LogLevel.Info,
"No eligible candidates to lossless compress from filesystem '{0}'. Next scheduled filesystem check {1}",
fs.Filesystem.Description, scheduledTime);
}
else
{
Platform.Log(LogLevel.Info,
"Completed inserting lossless compress candidates into WorkQueue: {0}. {1} studies inserted. Next scheduled filesystem check {2}",
fs.Filesystem.Description, _studiesInserted,
scheduledTime);
}
UnlockServiceLock(item, true, scheduledTime);
}
protected override string GetTemporaryPath()
{
IList <StudyStorageLocation> storages =
StudyStorageLocation.FindStorageLocations(StudyStorage.Load(_item.StudyStorageKey));
if (storages == null || storages.Count == 0)
{
// ???
return(base.GetTemporaryPath());
}
ServerFilesystemInfo filesystem = FilesystemMonitor.Instance.GetFilesystemInfo(storages[0].FilesystemKey);
if (filesystem == null)
{
// not ready?
return(base.GetTemporaryPath());
}
string basePath = GetTempPathRoot();
if (String.IsNullOrEmpty(basePath))
{
basePath = Path.Combine(filesystem.Filesystem.FilesystemPath, "temp");
}
String tempDirectory = Path.Combine(basePath,
String.Format("{0}-{1}", _item.WorkQueueTypeEnum.Lookup, _item.GetKey()));
for (int i = 2; i < 1000; i++)
{
if (!Directory.Exists(tempDirectory))
{
break;
}
tempDirectory = Path.Combine(basePath, String.Format("{0}-{1}({2})",
_item.WorkQueueTypeEnum.Lookup, _item.GetKey(), i));
}
if (!Directory.Exists(tempDirectory))
{
Directory.CreateDirectory(tempDirectory);
}
return(tempDirectory);
}
/// <summary>
/// Inserts a <see cref="StudyIntegrityQueue"/> entry for manual reconciliation.
/// </summary>
/// <param name="file">The DICOM file that needs to be reconciled.</param>
/// <param name="reason">The type of <see cref="StudyIntegrityQueue"/> entry to be inserted.</param>
/// <param name="uid">A UID to delete on insert.</param>
/// <remarks>
/// A copy of the DICOM file will be stored in a special folder allocated for
/// reconciliation purpose. The caller is responsible for managing the original copy.
/// </remarks>
public void ScheduleReconcile(DicomFile file, StudyIntegrityReasonEnum reason, WorkQueueUid uid)
{
Platform.CheckForNullReference(_context.StudyLocation, "_context.StudyLocation");
Platform.Log(LogLevel.Info, "Scheduling new manual reconciliation for SOP {0}", file.MediaStorageSopInstanceUid);
ServerFilesystemInfo fs = FilesystemMonitor.Instance.GetFilesystemInfo(_context.StudyLocation.FilesystemKey);
Platform.CheckForNullReference(fs, "fs");
ReconcileStorage reconcileStorage = new ReconcileStorage(_context.StudyLocation, _context.Group);
using (ServerCommandProcessor processor = new ServerCommandProcessor("Schedule Manual Reconciliation"))
{
string path = reconcileStorage.GetSopInstancePath(file.DataSet[DicomTags.SopInstanceUid].ToString());
DirectoryInfo dir = new DirectoryInfo(path);
if (dir.Parent != null)
{
CreateDirectoryCommand mkdir = new CreateDirectoryCommand(dir.Parent.FullName);
processor.AddCommand(mkdir);
}
SaveDicomFileCommand saveFileCommand = new SaveDicomFileCommand(path, file, true);
processor.AddCommand(saveFileCommand);
InsertSIQCommand updateStudyCommand = new InsertSIQCommand(_context.StudyLocation, reason, file, _context.Group, reconcileStorage);
processor.AddCommand(updateStudyCommand);
if (uid != null)
{
processor.AddCommand(new DeleteWorkQueueUidCommand(uid));
}
if (processor.Execute() == false)
{
throw new ApplicationException(String.Format("Unable to schedule image reconcilation : {0}", processor.FailureReason), processor.FailureException);
}
}
}
protected override string GetTemporaryPath()
{
ServerFilesystemInfo filesystem = FilesystemMonitor.Instance.GetFilesystemInfo(_item.FilesystemKey);
if (filesystem == null)
{
// not ready?
return(base.GetTemporaryPath());
}
else
{
String basePath = GetTempPathRoot();
if (String.IsNullOrEmpty(basePath))
{
basePath = Path.Combine(filesystem.Filesystem.FilesystemPath, "temp");
}
String tempDirectory = Path.Combine(basePath, String.Format("{0}-{1}", _item.ServiceLockTypeEnum.Lookup, _item.GetKey()));
for (int i = 2; i < 1000; i++)
{
if (!Directory.Exists(tempDirectory))
{
break;
}
tempDirectory = Path.Combine(basePath, String.Format("{0}-{1}({2})", _item.ServiceLockTypeEnum.Lookup, _item.GetKey(), i));
}
if (!Directory.Exists(tempDirectory))
{
Directory.CreateDirectory(tempDirectory);
}
return(tempDirectory);
}
}
protected override bool CanStart()
{
Model.WorkQueue item = WorkQueueItem;
IList <Model.WorkQueue> relatedItems = FindRelatedWorkQueueItems(item,
new []
{
WorkQueueTypeEnum.StudyProcess,
WorkQueueTypeEnum.ReconcileStudy,
WorkQueueTypeEnum.ProcessDuplicate,
WorkQueueTypeEnum.CleanupDuplicate,
WorkQueueTypeEnum.CleanupStudy
}, null);
if (!(relatedItems == null || relatedItems.Count == 0))
{
PostponeItem("Study is being processed or reconciled.");
return(false);
}
_filesystem = FilesystemMonitor.Instance.GetFilesystemInfo(StorageLocation.FilesystemKey);
Platform.CheckForNullReference(_filesystem, "filesystem");
if (_filesystem.Full)
{
PostponeItem("The Filesystem is full.");
return(false);
}
if (!_filesystem.Writeable)
{
PostponeItem("The Filesystem is not writeable.");
return(false);
}
return(true);
}
/// <summary>
/// Do the restore.
/// </summary>
/// <param name="queueItem">The queue item to restore.</param>
public void Run(RestoreQueue queueItem)
{
using (RestoreProcessorContext context = new RestoreProcessorContext(queueItem))
{
try
{
// Load up related classes.
using (IReadContext readContext = _nasArchive.PersistentStore.OpenReadContext())
{
_archiveStudyStorage = ArchiveStudyStorage.Load(readContext, queueItem.ArchiveStudyStorageKey);
_serverSyntax = ServerTransferSyntax.Load(readContext, _archiveStudyStorage.ServerTransferSyntaxKey);
_syntax = TransferSyntax.GetTransferSyntax(_serverSyntax.Uid);
StudyStorageLocationQueryParameters parms = new StudyStorageLocationQueryParameters
{
StudyStorageKey = queueItem.StudyStorageKey
};
IQueryStudyStorageLocation broker = readContext.GetBroker <IQueryStudyStorageLocation>();
_location = broker.FindOne(parms);
if (_location == null)
{
_studyStorage = StudyStorage.Load(readContext, queueItem.StudyStorageKey);
if (_studyStorage == null)
{
DateTime scheduleTime = Platform.Time.AddMinutes(5);
Platform.Log(LogLevel.Error, "Unable to find storage location, rescheduling restore request to {0}",
scheduleTime);
queueItem.FailureDescription = "Unable to find storage location, rescheduling request.";
_nasArchive.UpdateRestoreQueue(queueItem, RestoreQueueStatusEnum.Pending, scheduleTime);
return;
}
}
}
if (_location == null)
{
Platform.Log(LogLevel.Info, "Starting restore of nearline study: {0}", _studyStorage.StudyInstanceUid);
}
else
{
Platform.Log(LogLevel.Info, "Starting restore of online study: {0}", _location.StudyInstanceUid);
}
// If restoring a Nearline study, select a filesystem
string destinationFolder;
if (_location == null)
{
ServerFilesystemInfo fs = _nasArchive.Selector.SelectFilesystem();
if (fs == null)
{
DateTime scheduleTime = Platform.Time.AddMinutes(5);
Platform.Log(LogLevel.Error, "No writeable filesystem for restore, rescheduling restore request to {0}",
scheduleTime);
queueItem.FailureDescription = "No writeable filesystem for restore, rescheduling request.";
_nasArchive.UpdateRestoreQueue(queueItem, RestoreQueueStatusEnum.Pending, scheduleTime);
return;
}
destinationFolder = Path.Combine(fs.Filesystem.FilesystemPath, _nasArchive.ServerPartition.PartitionFolder);
}
else
{
destinationFolder = _location.GetStudyPath();
}
// Get the zip file path from the xml data in the ArchiveStudyStorage entry
// Also store the "StudyFolder" for use below
string studyFolder = String.Empty;
string filename = String.Empty;
string studyInstanceUid = String.Empty;
XmlElement element = _archiveStudyStorage.ArchiveXml.DocumentElement;
if (element != null)
{
foreach (XmlElement node in element.ChildNodes)
{
if (node.Name.Equals("StudyFolder"))
{
studyFolder = node.InnerText;
}
else if (node.Name.Equals("Filename"))
{
filename = node.InnerText;
}
else if (node.Name.Equals("Uid"))
{
studyInstanceUid = node.InnerText;
}
}
}
string zipFile = Path.Combine(_nasArchive.NasPath, studyFolder);
zipFile = Path.Combine(zipFile, studyInstanceUid);
zipFile = Path.Combine(zipFile, filename);
// Do a test read of the zip file. If it succeeds, the file is available, if it
// fails, we just set back to pending and recheck.
try
{
FileStream stream = File.OpenRead(zipFile);
// Read a byte, just in case that makes a difference.
stream.ReadByte();
stream.Close();
stream.Dispose();
}
catch (Exception ex)
{
DateTime scheduledTime = Platform.Time.AddSeconds(NasSettings.Default.ReadFailRescheduleDelaySeconds);
Platform.Log(LogLevel.Error, ex, "Archive {0} for Study {1} is unreadable, rescheduling restore to {2}",
zipFile, _studyStorage == null ? (_location == null ? string.Empty : _location.StudyInstanceUid) : _studyStorage.StudyInstanceUid,
scheduledTime);
// Just reschedule in "Restoring" state, the file is unreadable.
_nasArchive.UpdateRestoreQueue(queueItem, RestoreQueueStatusEnum.Restoring,
scheduledTime);
return;
}
if (_location == null)
{
RestoreNearlineStudy(queueItem, zipFile, destinationFolder, studyFolder);
}
else
{
RestoreOnlineStudy(queueItem, zipFile, destinationFolder);
}
}
catch (Exception e)
{
Platform.Log(LogLevel.Error, e, "Unexpected exception processing restore request for {0} on archive {1}",
_studyStorage == null ? (_location == null ? string.Empty : _location.StudyInstanceUid) : _studyStorage.StudyInstanceUid,
_nasArchive.PartitionArchive.Description);
queueItem.FailureDescription = e.Message;
_nasArchive.UpdateRestoreQueue(queueItem, RestoreQueueStatusEnum.Failed, Platform.Time);
}
}
}
protected override bool CanStart()
{
Model.WorkQueue item = WorkQueueItem;
IList<Model.WorkQueue> relatedItems = FindRelatedWorkQueueItems(item,
new []
{
WorkQueueTypeEnum.StudyProcess,
WorkQueueTypeEnum.ReconcileStudy,
WorkQueueTypeEnum.ProcessDuplicate,
WorkQueueTypeEnum.CleanupDuplicate,
WorkQueueTypeEnum.CleanupStudy
}, null);
if (! (relatedItems == null || relatedItems.Count == 0))
{
PostponeItem("Study is being processed or reconciled.");
return false;
}
_filesystem = FilesystemMonitor.Instance.GetFilesystemInfo(StorageLocation.FilesystemKey);
Platform.CheckForNullReference(_filesystem, "filesystem");
if (_filesystem.Full)
{
PostponeItem("The Filesystem is full.");
return false;
}
if (!_filesystem.Writeable)
{
PostponeItem("The Filesystem is not writeable.");
return false;
}
return true;
}
public ServerFilesystemInfo(ServerFilesystemInfo copy)
{
_filesystem = copy.Filesystem;
_online = copy.Online;
_freeBytes = copy.FreeBytes;
_totalBytes = copy.TotalBytes;
}
protected override void OnProcess(Model.ServiceLock item)
{
DirectoryImportSettings settings = DirectoryImportSettings.Default;
// #10419: Avoid running multiple instances of Import service - which can lead to race condition on the input files
if (CheckMultipleInstances())
{
// Instead of disabling duplicate entries automatically, decided it's easier and safer to let user figure out which one should be disabled.
Platform.Log(LogLevel.Warn, "Detect multiple active instances of the Import File Service. Go to Service Scheduling configuration and make sure only one of them is active.");
DateTime newScheduledTime = Platform.Time.AddSeconds(Math.Max(settings.RecheckDelaySeconds, 60));
UnlockServiceLock(item, true, newScheduledTime);
return;
}
ServerFilesystemInfo filesystem = EnsureFilesystemIsValid(item);
if (filesystem != null)
{
Platform.Log(LogLevel.Debug, "Start importing dicom files from {0}", filesystem.Filesystem.FilesystemPath);
foreach (ServerPartition partition in ServerPartitionMonitor.Instance)
{
if (partition.Enabled)
{
String incomingFolder = partition.GetIncomingFolder();
if (!string.IsNullOrEmpty(incomingFolder))
{
DirectoryImporterParameters parms = new DirectoryImporterParameters
{
Directory = new DirectoryInfo(incomingFolder),
PartitionAE = partition.AeTitle,
MaxImages = settings.MaxBatchSize,
Delay = settings.ImageDelay,
Filter = "*.*"
};
if (!parms.Directory.Exists)
{
parms.Directory.Create();
}
DirectoryImporterBackgroundProcess process = new DirectoryImporterBackgroundProcess(parms);
process.SopImported += delegate { _importedSopCounter++; };
process.RestoreTriggered += delegate { _restoreTriggered = true; };
_queue.Enqueue(process);
}
}
}
// start the processes.
for (int n = 0; n < settings.MaxConcurrency && n < _queue.Count; n++)
{
LaunchNextBackgroundProcess();
}
_allCompleted.WaitOne();
if (CancelPending)
{
Platform.Log(LogLevel.Info, "All import processes have completed gracefully.");
}
}
if (_restoreTriggered)
{
DateTime newScheduledTime = Platform.Time.AddSeconds(Math.Max(settings.RecheckDelaySeconds, 60));
Platform.Log(LogLevel.Info, "Some Study/Studies need to be restored first. File Import will resume until {0}", newScheduledTime);
UnlockServiceLock(item, true, newScheduledTime);
}
else
{
UnlockServiceLock(item, true, Platform.Time.AddSeconds(_importedSopCounter > 0 ? 5 : settings.RecheckDelaySeconds));
}
}
private void MigrateStudies(Model.ServiceLock item, ServerFilesystemInfo fs)
{
ServerFilesystemInfo newFilesystem = FilesystemMonitor.Instance.GetLowerTierFilesystemForStorage(fs);
if (newFilesystem == null)
{
Platform.Log(LogLevel.Warn,
"No writable storage in lower tiers. Tier-migration for '{0}' is disabled at this time.",
fs.Filesystem.Description);
return;
}
Platform.Log(LogLevel.Info, "Starting Tier Migration from {0}", fs.Filesystem.Description);
try
{
DoStudyMigrate(fs, item);
}
catch (Exception e)
{
Platform.Log(LogLevel.Error, e, "Unexpected exception when scheduling tier-migration.");
}
Platform.Log(LogLevel.Info, "{0} studies have been scheduled for migration from filesystem '{1}'",
_studiesMigrated, fs.Filesystem.Description);
}
/// <summary>
/// Migrates the study to new tier
/// </summary>
/// <param name="storage"></param>
/// <param name="newFilesystem"></param>
private void DoMigrateStudy(StudyStorageLocation storage, ServerFilesystemInfo newFilesystem)
{
Platform.CheckForNullReference(storage, "storage");
Platform.CheckForNullReference(newFilesystem, "newFilesystem");
TierMigrationStatistics stat = new TierMigrationStatistics {StudyInstanceUid = storage.StudyInstanceUid};
stat.ProcessSpeed.Start();
StudyXml studyXml = storage.LoadStudyXml();
stat.StudySize = (ulong) studyXml.GetStudySize();
Platform.Log(LogLevel.Info, "About to migrate study {0} from {1} to {2}",
storage.StudyInstanceUid, storage.FilesystemTierEnum, newFilesystem.Filesystem.Description);
string newPath = Path.Combine(newFilesystem.Filesystem.FilesystemPath, storage.PartitionFolder);
DateTime startTime = Platform.Time;
DateTime lastLog = Platform.Time;
int fileCounter = 0;
ulong bytesCopied = 0;
long instanceCountInXml = studyXml.NumberOfStudyRelatedInstances;
using (ServerCommandProcessor processor = new ServerCommandProcessor("Migrate Study"))
{
TierMigrationContext context = new TierMigrationContext
{
OriginalStudyLocation = storage,
Destination = newFilesystem
};
string origFolder = context.OriginalStudyLocation.GetStudyPath();
processor.AddCommand(new CreateDirectoryCommand(newPath));
newPath = Path.Combine(newPath, context.OriginalStudyLocation.StudyFolder);
processor.AddCommand(new CreateDirectoryCommand(newPath));
newPath = Path.Combine(newPath, context.OriginalStudyLocation.StudyInstanceUid);
// don't create this directory so that it won't be backed up by MoveDirectoryCommand
CopyDirectoryCommand copyDirCommand = new CopyDirectoryCommand(origFolder, newPath,
delegate (string path)
{
// Update the progress. This is useful if the migration takes long time to complete.
FileInfo file = new FileInfo(path);
bytesCopied += (ulong)file.Length;
fileCounter++;
if (file.Extension != null && file.Extension.Equals(ServerPlatform.DicomFileExtension, StringComparison.InvariantCultureIgnoreCase))
{
TimeSpan elapsed = Platform.Time - lastLog;
TimeSpan totalElapsed = Platform.Time - startTime;
double speedInMBPerSecond = 0;
if (totalElapsed.TotalSeconds > 0)
{
speedInMBPerSecond = (bytesCopied / 1024f / 1024f) / totalElapsed.TotalSeconds;
}
if (elapsed > TimeSpan.FromSeconds(WorkQueueSettings.Instance.TierMigrationProgressUpdateInSeconds))
{
#region Log Progress
StringBuilder stats = new StringBuilder();
if (instanceCountInXml != 0)
{
float pct = (float)fileCounter / instanceCountInXml;
stats.AppendFormat("{0} files moved [{1:0.0}MB] since {2} ({3:0}% completed). Speed={4:0.00}MB/s",
fileCounter, bytesCopied / 1024f / 1024f, startTime, pct * 100, speedInMBPerSecond);
}
else
{
stats.AppendFormat("{0} files moved [{1:0.0}MB] since {2}. Speed={3:0.00}MB/s",
fileCounter, bytesCopied / 1024f / 1024f, startTime, speedInMBPerSecond);
}
Platform.Log(LogLevel.Info, "Tier migration for study {0}: {1}", storage.StudyInstanceUid, stats.ToString());
try
{
using (IUpdateContext ctx = PersistentStoreRegistry.GetDefaultStore().OpenUpdateContext(UpdateContextSyncMode.Flush))
{
IWorkQueueEntityBroker broker = ctx.GetBroker<IWorkQueueEntityBroker>();
WorkQueueUpdateColumns parameters = new WorkQueueUpdateColumns
{FailureDescription = stats.ToString()};
broker.Update(WorkQueueItem.GetKey(), parameters);
ctx.Commit();
}
}
catch
{
// can't log the progress so far... just ignore it
}
finally
{
lastLog = DateTime.Now;
}
#endregion
}
}
});
processor.AddCommand(copyDirCommand);
DeleteDirectoryCommand delDirCommand = new DeleteDirectoryCommand(origFolder, false)
{RequiresRollback = false};
processor.AddCommand(delDirCommand);
TierMigrateDatabaseUpdateCommand updateDbCommand = new TierMigrateDatabaseUpdateCommand(context);
processor.AddCommand(updateDbCommand);
Platform.Log(LogLevel.Info, "Start migrating study {0}.. expecting {1} to be moved", storage.StudyInstanceUid, ByteCountFormatter.Format(stat.StudySize));
if (!processor.Execute())
{
if (processor.FailureException != null)
throw processor.FailureException;
throw new ApplicationException(processor.FailureReason);
}
stat.DBUpdate = updateDbCommand.Statistics;
stat.CopyFiles = copyDirCommand.CopySpeed;
stat.DeleteDirTime = delDirCommand.Statistics;
}
stat.ProcessSpeed.SetData(bytesCopied);
stat.ProcessSpeed.End();
Platform.Log(LogLevel.Info, "Successfully migrated study {0} from {1} to {2} in {3} [ {4} files, {5} @ {6}, DB Update={7}, Remove Dir={8}]",
storage.StudyInstanceUid,
storage.FilesystemTierEnum,
newFilesystem.Filesystem.FilesystemTierEnum,
TimeSpanFormatter.Format(stat.ProcessSpeed.ElapsedTime),
fileCounter,
ByteCountFormatter.Format(bytesCopied),
stat.CopyFiles.FormattedValue,
stat.DBUpdate.FormattedValue,
stat.DeleteDirTime.FormattedValue);
string originalPath = storage.GetStudyPath();
if (Directory.Exists(storage.GetStudyPath()))
{
Platform.Log(LogLevel.Info, "Original study folder could not be deleted. It must be cleaned up manually: {0}", originalPath);
ServerPlatform.Alert(AlertCategory.Application, AlertLevel.Warning, WorkQueueItem.WorkQueueTypeEnum.ToString(), 1000, GetWorkQueueContextData(WorkQueueItem), TimeSpan.Zero,
"Study has been migrated to a new tier. Original study folder must be cleaned up manually: {0}", originalPath);
}
UpdateAverageStatistics(stat);
}
/// <summary>
/// Main <see cref="ServiceLock"/> processing routine.
/// </summary>
/// <param name="item">The <see cref="ServiceLock"/> item to process.</param>
protected override void OnProcess(Model.ServiceLock item)
{
ServiceLockSettings settings = ServiceLockSettings.Default;
ServerFilesystemInfo fs = FilesystemMonitor.Instance.GetFilesystemInfo(item.FilesystemKey);
UpdateState(item, fs);
DateTime scheduledTime;
if (!fs.Online)
{
Platform.Log(LogLevel.Info, "Filesystem {0} is not online. Watermark is not checked at this point.", fs.Filesystem.Description);
scheduledTime = Platform.Time.AddMinutes(settings.FilesystemDeleteRecheckDelay);
}
else
{
_bytesToRemove = FilesystemMonitor.Instance.CheckFilesystemBytesToRemove(item.FilesystemKey);
if (fs.AboveHighWatermark)
{
int count = CheckWorkQueueCount(item);
if (count > 0)
{
// delay to avoid overshoot
Platform.Log(LogLevel.Info,
"Delaying Filesystem ServiceLock check, {0} StudyDelete, StudyPurge or MigrateStudy items still in the WorkQueue for Filesystem: {1} (Current: {2}, High Watermark: {3})",
count, fs.Filesystem.Description, fs.UsedSpacePercentage, fs.Filesystem.HighWatermark);
scheduledTime = Platform.Time.AddMinutes(settings.FilesystemDeleteRecheckDelay);
}
else
{
Platform.Log(LogLevel.Info, "Filesystem above high watermark: {0} (Current: {1}, High Watermark: {2}",
fs.Filesystem.Description, fs.UsedSpacePercentage, fs.Filesystem.HighWatermark);
MigrateStudies(item, fs);
if (_bytesToRemove > 0 && !CancelPending)
{
DeleteStudies(item, fs);
}
if (_bytesToRemove > 0 && !CancelPending)
{
PurgeStudies(item, fs);
}
if (_studiesDeleted + _studiesMigrated + _studiesPurged == 0)
{
Platform.Log(LogLevel.Warn, "Fileystem '{0}' is above high watermark but no studies can be deleted, migrated or purged at this point", fs.Filesystem.Description);
}
scheduledTime = Platform.Time.AddMinutes(settings.FilesystemDeleteRecheckDelay);
}
}
else
{
scheduledTime = Platform.Time.AddMinutes(settings.FilesystemDeleteCheckInterval);
}
}
UnlockServiceLock(item, true, scheduledTime);
}
private bool ArchiveLogs(ServerFilesystemInfo archiveFs)
{
try
{
using (ServerExecutionContext context = new ServerExecutionContext())
{
string archivePath = Path.Combine(archiveFs.Filesystem.FilesystemPath, "ApplicationLog");
ApplicationLogSelectCriteria criteria = new ApplicationLogSelectCriteria();
criteria.Timestamp.SortAsc(0);
IApplicationLogEntityBroker broker = context.ReadContext.GetBroker <IApplicationLogEntityBroker>();
ApplicationLog firstLog = broker.FindOne(criteria);
if (firstLog == null)
{
return(true);
}
DateTime currentCutOffTime = firstLog.Timestamp.AddMinutes(5);
int cachedDays = ServiceLockSettings.Default.ApplicationLogCachedDays;
if (cachedDays < 0)
{
cachedDays = 0;
}
DateTime cutOffTime = Platform.Time.Date.AddDays(cachedDays * -1);
if (currentCutOffTime > cutOffTime)
{
return(true);
}
using (
ImageServerLogWriter <ApplicationLog> writer =
new ImageServerLogWriter <ApplicationLog>(archivePath, "ApplicationLog"))
{
while (currentCutOffTime < cutOffTime)
{
if (!ArchiveTimeRange(writer, currentCutOffTime))
{
writer.FlushLog();
return(false);
}
currentCutOffTime = currentCutOffTime.AddMinutes(5);
}
// Now flush the last potential 5 minutes.
if (!ArchiveTimeRange(writer, cutOffTime))
{
writer.FlushLog();
return(false);
}
writer.FlushLog();
}
return(true);
}
}
catch (Exception e)
{
Platform.Log(LogLevel.Error, e, "Unexpected exception when writing log file.");
return(false);
}
}
/// <summary>
/// Updates the 'State' of the filesystem associated with the 'FilesystemDelete' <see cref="ServiceLock"/> item
/// </summary>
/// <param name="item"></param>
/// <param name="fs"></param>
private static void UpdateState(Model.ServiceLock item, ServerFilesystemInfo fs)
{
FilesystemState state = null;
if (item.State != null && item.State.DocumentElement!=null)
{
//load from datatabase
state = XmlUtils.Deserialize<FilesystemState>(item.State.DocumentElement);
}
if (state == null)
state = new FilesystemState();
if (fs.AboveHighWatermark)
{
// we don't want to generate alert if the filesystem is offline or not accessible.
if (fs.Online && (fs.Readable || fs.Writeable))
{
TimeSpan ALERT_INTERVAL = TimeSpan.FromMinutes(ServiceLockSettings.Default.HighWatermarkAlertInterval);
if (state.AboveHighWatermarkTimestamp == null)
state.AboveHighWatermarkTimestamp = Platform.Time;
TimeSpan elapse = (state.LastHighWatermarkAlertTimestamp != null) ? Platform.Time - state.LastHighWatermarkAlertTimestamp.Value : Platform.Time - state.AboveHighWatermarkTimestamp.Value;
if (elapse.Duration() >= ALERT_INTERVAL)
{
ServerPlatform.Alert(AlertCategory.System, AlertLevel.Warning, "Filesystem",
AlertTypeCodes.LowResources, null, TimeSpan.Zero,
SR.AlertFilesystemAboveHW,
fs.Filesystem.Description,
TimeSpanFormatter.Format(Platform.Time - state.AboveHighWatermarkTimestamp.Value, true));
state.LastHighWatermarkAlertTimestamp = Platform.Time;
}
}
else
{
state.AboveHighWatermarkTimestamp = null;
state.LastHighWatermarkAlertTimestamp = null;
}
}
else
{
state.AboveHighWatermarkTimestamp = null;
state.LastHighWatermarkAlertTimestamp = null;
}
XmlDocument stateXml = new XmlDocument();
stateXml.AppendChild(stateXml.ImportNode(XmlUtils.Serialize(state), true));
IPersistentStore store = PersistentStoreRegistry.GetDefaultStore();
using (IUpdateContext ctx = store.OpenUpdateContext(UpdateContextSyncMode.Flush))
{
ServiceLockUpdateColumns columns = new ServiceLockUpdateColumns();
columns.State = stateXml;
IServiceLockEntityBroker broker = ctx.GetBroker<IServiceLockEntityBroker>();
broker.Update(item.GetKey(), columns);
ctx.Commit();
}
}
private bool ArchiveLogs(ServerFilesystemInfo archiveFs)
{
try
{
using (ServerExecutionContext context = new ServerExecutionContext())
{
string archivePath = Path.Combine(archiveFs.Filesystem.FilesystemPath, "ApplicationLog");
ApplicationLogSelectCriteria criteria = new ApplicationLogSelectCriteria();
criteria.Timestamp.SortAsc(0);
IApplicationLogEntityBroker broker = context.ReadContext.GetBroker<IApplicationLogEntityBroker>();
ApplicationLog firstLog = broker.FindOne(criteria);
if (firstLog == null)
return true;
DateTime currentCutOffTime = firstLog.Timestamp.AddMinutes(5);
int cachedDays = ServiceLockSettings.Default.ApplicationLogCachedDays;
if (cachedDays < 0) cachedDays = 0;
DateTime cutOffTime = Platform.Time.Date.AddDays(cachedDays*-1);
if (currentCutOffTime > cutOffTime)
return true;
using (
ImageServerLogWriter<ApplicationLog> writer =
new ImageServerLogWriter<ApplicationLog>(archivePath, "ApplicationLog"))
{
while (currentCutOffTime < cutOffTime)
{
if (!ArchiveTimeRange(writer, currentCutOffTime))
{
writer.FlushLog();
return false;
}
currentCutOffTime = currentCutOffTime.AddMinutes(5);
}
// Now flush the last potential 5 minutes.
if (!ArchiveTimeRange(writer, cutOffTime))
{
writer.FlushLog();
return false;
}
writer.FlushLog();
}
return true;
}
}
catch (Exception e)
{
Platform.Log(LogLevel.Error, e, "Unexpected exception when writing log file.");
return false;
}
}
/// <summary>
/// Do the actual StudyPurge
/// </summary>
/// <param name="item"></param>
/// <param name="fs">The filesystem being worked on.</param>
private void DoStudyPurge(ServerFilesystemInfo fs, Model.ServiceLock item)
{
DateTime deleteTime = Platform.Time;
FilesystemQueueTypeEnum type = FilesystemQueueTypeEnum.PurgeStudy;
while (_bytesToRemove > 0)
{
Platform.Log(LogLevel.Debug,
"{1:0.0} MBs needs to be removed from '{0}'. Querying for studies that can be purged",
fs.Filesystem.Description, _bytesToRemove / (1024 * 1024));
IList<FilesystemQueue> list =
GetFilesystemQueueCandidates(item, deleteTime, type, false);
if (list.Count > 0)
{
ProcessStudyPurgeCandidates(list);
if (CancelPending) break;
}
else
{
// No candidates
break;
}
}
}
/// <summary>
/// Do the actual Study migration.
/// </summary>
/// <param name="fs">The filesystem</param>
/// <param name="item">The ServiceLock item being processed.</param>
private void DoStudyMigrate( ServerFilesystemInfo fs, Model.ServiceLock item)
{
FilesystemQueueTypeEnum type = FilesystemQueueTypeEnum.TierMigrate;
while (_bytesToRemove > 0)
{
TimeSpanStatistics queryTime = new TimeSpanStatistics("Query Time");
queryTime.Start();
IList<FilesystemQueue> list = GetFilesystemQueueCandidates(item, Platform.Time, type, false);
queryTime.End();
Platform.Log(LogLevel.Info,
"{1:0.0} MBs needs to be removed from '{0}'. Found {2} studies that can be migrated in {3} seconds",
fs.Filesystem.Description, _bytesToRemove / (1024 * 1024),
list.Count,
queryTime.Value.TotalSeconds);
if (list.Count > 0)
{
ProcessStudyMigrateCandidates(list);
if (CancelPending) break;
}
else
{
break;
}
}
}
/// <summary>
/// Updates the 'State' of the filesystem associated with the 'FilesystemDelete' <see cref="ServiceLock"/> item
/// </summary>
/// <param name="item"></param>
/// <param name="fs"></param>
private static void UpdateState(Model.ServiceLock item, ServerFilesystemInfo fs)
{
FilesystemState state = null;
if (item.State != null && item.State.DocumentElement != null)
{
//load from datatabase
state = XmlUtils.Deserialize <FilesystemState>(item.State.DocumentElement);
}
if (state == null)
{
state = new FilesystemState();
}
if (fs.AboveHighWatermark)
{
// we don't want to generate alert if the filesystem is offline or not accessible.
if (fs.Online && (fs.Readable || fs.Writeable))
{
TimeSpan ALERT_INTERVAL = TimeSpan.FromMinutes(ServiceLockSettings.Default.HighWatermarkAlertInterval);
if (state.AboveHighWatermarkTimestamp == null)
{
state.AboveHighWatermarkTimestamp = Platform.Time;
}
TimeSpan elapse = (state.LastHighWatermarkAlertTimestamp != null) ? Platform.Time - state.LastHighWatermarkAlertTimestamp.Value : Platform.Time - state.AboveHighWatermarkTimestamp.Value;
if (elapse.Duration() >= ALERT_INTERVAL)
{
ServerPlatform.Alert(AlertCategory.System, AlertLevel.Warning, "Filesystem",
AlertTypeCodes.LowResources, null, TimeSpan.Zero,
SR.AlertFilesystemAboveHW,
fs.Filesystem.Description,
TimeSpanFormatter.Format(Platform.Time - state.AboveHighWatermarkTimestamp.Value, true));
state.LastHighWatermarkAlertTimestamp = Platform.Time;
}
}
else
{
state.AboveHighWatermarkTimestamp = null;
state.LastHighWatermarkAlertTimestamp = null;
}
}
else
{
state.AboveHighWatermarkTimestamp = null;
state.LastHighWatermarkAlertTimestamp = null;
}
XmlDocument stateXml = new XmlDocument();
stateXml.AppendChild(stateXml.ImportNode(XmlUtils.Serialize(state), true));
IPersistentStore store = PersistentStoreRegistry.GetDefaultStore();
using (IUpdateContext ctx = store.OpenUpdateContext(UpdateContextSyncMode.Flush))
{
ServiceLockUpdateColumns columns = new ServiceLockUpdateColumns();
columns.State = stateXml;
IServiceLockEntityBroker broker = ctx.GetBroker <IServiceLockEntityBroker>();
broker.Update(item.GetKey(), columns);
ctx.Commit();
}
}
private bool ArchiveLogs(ServerFilesystemInfo archiveFs)
{
string archivePath = Path.Combine(archiveFs.Filesystem.FilesystemPath, "AlertLog");
DateTime cutOffTime = Platform.Time.Date.AddDays(ServiceLockSettings.Default.AlertCachedDays * -1);
AlertSelectCriteria criteria = new AlertSelectCriteria();
criteria.InsertTime.LessThan(cutOffTime);
criteria.InsertTime.SortAsc(0);
using (ServerExecutionContext context = new ServerExecutionContext())
{
IAlertEntityBroker broker = context.ReadContext.GetBroker <IAlertEntityBroker>();
ImageServerLogWriter <Alert> writer = new ImageServerLogWriter <Alert>(archivePath, "Alert");
List <ServerEntityKey> keyList = new List <ServerEntityKey>(500);
try
{
broker.Find(criteria, delegate(Alert result)
{
keyList.Add(result.Key);
// If configured, don't flush to disk. We just delete the contents of keyList below.
if (!ServiceLockSettings.Default.AlertDelete)
{
if (writer.WriteLog(result, result.InsertTime))
{
// The logs been flushed, delete the log entries cached.
using (
IUpdateContext update =
PersistentStoreRegistry.GetDefaultStore().OpenUpdateContext(UpdateContextSyncMode.Flush)
)
{
IApplicationLogEntityBroker updateBroker =
update.GetBroker <IApplicationLogEntityBroker>();
foreach (ServerEntityKey key in keyList)
{
updateBroker.Delete(key);
}
update.Commit();
}
keyList = new List <ServerEntityKey>();
}
}
});
writer.FlushLog();
if (keyList.Count > 0)
{
using (
IUpdateContext update =
PersistentStoreRegistry.GetDefaultStore().OpenUpdateContext(UpdateContextSyncMode.Flush))
{
IAlertEntityBroker updateBroker = update.GetBroker <IAlertEntityBroker>();
foreach (ServerEntityKey key in keyList)
{
updateBroker.Delete(key);
}
update.Commit();
}
}
}
catch (Exception e)
{
Platform.Log(LogLevel.Error, e, "Unexpected exception when purging Alert log files.");
writer.Dispose();
return(false);
}
writer.Dispose();
return(true);
}
}
/// <summary>
/// Migrates the study to new tier
/// </summary>
/// <param name="storage"></param>
/// <param name="newFilesystem"></param>
private void DoMigrateStudy(StudyStorageLocation storage, ServerFilesystemInfo newFilesystem)
{
Platform.CheckForNullReference(storage, "storage");
Platform.CheckForNullReference(newFilesystem, "newFilesystem");
TierMigrationStatistics stat = new TierMigrationStatistics {
StudyInstanceUid = storage.StudyInstanceUid
};
stat.ProcessSpeed.Start();
StudyXml studyXml = storage.LoadStudyXml();
stat.StudySize = (ulong)studyXml.GetStudySize();
Platform.Log(LogLevel.Info, "About to migrate study {0} from {1} to {2}",
storage.StudyInstanceUid, storage.FilesystemTierEnum, newFilesystem.Filesystem.Description);
string newPath = Path.Combine(newFilesystem.Filesystem.FilesystemPath, storage.PartitionFolder);
DateTime startTime = Platform.Time;
DateTime lastLog = Platform.Time;
int fileCounter = 0;
ulong bytesCopied = 0;
long instanceCountInXml = studyXml.NumberOfStudyRelatedInstances;
using (ServerCommandProcessor processor = new ServerCommandProcessor("Migrate Study"))
{
TierMigrationContext context = new TierMigrationContext
{
OriginalStudyLocation = storage,
Destination = newFilesystem
};
// The multiple CreateDirectoryCommands are done so that rollback of the directories being created happens properly if either of the directories already exist.
var origFolder = context.OriginalStudyLocation.GetStudyPath();
processor.AddCommand(new CreateDirectoryCommand(newPath));
newPath = Path.Combine(newPath, context.OriginalStudyLocation.StudyFolder);
processor.AddCommand(new CreateDirectoryCommand(newPath));
newPath = Path.Combine(newPath, context.OriginalStudyLocation.StudyInstanceUid);
// don't create this directory so that it won't be backed up by MoveDirectoryCommand
var copyDirCommand = new CopyDirectoryCommand(origFolder, newPath,
delegate(string path)
{
// Update the progress. This is useful if the migration takes long time to complete.
FileInfo file = new FileInfo(path);
bytesCopied += (ulong)file.Length;
fileCounter++;
if (file.Extension != null && file.Extension.Equals(ServerPlatform.DicomFileExtension, StringComparison.InvariantCultureIgnoreCase))
{
TimeSpan elapsed = Platform.Time - lastLog;
TimeSpan totalElapsed = Platform.Time - startTime;
double speedInMBPerSecond = 0;
if (totalElapsed.TotalSeconds > 0)
{
speedInMBPerSecond = (bytesCopied / 1024f / 1024f) / totalElapsed.TotalSeconds;
}
if (elapsed > TimeSpan.FromSeconds(WorkQueueSettings.Instance.TierMigrationProgressUpdateInSeconds))
{
#region Log Progress
StringBuilder stats = new StringBuilder();
if (instanceCountInXml != 0)
{
float pct = (float)fileCounter / instanceCountInXml;
stats.AppendFormat("{0} files moved [{1:0.0}MB] since {2} ({3:0}% completed). Speed={4:0.00}MB/s",
fileCounter, bytesCopied / 1024f / 1024f, startTime, pct * 100, speedInMBPerSecond);
}
else
{
stats.AppendFormat("{0} files moved [{1:0.0}MB] since {2}. Speed={3:0.00}MB/s",
fileCounter, bytesCopied / 1024f / 1024f, startTime, speedInMBPerSecond);
}
Platform.Log(LogLevel.Info, "Tier migration for study {0}: {1}", storage.StudyInstanceUid, stats.ToString());
try
{
using (IUpdateContext ctx = PersistentStoreRegistry.GetDefaultStore().OpenUpdateContext(UpdateContextSyncMode.Flush))
{
IWorkQueueEntityBroker broker = ctx.GetBroker <IWorkQueueEntityBroker>();
WorkQueueUpdateColumns parameters = new WorkQueueUpdateColumns
{
FailureDescription = stats.ToString()
};
broker.Update(WorkQueueItem.GetKey(), parameters);
ctx.Commit();
}
}
catch
{
// can't log the progress so far... just ignore it
}
finally
{
lastLog = DateTime.Now;
}
#endregion
}
}
});
processor.AddCommand(copyDirCommand);
DeleteDirectoryCommand delDirCommand = new DeleteDirectoryCommand(origFolder, false)
{
RequiresRollback = false
};
processor.AddCommand(delDirCommand);
TierMigrateDatabaseUpdateCommand updateDbCommand = new TierMigrateDatabaseUpdateCommand(context);
processor.AddCommand(updateDbCommand);
Platform.Log(LogLevel.Info, "Start migrating study {0}.. expecting {1} to be moved", storage.StudyInstanceUid, ByteCountFormatter.Format(stat.StudySize));
if (!processor.Execute())
{
if (processor.FailureException != null)
{
throw processor.FailureException;
}
throw new ApplicationException(processor.FailureReason);
}
stat.DBUpdate = updateDbCommand.Statistics;
stat.CopyFiles = copyDirCommand.CopySpeed;
stat.DeleteDirTime = delDirCommand.Statistics;
}
stat.ProcessSpeed.SetData(bytesCopied);
stat.ProcessSpeed.End();
Platform.Log(LogLevel.Info, "Successfully migrated study {0} from {1} to {2} in {3} [ {4} files, {5} @ {6}, DB Update={7}, Remove Dir={8}]",
storage.StudyInstanceUid,
storage.FilesystemTierEnum,
newFilesystem.Filesystem.FilesystemTierEnum,
TimeSpanFormatter.Format(stat.ProcessSpeed.ElapsedTime),
fileCounter,
ByteCountFormatter.Format(bytesCopied),
stat.CopyFiles.FormattedValue,
stat.DBUpdate.FormattedValue,
stat.DeleteDirTime.FormattedValue);
string originalPath = storage.GetStudyPath();
if (Directory.Exists(storage.GetStudyPath()))
{
Platform.Log(LogLevel.Info, "Original study folder could not be deleted. It must be cleaned up manually: {0}", originalPath);
ServerPlatform.Alert(AlertCategory.Application, AlertLevel.Warning, WorkQueueItem.WorkQueueTypeEnum.ToString(), 1000, GetWorkQueueContextData(WorkQueueItem), TimeSpan.Zero,
"Study has been migrated to a new tier. Original study folder must be cleaned up manually: {0}", originalPath);
}
UpdateAverageStatistics(stat);
}
protected override void OnProcess(Model.ServiceLock item)
{
ServiceLockSettings settings = ServiceLockSettings.Default;
ServerFilesystemInfo archiveFilesystem = null;
if (item.FilesystemKey != null)
{
archiveFilesystem = FilesystemMonitor.Instance.GetFilesystemInfo(item.FilesystemKey);
if (archiveFilesystem == null)
{
Platform.Log(LogLevel.Warn, "Filesystem for archiving alerts is no longer valid. Assigning new filesystem.");
item.FilesystemKey = null;
UpdateFilesystemKey(item);
}
}
if (archiveFilesystem == null)
{
ServerFilesystemInfo selectedFs = null;
foreach (ServerFilesystemInfo fs in FilesystemMonitor.Instance.GetFilesystems())
{
if (selectedFs == null)
{
selectedFs = fs;
}
else if (fs.Filesystem.FilesystemTierEnum.Enum > selectedFs.Filesystem.FilesystemTierEnum.Enum)
{
selectedFs = fs; // Lower tier
}
else if ((fs.Filesystem.FilesystemTierEnum.Enum == selectedFs.Filesystem.FilesystemTierEnum.Enum) &&
(fs.FreeBytes > selectedFs.FreeBytes))
{
selectedFs = fs; // same tier
}
}
if (selectedFs == null)
{
Platform.Log(LogLevel.Info, "No writable filesystems for archiving Alerts, delaying archival.");
UnlockServiceLock(item, true, Platform.Time.AddHours(2));
return;
}
item.FilesystemKey = selectedFs.Filesystem.Key;
archiveFilesystem = selectedFs;
UpdateFilesystemKey(item);
Platform.Log(LogLevel.Info, "Selecting Filesystem {0} for archiving of Alerts",
selectedFs.Filesystem.Description);
}
DateTime scheduledTime;
if (!archiveFilesystem.Writeable)
{
Platform.Log(LogLevel.Info, "Filesystem {0} is not writeable. Unable to archive Alert log files.", archiveFilesystem.Filesystem.Description);
scheduledTime = Platform.Time.AddMinutes(settings.AlertRecheckDelay);
}
else
{
Platform.Log(LogLevel.Info, "Checking for Alert logs to purge to: {0}",
archiveFilesystem.Filesystem.Description);
if (!ArchiveLogs(archiveFilesystem))
{
scheduledTime = Platform.Time.AddMinutes(settings.AlertRecheckDelay);
}
else
{
DateTime tomorrow = Platform.Time.Date.AddDays(1);
// Set for 12:01 tomorrow morning
scheduledTime = new DateTime(tomorrow.Year, tomorrow.Month, tomorrow.Day, 0, 1, 0);
Platform.Log(LogLevel.Info, "Completed archival of Alert logs, rescheduling log archive for {0}",
scheduledTime.ToLongTimeString());
}
}
UnlockServiceLock(item, true, scheduledTime);
}
public void RestoreNearlineStudy(RestoreQueue queueItem, string zipFile, string studyFolder)
{
ServerFilesystemInfo fs = _hsmArchive.Selector.SelectFilesystem();
if (fs == null)
{
DateTime scheduleTime = Platform.Time.AddMinutes(5);
Platform.Log(LogLevel.Error, "No writeable filesystem for restore, rescheduling restore request to {0}", scheduleTime);
queueItem.FailureDescription = "No writeable filesystem for restore, rescheduling restore request";
_hsmArchive.UpdateRestoreQueue(queueItem, RestoreQueueStatusEnum.Pending, scheduleTime);
return;
}
string destinationFolder = Path.Combine(fs.Filesystem.FilesystemPath, _hsmArchive.ServerPartition.PartitionFolder);
StudyStorageLocation restoredLocation = null;
try
{
using (var processor = new ServerCommandProcessor("HSM Restore Offline Study"))
{
processor.AddCommand(new CreateDirectoryCommand(destinationFolder));
destinationFolder = Path.Combine(destinationFolder, studyFolder);
processor.AddCommand(new CreateDirectoryCommand(destinationFolder));
destinationFolder = Path.Combine(destinationFolder, _studyStorage.StudyInstanceUid);
processor.AddCommand(new CreateDirectoryCommand(destinationFolder));
processor.AddCommand(new ExtractZipCommand(zipFile, destinationFolder));
// We rebuild the StudyXml, in case any settings or issues have happened since archival
processor.AddCommand(new RebuildStudyXmlCommand(_studyStorage.StudyInstanceUid, destinationFolder));
// Apply the rules engine.
var context =
new ServerActionContext(null, fs.Filesystem.GetKey(), _hsmArchive.ServerPartition,
queueItem.StudyStorageKey)
{
CommandProcessor = processor
};
processor.AddCommand(
new ApplyRulesCommand(destinationFolder, _studyStorage.StudyInstanceUid, context));
// Do the actual insert into the DB
var insertStorageCommand = new InsertFilesystemStudyStorageCommand(
_hsmArchive.PartitionArchive.ServerPartitionKey,
_studyStorage.StudyInstanceUid,
studyFolder,
fs.Filesystem.GetKey(), _syntax);
processor.AddCommand(insertStorageCommand);
if (!processor.Execute())
{
Platform.Log(LogLevel.Error, "Unexpected error processing restore request for {0} on archive {1}",
_studyStorage.StudyInstanceUid, _hsmArchive.PartitionArchive.Description);
queueItem.FailureDescription = processor.FailureReason;
_hsmArchive.UpdateRestoreQueue(queueItem, RestoreQueueStatusEnum.Failed, Platform.Time);
}
else
{
restoredLocation = insertStorageCommand.Location;
// Unlock the Queue Entry
using (
IUpdateContext update = PersistentStoreRegistry.GetDefaultStore().OpenUpdateContext(UpdateContextSyncMode.Flush))
{
bool retVal = _hsmArchive.UpdateRestoreQueue(update, queueItem, RestoreQueueStatusEnum.Completed, Platform.Time.AddSeconds(60));
var studyLock = update.GetBroker <ILockStudy>();
var parms = new LockStudyParameters
{
StudyStorageKey = queueItem.StudyStorageKey,
QueueStudyStateEnum = QueueStudyStateEnum.Idle
};
retVal = retVal && studyLock.Execute(parms);
if (!parms.Successful || !retVal)
{
string message =
String.Format("Study {0} on partition {1} failed to unlock.", _studyStorage.StudyInstanceUid,
_hsmArchive.ServerPartition.Description);
Platform.Log(LogLevel.Info, message);
throw new ApplicationException(message);
}
update.Commit();
Platform.Log(LogLevel.Info, "Successfully restored study: {0} on archive {1}", _studyStorage.StudyInstanceUid,
_hsmArchive.PartitionArchive.Description);
OnStudyRestored(restoredLocation);
}
}
}
}
catch (StudyIntegrityValidationFailure ex)
{
Debug.Assert(restoredLocation != null);
// study has been restored but it seems corrupted. Need to reprocess it.
ReprocessStudy(restoredLocation, ex.Message);
}
catch (Exception e)
{
Platform.Log(LogLevel.Error, e, "Unexpected exception processing restore request for {0} on archive {1}",
_studyStorage.StudyInstanceUid, _hsmArchive.PartitionArchive.Description);
_hsmArchive.UpdateRestoreQueue(queueItem, RestoreQueueStatusEnum.Failed, Platform.Time);
}
}
private bool ArchiveLogs(ServerFilesystemInfo archiveFs)
{
string archivePath = Path.Combine(archiveFs.Filesystem.FilesystemPath, "AlertLog");
DateTime cutOffTime = Platform.Time.Date.AddDays(ServiceLockSettings.Default.AlertCachedDays*-1);
AlertSelectCriteria criteria = new AlertSelectCriteria();
criteria.InsertTime.LessThan(cutOffTime);
criteria.InsertTime.SortAsc(0);
using (ServerExecutionContext context = new ServerExecutionContext())
{
IAlertEntityBroker broker = context.ReadContext.GetBroker<IAlertEntityBroker>();
ImageServerLogWriter<Alert> writer = new ImageServerLogWriter<Alert>(archivePath, "Alert");
List<ServerEntityKey> keyList = new List<ServerEntityKey>(500);
try
{
broker.Find(criteria, delegate(Alert result)
{
keyList.Add(result.Key);
// If configured, don't flush to disk. We just delete the contents of keyList below.
if (!ServiceLockSettings.Default.AlertDelete)
{
if (writer.WriteLog(result, result.InsertTime))
{
// The logs been flushed, delete the log entries cached.
using (
IUpdateContext update =
PersistentStoreRegistry.GetDefaultStore().OpenUpdateContext(UpdateContextSyncMode.Flush)
)
{
IApplicationLogEntityBroker updateBroker =
update.GetBroker<IApplicationLogEntityBroker>();
foreach (ServerEntityKey key in keyList)
updateBroker.Delete(key);
update.Commit();
}
keyList = new List<ServerEntityKey>();
}
}
});
writer.FlushLog();
if (keyList.Count > 0)
{
using (
IUpdateContext update =
PersistentStoreRegistry.GetDefaultStore().OpenUpdateContext(UpdateContextSyncMode.Flush))
{
IAlertEntityBroker updateBroker = update.GetBroker<IAlertEntityBroker>();
foreach (ServerEntityKey key in keyList)
updateBroker.Delete(key);
update.Commit();
}
}
}
catch (Exception e)
{
Platform.Log(LogLevel.Error, e, "Unexpected exception when purging Alert log files.");
writer.Dispose();
return false;
}
writer.Dispose();
return true;
}
}
private void PurgeStudies(Model.ServiceLock item, ServerFilesystemInfo fs)
{
Platform.Log(LogLevel.Info, "Starting query for Filesystem Purge candidates on '{0}'.",
fs.Filesystem.Description);
try
{
DoStudyPurge(fs, item);
}
catch (Exception e)
{
Platform.Log(LogLevel.Error, e, "Unexpected exception when processing StudyDelete records.");
}
Platform.Log(LogLevel.Info, "{0} studies have been scheduled for purging from filesystem '{1}'",
_studiesPurged, fs.Filesystem.Description);
}
