public void TestFactoryConstruction() { var rootDirectory = new DirectoryInfo(TestContext.CurrentContext.WorkDirectory); var layout = new SMICacheLayout(rootDirectory, new SMICachePathResolver("CT")); var downloadDirectory = layout.GetLoadCacheDirectory(new ThrowImmediatelyDataLoadEventListener()); Assert.AreEqual(downloadDirectory.FullName, Path.Combine(rootDirectory.FullName, "CT")); }
public override SMIDataChunk DoGetChunk(ICacheFetchRequest request, IDataLoadEventListener listener, GracefulCancellationToken cancellationToken) { listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, $"ProcessBasedCacheSource version is {typeof(ProcessBasedCacheSource).Assembly.GetName().Version}. Assembly is {typeof(ProcessBasedCacheSource).Assembly} ")); // Where we are putting the files var cacheDir = new LoadDirectory(Request.CacheProgress.LoadProgress.LoadMetadata.LocationOfFlatFiles).Cache; var cacheLayout = new SMICacheLayout(cacheDir, new SMICachePathResolver("ALL")); Chunk = new SMIDataChunk(Request) { FetchDate = Request.Start, Modality = "ALL", Layout = cacheLayout }; var workingDirectory = cacheLayout.GetLoadCacheDirectory(listener); listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Working directory is:" + workingDirectory)); listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Fetch Start is:" + request.Start)); listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Fetch End is:" + request.End)); listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Command is:" + Command)); listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Args template is:" + Args)); listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Datetime format is:" + TimeFormat)); string args = Args .Replace("%s", request.Start.ToString(TimeFormat)) .Replace("%e", request.End.ToString(TimeFormat)) .Replace("%d", workingDirectory.FullName); listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Args resolved is:" + args)); using (var p = new Process()) { p.StartInfo.FileName = Command; p.StartInfo.Arguments = args; p.StartInfo.UseShellExecute = false; p.StartInfo.RedirectStandardOutput = true; p.OutputDataReceived += (sender, a) => listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, a.Data)); p.Start(); p.BeginOutputReadLine(); p.WaitForExit(); listener.OnNotify(this, new NotifyEventArgs(p.ExitCode == 0 ? ProgressEventType.Information : ProgressEventType.Warning, "Process exited with code " + p.ExitCode)); if (p.ExitCode != 0 && ThrowOnNonZeroExitCode) { throw new Exception("Process exited with code " + p.ExitCode); } } return(Chunk); }
private void SaveSopInstance(DicomCStoreRequest request, SMICacheLayout cacheLayout, IDataLoadEventListener listener) { var instUid = request.SOPInstanceUID.UID; if (!request.HasDataset) { return; } // Create filepath and save var workingDirectory = cacheLayout.GetLoadCacheDirectory(listener); var filename = instUid + ".dcm"; var filepath = Path.Combine(workingDirectory.FullName, filename); request.File.Save(filepath); }
public override SMIDataChunk DoGetChunk(ICacheFetchRequest cacheRequest, IDataLoadEventListener listener, GracefulCancellationToken cancellationToken) { listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, $"CFindSource version is {typeof(CFindSource).Assembly.GetName().Version}. Assembly is {typeof(PACSSource).Assembly} ")); listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, $"Fo-Dicom version is {typeof(DicomClient).Assembly.GetName().Version}. Assembly is {typeof(DicomClient).Assembly} ")); var dicomConfiguration = GetConfiguration(); var requestSender = new DicomRequestSender(dicomConfiguration, listener); var dateFrom = Request.Start; var dateTo = Request.End; CachingSCP.LocalAet = LocalAETitle; CachingSCP.Listener = listener; if (PatientIdWhitelistColumnInfo != null && !IgnoreWhiteList) { GetWhitelist(listener); } //temp dir var cacheDir = new LoadDirectory(Request.CacheProgress.LoadProgress.LoadMetadata.LocationOfFlatFiles).Cache; var cacheLayout = new SMICacheLayout(cacheDir, new SMICachePathResolver(Modality)); Chunk = new SMIDataChunk(Request) { FetchDate = dateFrom, Modality = Modality, Layout = cacheLayout }; // Create filepath for the results of the C-Find var workingDirectory = cacheLayout.GetLoadCacheDirectory(listener); var filename = $"{dateFrom:yyyyMMddhhmmss}.csv"; var filepath = Path.Combine(workingDirectory.FullName, filename); var sw = new StreamWriter(filepath); var writer = new CsvWriter(sw, new CsvConfiguration(CultureInfo.CurrentCulture)); WriteHeaders(writer); DicomClient client = new DicomClient(dicomConfiguration.RemoteAetUri.Host, dicomConfiguration.RemoteAetUri.Port, false, dicomConfiguration.LocalAetTitle, dicomConfiguration.RemoteAetTitle); try { // Find a list of studies #region Query listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Requesting Studies from " + dateFrom + " to " + dateTo)); int responses = 0; var request = CreateStudyRequestByDateRangeForModality(dateFrom, dateTo, Modality); request.OnResponseReceived += (req, response) => { if (Filter(Whitelist, response)) { Interlocked.Increment(ref responses); WriteResult(writer, response); } }; requestSender.ThrottleRequest(request, client, cancellationToken.AbortToken); listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Debug, "Total filtered studies for " + dateFrom + " to " + dateTo + " is " + responses)); #endregion } finally { writer.Dispose(); } return(Chunk); }
public override SMIDataChunk DoGetChunk(ICacheFetchRequest cacheRequest, IDataLoadEventListener listener, GracefulCancellationToken cancellationToken) { #region assigns var dicomConfiguration = GetConfiguration(); var requestSender = new DicomRequestSender(dicomConfiguration, listener); var dateFrom = Request.Start; var dateTo = Request.End; var hasTransferTimedOut = false; CachingSCP.LocalAet = LocalAETitle; CachingSCP.Listener = listener; if (PatientIdWhitelistColumnInfo != null && !IgnoreWhiteList) { GetWhitelist(listener); } //temp dir var cacheDir = new LoadDirectory(Request.CacheProgress.LoadProgress.LoadMetadata.LocationOfFlatFiles).Cache; var cacheLayout = new SMICacheLayout(cacheDir, new SMICachePathResolver(Modality)); Chunk = new SMIDataChunk(Request) { FetchDate = dateFrom, Modality = Modality, Layout = cacheLayout }; // IOrder order = new ItemsBasedOrder(dateFrom, dateTo, PlacementMode.PlaceThenFill,OrderLevel, listener); IOrder order = new HierarchyBasedOrder(dateFrom, dateTo, PlacementMode.PlaceThenFill, OrderLevel, listener); IPicker picker = null; var pickerFilled = false; var transferTimeOutTimer = new Timer(dicomConfiguration.TransferTimeOutInMilliseconds); transferTimeOutTimer.Elapsed += (source, eventArgs) => { hasTransferTimedOut = true; listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Transfer Timeout Exception Generated")); throw new TimeoutException("Transfer Timeout Exception"); }; CachingSCP.OnEndProcessingCStoreRequest = (storeRequest, storeResponse) => { var item = new Item(storeRequest); transferTimeOutTimer.Reset(); if (picker != null) { picker.Fill(item); pickerFilled = picker.IsFilled(); } SaveSopInstance(storeRequest, cacheLayout, listener); listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Debug, "Stored sopInstance" + storeRequest.SOPInstanceUID.UID)); }; #endregion //helps with tyding up resources if we abort or through an exception and neatly avoids -> Access to disposed closure using (var server = (DicomServer <CachingSCP>)DicomServer.Create <CachingSCP>(dicomConfiguration.LocalAetUri.Port)) { try { // Find a list of studies #region Query listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Requesting Studies from " + dateFrom + " to " + dateTo)); var studyUids = new List <string>(); var request = CreateStudyRequestByDateRangeForModality(dateFrom, dateTo, Modality); request.OnResponseReceived += (req, response) => { if (Filter(_whitelist, response)) { studyUids.Add(response.Dataset.GetSingleValue <string>(DicomTag.StudyInstanceUID)); } }; requestSender.ThrottleRequest(request, cancellationToken); listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Debug, "Total filtered studies for " + dateFrom + " to " + dateTo + "is " + studyUids.Count)); foreach (var studyUid in studyUids) { listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Debug, "Sending series query for study" + studyUid)); var seriesUids = new List <string>(); request = CreateSeriesRequestByStudyUid(studyUid); request.OnResponseReceived += (req, response) => { if (response.Dataset == null) { return; } var seriesInstanceUID = response.Dataset.GetSingleValue <string>(DicomTag.SeriesInstanceUID); if (seriesInstanceUID != null) { seriesUids.Add(seriesInstanceUID); } }; requestSender.ThrottleRequest(request, cancellationToken); listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Debug, "Total series for " + studyUid + "is " + seriesUids.Count)); foreach (var seriesUid in seriesUids) { listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Debug, "Sending image query for series" + seriesUid)); request = CreateSopRequestBySeriesUid(seriesUid); int imageCount = 0; request.OnResponseReceived += (req, response) => { if (response.Dataset == null) { return; } var sopUid = response.Dataset.GetSingleValue <string>(DicomTag.SOPInstanceUID); var patientId = response.Dataset.GetSingleValue <string>(DicomTag.PatientID); if (sopUid != null && patientId != null) { //Place order order.Place(patientId, studyUid, seriesUid, sopUid); imageCount++; } }; requestSender.ThrottleRequest(request, cancellationToken); listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Debug, "Successfully finished image query for " + seriesUid + " Toal images in series = " + imageCount)); } listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Debug, "Successfully finished series query for " + studyUid)); } listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Debug, "Successfully finished query phase")); #endregion //go and get them #region Retrieval var transferStopwatch = new Stopwatch(); //start building request to fill orders //get the picker - the for loop avoids sleeping after all the transfers have finished and attempting dequeue on empty queue for (int delay = 0, transferTimerPollingPeriods; order.HasNextPicker() && !hasTransferTimedOut; delay = (int)(dicomConfiguration.TransferDelayFactor * transferTimerPollingPeriods * dicomConfiguration.TransferPollingInMilliseconds) + dicomConfiguration.TransferCooldownInMilliseconds ) { transferStopwatch.Restart(); //delay value in mills if (delay != 0) { listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Transfers sleeping for " + delay / 1000 + "seconds")); Task.Delay(delay, cancellationToken.AbortToken).Wait(cancellationToken.AbortToken); } //set this here prior to request pickerFilled = false; transferTimerPollingPeriods = 0; //get next picker picker = order.NextPicker(); // A CMove will be performed if the storescp exists and this storescp is known to the QRSCP: var cMoveRequest = picker.GetDicomCMoveRequest(LocalAETitle); /* this won't work which means we cannot enforce (low) priority * cMoveRequest.Priority=DicomPriority.Low;*/ cMoveRequest.OnResponseReceived += (requ, response) => { if (response.Status.State == DicomState.Pending) { listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Debug, "Request: " + requ.ToString() + "items remaining: " + response.Remaining)); } else if (response.Status.State == DicomState.Success) { listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Debug, "Request: " + requ.ToString() + "completed successfully")); } else if (response.Status.State == DicomState.Failure) { listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Debug, "Request: " + requ.ToString() + "failed to download: " + response.Failures)); } }; listener.OnProgress(this, new ProgressEventArgs(CMoveRequestToString(cMoveRequest), new ProgressMeasurement(picker.Filled(), ProgressType.Records, picker.Total()), transferStopwatch.Elapsed)); //do not use requestSender.ThrottleRequest(cMoveRequest, cancellationToken); //TODO is there any need to throtttle this request given its lifetime DicomClient client = new DicomClient(); requestSender.ThrottleRequest(cMoveRequest, client, cancellationToken); transferTimeOutTimer.Reset(); while (!pickerFilled && !hasTransferTimedOut) { Task.Delay(dicomConfiguration.TransferPollingInMilliseconds, cancellationToken.AbortToken) .Wait(cancellationToken.AbortToken); transferTimerPollingPeriods++; } transferTimeOutTimer.Stop(); client.Release(); listener.OnProgress(this, new ProgressEventArgs(CMoveRequestToString(cMoveRequest), new ProgressMeasurement(picker.Filled(), ProgressType.Records, picker.Total()), transferStopwatch.Elapsed)); } #endregion } finally { server.Stop(); } } return(Chunk); }
public override SMIDataChunk DoGetChunk(ICacheFetchRequest cacheRequest, IDataLoadEventListener listener, GracefulCancellationToken cancellationToken) { listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, $"PACSSource version is {typeof(PACSSource).Assembly.GetName().Version}. Assembly is {typeof(PACSSource).Assembly} ")); listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, $"Fo-Dicom version is {typeof(DicomClient).Assembly.GetName().Version}. Assembly is {typeof(DicomClient).Assembly} ")); var dicomConfiguration = GetConfiguration(); var requestSender = new DicomRequestSender(dicomConfiguration, listener); var dateFrom = Request.Start; var dateTo = Request.End; CachingSCP.LocalAet = LocalAETitle; CachingSCP.Listener = listener; if (PatientIdWhitelistColumnInfo != null && !IgnoreWhiteList) { GetWhitelist(listener); } //temp dir var cacheDir = new LoadDirectory(Request.CacheProgress.LoadProgress.LoadMetadata.LocationOfFlatFiles).Cache; var cacheLayout = new SMICacheLayout(cacheDir, new SMICachePathResolver(Modality)); Chunk = new SMIDataChunk(Request) { FetchDate = dateFrom, Modality = Modality, Layout = cacheLayout }; ConcurrentBag <StudyToFetch> studiesToOrder = new ConcurrentBag <StudyToFetch>(); CachingSCP.OnEndProcessingCStoreRequest = (storeRequest, storeResponse) => { SaveSopInstance(storeRequest, cacheLayout, listener); listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Debug, "Stored sopInstance" + storeRequest.SOPInstanceUID.UID)); }; //helps with tyding up resources if we abort or through an exception and neatly avoids -> Access to disposed closure using (var server = (DicomServer <CachingSCP>)DicomServer.Create <CachingSCP>(dicomConfiguration.LocalAetUri.Port)) { DicomClient client = new DicomClient(dicomConfiguration.RemoteAetUri.Host, dicomConfiguration.RemoteAetUri.Port, false, dicomConfiguration.LocalAetTitle, dicomConfiguration.RemoteAetTitle); try { // Find a list of studies #region Query listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Requesting Studies from " + dateFrom + " to " + dateTo)); var request = CreateStudyRequestByDateRangeForModality(dateFrom, dateTo, Modality); request.OnResponseReceived += (req, response) => { if (Filter(_whitelist, response)) { studiesToOrder.Add(new StudyToFetch(response.Dataset.GetSingleValue <string>(DicomTag.StudyInstanceUID))); } }; requestSender.ThrottleRequest(request, client, cancellationToken.AbortToken); listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Debug, "Total filtered studies for " + dateFrom + " to " + dateTo + "is " + studiesToOrder.Count)); #endregion //go and get them #region Retrieval var transferStopwatch = new Stopwatch(); StudyToFetch current; int consecutiveFailures = 0; //While we have things to fetch while (studiesToOrder.TryTake(out current)) { transferStopwatch.Restart(); //delay value in mills if (dicomConfiguration.TransferCooldownInMilliseconds != 0) { listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Transfers sleeping for " + dicomConfiguration.TransferCooldownInMilliseconds / 1000 + "seconds")); Task.Delay(dicomConfiguration.TransferCooldownInMilliseconds, cancellationToken.AbortToken).Wait(cancellationToken.AbortToken); } bool done = false; //Build fetch command that Study var cMoveRequest = CreateCMoveByStudyUid(LocalAETitle, current.StudyUid, listener); //Register callbacks cMoveRequest.OnResponseReceived += (requ, response) => { listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Debug, $"Got {response.Status.State} response for {requ}. Items remaining {response.Remaining}")); switch (response.Status.State) { case DicomState.Pending: case DicomState.Warning: // ignore break; case DicomState.Cancel: case DicomState.Failure: consecutiveFailures++; if (current.RetryCount < MaxRetries) { // put it back in the bag with a increased retry count current.RetryCount++; studiesToOrder.Add(current); } // final state done = true; break; case DicomState.Success: // final state consecutiveFailures = 0; done = true; break; } }; //send the command to the server //tell user what we are sending listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, CMoveRequestToString(cMoveRequest, current.RetryCount + 1))); //do not use requestSender.ThrottleRequest(cMoveRequest, cancellationToken); //TODO is there any need to throtttle this request given its lifetime requestSender.ThrottleRequest(cMoveRequest, client, cancellationToken.AbortToken); //enforce a minimum timeout var swStudyTransfer = Stopwatch.StartNew(); bool hasTransferTimedOut = false; do { Task.Delay(Math.Max(100, dicomConfiguration.TransferPollingInMilliseconds), cancellationToken.AbortToken) .Wait(cancellationToken.AbortToken); hasTransferTimedOut = swStudyTransfer.ElapsedMilliseconds > dicomConfiguration.TransferTimeOutInMilliseconds; }while(!done && !hasTransferTimedOut); // Study has finished being fetched (or timed out) if (hasTransferTimedOut) { listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Abandonning fetch of study " + current.StudyUid)); } if (consecutiveFailures > 5) { throw new Exception("Too many consecutive failures, giving up"); } // 1 failure = study not available, 2 failures = system is having a bad day? if (consecutiveFailures > 1) { //wait 4 minutes then 6 minutes then 8 minutes, eventually server will start responding again? int sleepFor = consecutiveFailures * 2 * 60_000; listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, $"Sleeping for {sleepFor}ms due to {consecutiveFailures} consecutive failures")); Task.Delay(sleepFor, cancellationToken.AbortToken) .Wait(cancellationToken.AbortToken); } } #endregion } finally { server.Stop(); } } return(Chunk); }