public void ProcessJobs_ShallRetryUpTo3Times() { var countDownEvent = new CountdownEvent(3); _jobStore.Setup(p => p.Add(It.IsAny <InferenceJob>(), It.IsAny <bool>())) .Callback(() => { countDownEvent.Signal(); }) .Throws(new System.Exception()); _configuration.AeTitle = _aeTitle; _configuration.ProcessorSettings.Add("timeout", "1"); _configuration.ProcessorSettings.Add("jobRetryDelay", "100"); _configuration.ProcessorSettings.Add("pipeline-first", "PIPELINE1"); var processor = new AeTitleJobProcessor(_configuration, _notificationService, _loggerFactory.Object, _jobStore.Object, _cleanupQueue.Object, _dicomToolkit.Object, _cancellationTokenSource.Token); _notificationService.NewInstanceStored(_instances.First()); Assert.True(countDownEvent.Wait(10000)); _jobStore.Verify(p => p.Add(It.IsAny <InferenceJob>(), false), Times.Exactly(3)); _logger.VerifyLogging($"Failed to submit job, will retry later: PatientId={_instances.First().PatientId}, Study={_instances.First().StudyInstanceUid}", LogLevel.Information, Times.AtLeast(1)); _logger.VerifyLogging($"Failed to submit job after 3 retries: PatientId={_instances.First().PatientId}, Study={_instances.First().StudyInstanceUid}", LogLevel.Error, Times.Once()); }
/// <summary> /// Saves specified <code>InstanceStorage</code> to disk. /// </summary> /// <param name="request">Instance of <code>DicomCStoreRequest</code> to be stored to disk.</param> /// <param name="instanceStorage">Instance of <code>InstanceStorage</code></param> public void Save(DicomCStoreRequest request, InstanceStorageInfo instanceStorage) { Guard.Against.Null(instanceStorage, nameof(instanceStorage)); if (ShouldBeIgnored(instanceStorage.SopClassUid)) { _logger.Log(LogLevel.Warning, "Instance with SOP Class {0} ignored based on configured AET {1}", instanceStorage.SopClassUid, Configuration.AeTitle); return; } Policy.Handle <Exception>() .WaitAndRetry(3, (retryAttempt) => { return(retryAttempt == 1 ? TimeSpan.FromMilliseconds(250) : TimeSpan.FromMilliseconds(500)); }, (exception, retryCount, context) => { _logger.Log(LogLevel.Error, "Failed to save instance, retry count={retryCount}: {exception}", retryCount, exception); }) .Execute(() => { if (ShouldSaveInstance(instanceStorage)) { _logger.Log(LogLevel.Information, "Saving {path}.", instanceStorage.InstanceStorageFullPath); _dicomToolkit.Save(request.File, instanceStorage.InstanceStorageFullPath); _logger.Log(LogLevel.Debug, "Instance saved successfully."); _instanceStoredNotificationService.NewInstanceStored(instanceStorage); _logger.Log(LogLevel.Information, "Instance stored and notified successfully."); } }); }