public void ProcessJobs_ShallRetryUpTo3Times()
{
var countDownEvent = new CountdownEvent(3);
_jobStore.Setup(p => p.Add(It.IsAny <InferenceJob>(), It.IsAny <bool>()))
.Callback(() =>
{
countDownEvent.Signal();
})
.Throws(new System.Exception());
_configuration.AeTitle = _aeTitle;
_configuration.ProcessorSettings.Add("timeout", "1");
_configuration.ProcessorSettings.Add("jobRetryDelay", "100");
_configuration.ProcessorSettings.Add("pipeline-first", "PIPELINE1");
var processor = new AeTitleJobProcessor(_configuration, _notificationService, _loggerFactory.Object, _jobStore.Object, _cleanupQueue.Object, _dicomToolkit.Object, _cancellationTokenSource.Token);
_notificationService.NewInstanceStored(_instances.First());
Assert.True(countDownEvent.Wait(10000));
_jobStore.Verify(p => p.Add(It.IsAny <InferenceJob>(), false), Times.Exactly(3));
_logger.VerifyLogging($"Failed to submit job, will retry later: PatientId={_instances.First().PatientId}, Study={_instances.First().StudyInstanceUid}", LogLevel.Information, Times.AtLeast(1));
_logger.VerifyLogging($"Failed to submit job after 3 retries: PatientId={_instances.First().PatientId}, Study={_instances.First().StudyInstanceUid}", LogLevel.Error, Times.Once());
}
/// <summary>
/// Saves specified <code>InstanceStorage</code> to disk.
/// </summary>
/// <param name="request">Instance of <code>DicomCStoreRequest</code> to be stored to disk.</param>
/// <param name="instanceStorage">Instance of <code>InstanceStorage</code></param>
public void Save(DicomCStoreRequest request, InstanceStorageInfo instanceStorage)
{
Guard.Against.Null(instanceStorage, nameof(instanceStorage));
if (ShouldBeIgnored(instanceStorage.SopClassUid))
{
_logger.Log(LogLevel.Warning, "Instance with SOP Class {0} ignored based on configured AET {1}", instanceStorage.SopClassUid, Configuration.AeTitle);
return;
}
Policy.Handle <Exception>()
.WaitAndRetry(3,
(retryAttempt) =>
{
return(retryAttempt == 1 ? TimeSpan.FromMilliseconds(250) : TimeSpan.FromMilliseconds(500));
},
(exception, retryCount, context) =>
{
_logger.Log(LogLevel.Error, "Failed to save instance, retry count={retryCount}: {exception}", retryCount, exception);
})
.Execute(() =>
{
if (ShouldSaveInstance(instanceStorage))
{
_logger.Log(LogLevel.Information, "Saving {path}.", instanceStorage.InstanceStorageFullPath);
_dicomToolkit.Save(request.File, instanceStorage.InstanceStorageFullPath);
_logger.Log(LogLevel.Debug, "Instance saved successfully.");
_instanceStoredNotificationService.NewInstanceStored(instanceStorage);
_logger.Log(LogLevel.Information, "Instance stored and notified successfully.");
}
});
}
