public async Task DeveloperForgetsEndPrepare() { var exceptionCount = 0; var failureCount = 0; var successCount = 0; var profile = RetryLogic .BeginPrepare() .Handle <FileNotFoundException>() .AtMost(10) .WithConstantInterval(TimeSpan.FromMilliseconds(0)) .OnException <FileNotFoundException>(e => ++ exceptionCount) .OnFailure(() => ++ failureCount); ////.EndPrepare(); // developer forgets tha line ////await RetryLogic.DoAsync(ActionThrowExceptionAsync, profile) try { await profile.DoAsync(ActionThrowExceptionAsync) .AtMost(5) .OnSuccess(x => { ++successCount; }) .RunAsync(); Assert.Fail("Should have thrown a InvalidOperationException"); } catch (InvalidOperationException ex) { // ok } }
public void Function_Should_Be_Tried_3Times_With_ExponentialInterval() { var exceptionCount = 0; var failureCount = 0; var successCount = 0; var watch = new Stopwatch(); watch.Start(); RetryLogic .Do <int>(this.ThrowException) .Handle <FileNotFoundException>() .AtMost(4) .WithExponentialInterval(TimeSpan.FromMilliseconds(2000), 2) .OnException <FileNotFoundException>(e => ++ exceptionCount) .OnFailure(() => ++ failureCount) .OnSuccess(x => { ++successCount; }) .Run(); watch.Stop(); watch.Elapsed.ShouldBeGreaterThanOrEqualTo(TimeSpan.FromMilliseconds(12000)); exceptionCount.ShouldEqual(4); failureCount.ShouldEqual(1); successCount.ShouldEqual(0); }
public async Task FuncAsync_Should_Be_Tried_3Times_With_ConstantInterval() { var exceptionCount = 0; var failureCount = 0; var successCount = 0; var watch = new Stopwatch(); watch.Start(); await RetryLogic .DoAsync <int>(this.ThrowExceptionAsync) .Handle <FileNotFoundException>() .AtMost(3) .WithConstantInterval(TimeSpan.FromMilliseconds(2000)) .OnException <FileNotFoundException>(e => ++ exceptionCount) .OnFailure(() => ++ failureCount) .OnSuccess(x => { ++successCount; }) .RunAsync(); watch.Stop(); watch.Elapsed.ShouldBeGreaterThanOrEqualTo(TimeSpan.FromMilliseconds(2000 * 2)); exceptionCount.ShouldEqual(3); failureCount.ShouldEqual(1); successCount.ShouldEqual(0); }
public async Task ActionAsync_Should_Try_10_Times() { var exceptionCount = 0; var failureCount = 0; var successCount = 0; var watch = new Stopwatch(); watch.Start(); var profile = RetryLogic .BeginPrepare() .Handle <FileNotFoundException>() .AtMost(10) .WithConstantInterval(TimeSpan.FromMilliseconds(0)) .OnException <FileNotFoundException>(e => ++ exceptionCount) .OnFailure(() => ++ failureCount) .EndPrepare(); await profile.DoAsync(ActionThrowExceptionAsync) .OnSuccess(x => { ++successCount; }) .RunAsync(); watch.Stop(); exceptionCount.ShouldEqual(10); failureCount.ShouldEqual(1); successCount.ShouldEqual(0); }
public async Task ActionAsync_Should_Be_A_Success() { var exceptionCount = 0; var failureCount = 0; var successCount = 0; var watch = new Stopwatch(); watch.Start(); await RetryLogic .DoAsync(this.ActionNoExceptionAsync) .Handle <FileNotFoundException>() .AtMost(3) .WithConstantInterval(TimeSpan.FromMilliseconds(2000)) .OnException <FileNotFoundException>(e => ++ exceptionCount) .OnFailure(() => ++ failureCount) .OnSuccess(x => { ++successCount; }) .RunAsync(); watch.Stop(); exceptionCount.ShouldEqual(0); failureCount.ShouldEqual(0); successCount.ShouldEqual(1); }
public void ShouldNotRetryOnNonTransientErrors(Exception error) { var retryLogic = new RetryLogic(TimeSpan.FromSeconds(5), null); var work = CreateFailingWork(0, error); var exc = Record.Exception(() => retryLogic.Retry(() => work.Work(null))); exc.Should().Be(error); work.Invocations.Should().Be(1); }
public void ShouldNotRetryOnSuccess() { var retryLogic = new RetryLogic(TimeSpan.FromSeconds(5), null); var work = CreateFailingWork(5); var result = retryLogic.Retry(() => work.Work(null)); result.Should().Be(5); work.Invocations.Should().Be(1); }
public void ShouldRetryOnTransientErrors(Exception error) { var retryLogic = new RetryLogic(TimeSpan.FromSeconds(5), null); var work = CreateFailingWork(5, error); var result = retryLogic.Retry(() => work.Work(null)); result.Should().Be(5); work.Invocations.Should().Be(2); }
public SendResult Send(ElasticemailMessage msg) { var result = new SendResult(); try { string validationErrors; if (!IsValid(msg, out validationErrors)) { result.ResultType = ResultType.Error; result.ErrorMessage = validationErrors; return result; } var client = new WebClient(); var values = new NameValueCollection(); values.Add("api_key", _configuration.ApiKey); values.Add("from", msg.From.Address); values.Add("from_name", msg.From.DisplayName); values.Add("to", string.Join(";", msg.To.Select(x => x.Address))); values.Add("subject", msg.Subject); values.Add(msg.IsBodyHtml ? "body_html" : "body_text", msg.Body); if (msg.ReplyTo != null) values.Add("reply_to", msg.ReplyTo.Address); var attachmentIds = new List<string>(); foreach (var attachment in msg.Attachments) { var attId = UploadAttachment(attachment.Key, attachment.Value); attachmentIds.Add(attId); } if (attachmentIds.Any()) values.Add("attachments", string.Join(";", attachmentIds)); var response = new RetryLogic(_configuration.PreferredRetryStrategy).Execute(() => client.UploadValues("https://api.elasticemail.com/mailer/send", values)); var responseString = Encoding.UTF8.GetString(response); Guid guid; if (Guid.TryParse(responseString, out guid)) { result.TransactionId = guid; result.ResultType = ResultType.Success; } else { result.ErrorMessage = responseString; result.ResultType = ResultType.Error; } } catch (Exception ex) { result.ResultType = ResultType.Error; result.ErrorMessage = ex.ToString(); } return result; }
private SqlRetryLogicBase GetRetryLogic() { SqlRetryLogicBase retryLogic = null; if (!_retryLogicPool.TryTake(out retryLogic)) { retryLogic = RetryLogic.Clone() as SqlRetryLogicBase; } else { retryLogic?.Reset(); } return(retryLogic); }
public void ShouldRetryAtLeastTwice() { var error = new TransientException("code", "message"); var logger = new Mock <IDriverLogger>(); var retryLogic = new RetryLogic(TimeSpan.FromSeconds(1), logger.Object); var work = CreateFailingWork(TimeSpan.FromSeconds(2), 1, error); var result = retryLogic.Retry(() => work.Work(null)); result.Should().Be(1); logger.Verify(x => x.Warn(error, It.Is <string>(s => s.StartsWith("Transaction failed and will be retried in"))), Times.Once); }
public void ShouldThrowServiceUnavailableWhenRetriesTimedOut() { var errorCount = 3; var exceptions = Enumerable.Range(1, errorCount).Select(i => new TransientException($"{i}", $"{i}")) .Cast <Exception>().ToArray(); var logger = new Mock <IDriverLogger>(); var retryLogic = new RetryLogic(TimeSpan.FromSeconds(2), logger.Object); var work = CreateFailingWork(TimeSpan.FromSeconds(1), 1, exceptions); var exc = Record.Exception(() => retryLogic.Retry(() => work.Work(null))); exc.Should().BeOfType <ServiceUnavailableException>() .Which.InnerException.Should().BeOfType <AggregateException>() .Which.InnerExceptions.Should().BeSubsetOf(exceptions); }
public void ShouldLogRetries(int errorCount) { var error = new TransientException("code", "message"); var logger = new Mock <IDriverLogger>(); var retryLogic = new RetryLogic(TimeSpan.FromMinutes(1), logger.Object); var work = CreateFailingWork(1, Enumerable.Range(1, errorCount).Select(x => error).Cast <Exception>().ToArray()); var result = retryLogic.Retry(() => work.Work(null)); result.Should().Be(1); logger.Verify(x => x.Warn(error, It.Is <string>(s => s.StartsWith("Transaction failed and will be retried in"))), Times.Exactly(errorCount)); }
/// <summary>Executes a function and applies retry logic, if enabled.</summary> public override async Task <TResult> ExecuteAsync <TResult>(object sender, Func <Task <TResult> > function, CancellationToken cancellationToken = default) { if (function == null) { throw SqlReliabilityUtil.ArgumentNull(nameof(function)); } SqlRetryLogicBase retryLogic = null; var exceptions = new List <Exception>(); retry: try { TResult result = await function.Invoke(); RetryLogicPoolAdd(retryLogic); return(result); } catch (Exception e) { if (RetryLogic.RetryCondition(sender) && RetryLogic.TransientPredicate(e)) { retryLogic = retryLogic ?? GetRetryLogic(); SqlClientEventSource.Log.TryTraceEvent("<sc.{0}.ExecuteAsync<TResult>|INFO> Found an action eligible for the retry policy (retried attempts = {1}).", TypeName, retryLogic.Current); exceptions.Add(e); if (retryLogic.TryNextInterval(out TimeSpan intervalTime)) { // The retrying event raises on each retry. ApplyRetryingEvent(sender, retryLogic, intervalTime, exceptions, e); await Task.Delay(intervalTime, cancellationToken); goto retry; } else { throw CreateException(exceptions, retryLogic); } } else { RetryLogicPoolAdd(retryLogic); throw; } } }
public async Task Retry_With_Minimal_Settings_Should_Not_Fail() { var watch = new Stopwatch(); watch.Start(); Exception exception = null; await RetryLogic.DoAsync(ActionThrowExceptionAsync) .OnFailureContinue() .Handle <FileNotFoundException>() .RunAsync(); watch.Stop(); watch.Elapsed.ShouldBeGreaterThanOrEqualTo(TimeSpan.FromMilliseconds(4010)); }
/// <summary>Executes a function and applies retry logic, if enabled.</summary> public override TResult Execute <TResult>(object sender, Func <TResult> function) { if (function == null) { throw new ArgumentNullException(nameof(function)); } SqlRetryLogicBase retryLogic = null; var exceptions = new List <Exception>(); retry: try { TResult result = function.Invoke(); RetryLogicPoolAdd(retryLogic); return(result); } catch (Exception e) { if (enableRetryLogic && RetryLogic.RetryCondition(sender) && RetryLogic.TransientPredicate(e)) { retryLogic = retryLogic ?? GetRetryLogic(); SqlClientEventSource.Log.TryTraceEvent("<sc.{0}.Execute<TResult>|INFO> Found an action eligible for the retry policy (retried attempts = {1}).", TypeName, retryLogic.Current); exceptions.Add(e); if (retryLogic.TryNextInterval(out TimeSpan intervalTime)) { // The retrying event raises on each retry. ApplyRetryingEvent(sender, retryLogic, intervalTime, exceptions, e); Thread.Sleep(intervalTime); goto retry; } else { throw CreateException(exceptions, retryLogic); } } else { RetryLogicPoolAdd(retryLogic); throw; } } }
public override TResult Execute <TResult>(object sender, Func <TResult> function) { // Create a list to save transient exceptions to report later if necessary IList <Exception> exceptions = new List <Exception>(); // Prepare it before reusing RetryLogic.Reset(); // Create an infinite loop to attempt the defined maximum number of tries do { try { // Try to invoke the function return(function.Invoke()); } // Catch any type of exception for further investigation catch (Exception e) { // Ask the RetryLogic object if this exception is a transient error if (RetryLogic.TransientPredicate(e)) { // Add the exception to the list of exceptions we've retried on exceptions.Add(e); // Ask the RetryLogic for the next delay time before the next attempt to run the function if (RetryLogic.TryNextInterval(out TimeSpan gapTime)) { Console.WriteLine($"Wait for {gapTime} before next try"); // Wait before next attempt Thread.Sleep(gapTime); } else { // Number of attempts has exceeded the maximum number of tries throw new AggregateException("The number of retries has exceeded the maximum number of attempts.", exceptions); } } else { // If the exception wasn't a transient failure throw the original exception throw; } } } while (true); }
public async Task FuncAsync_Should_Be_Tried_3Times_With_ConstantInterval_IsTraced() { string trace = ""; var listener = new TestTraceListener(); listener.TraceWrite += (s, e) => trace += e.Message; Trace.Listeners.Add(listener); try { var exceptionCount = 0; var failureCount = 0; var successCount = 0; var watch = new Stopwatch(); watch.Start(); await RetryLogic .DoAsync <int>(this.ThrowExceptionAsync) .Handle <FileNotFoundException>() .WithTrace(true) .AtMost(3) .WithConstantInterval(TimeSpan.FromMilliseconds(2000)) .OnException <FileNotFoundException>(e => ++ exceptionCount) .OnFailure(() => ++ failureCount) .OnSuccess(x => { ++successCount; }) .RunAsync(); watch.Stop(); watch.Elapsed.ShouldBeGreaterThanOrEqualTo(TimeSpan.FromMilliseconds(2000 * 2)); exceptionCount.ShouldEqual(3); failureCount.ShouldEqual(1); successCount.ShouldEqual(0); trace.ShouldContain("Max attempt number reached"); } finally { Trace.Listeners.Remove(listener); } }
public async Task Retry_Without_Settings_Should_Fail() { var watch = new Stopwatch(); watch.Start(); Exception exception = null; try { await RetryLogic.DoAsync(ActionThrowExceptionAsync) .RunAsync(); } catch (Exception ex) { exception = ex; } exception.ShouldBeType(typeof(InvalidOperationException)); watch.Stop(); }
public async Task ActionAsync_Should_Throw_An_Exception() { var exceptionCount = 0; var failureCount = 0; var successCount = 0; var watch = new Stopwatch(); watch.Start(); var profile = RetryLogic .BeginPrepare() .Handle <NullReferenceException>() .AtMost(3) .WithConstantInterval(TimeSpan.FromMilliseconds(2000)) .WithTrace(true) .OnException <FileNotFoundException>(e => ++ exceptionCount) .OnFailure(() => ++ failureCount) .EndPrepare(); try { ////await RetryLogic.DoAsync(ActionThrowExceptionAsync, profile) await profile.DoAsync(ActionThrowExceptionAsync) .OnSuccess(x => { ++successCount; }) .RunAsync(); } catch (Exception ex) { ex.ShouldBeType(typeof(FileNotFoundException)); } watch.Stop(); exceptionCount.ShouldEqual(0); failureCount.ShouldEqual(0); successCount.ShouldEqual(0); }
public DeliveryStatusResponse GetDeliveryStatus(Guid transactionId) { var response = new DeliveryStatusResponse(); try { var requestString = "https://api.elasticemail.com/mailer/status/" + transactionId + "?showstats=true"; var request = WebRequest.Create(requestString); Stream stream = new RetryLogic(_configuration.PreferredRetryStrategy).Execute(() => request.GetResponse().GetResponseStream()); var xmlString = new StreamReader(stream, Encoding.UTF8).ReadToEnd(); if (xmlString.ToLower().StartsWith("no job with transactionid")) { response.ErrorMessage = xmlString; response.ResultType = ResultType.Error; } else { try { XmlSerializer serializer = new XmlSerializer(typeof(job)); StringReader rdr = new StringReader(xmlString); var job = (job)serializer.Deserialize(rdr); var deliveryStatus = new DeliveryStatus(); deliveryStatus.Id = transactionId; deliveryStatus.Recipients = job.recipients; deliveryStatus.Delivered = job.delivered; deliveryStatus.Failed = job.failed; deliveryStatus.Pending = job.pending; deliveryStatus.Status = job.status; deliveryStatus.Clicked = job.clicked; deliveryStatus.Opened = job.opened; deliveryStatus.Unsubscribed = job.unsubscribed; response.DeliveryStatus = deliveryStatus; response.ResultType = ResultType.Success; } catch (Exception ex) { response.ErrorMessage = "Could not deserialize message: " + Environment.NewLine + ex; response.ResultType = ResultType.Error; } } } catch (Exception ex) { response.ErrorMessage = ex.ToString(); response.ResultType = ResultType.Error; } return response; }