public async Task DeveloperForgetsEndPrepare()
{
var exceptionCount = 0;
var failureCount = 0;
var successCount = 0;
var profile = RetryLogic
.BeginPrepare()
.Handle <FileNotFoundException>()
.AtMost(10)
.WithConstantInterval(TimeSpan.FromMilliseconds(0))
.OnException <FileNotFoundException>(e => ++ exceptionCount)
.OnFailure(() => ++ failureCount);
////.EndPrepare(); // developer forgets tha line
////await RetryLogic.DoAsync(ActionThrowExceptionAsync, profile)
try
{
await profile.DoAsync(ActionThrowExceptionAsync)
.AtMost(5)
.OnSuccess(x =>
{
++successCount;
})
.RunAsync();
Assert.Fail("Should have thrown a InvalidOperationException");
}
catch (InvalidOperationException ex)
{
// ok
}
}
public void Function_Should_Be_Tried_3Times_With_ExponentialInterval()
{
var exceptionCount = 0;
var failureCount = 0;
var successCount = 0;
var watch = new Stopwatch();
watch.Start();
RetryLogic
.Do <int>(this.ThrowException)
.Handle <FileNotFoundException>()
.AtMost(4)
.WithExponentialInterval(TimeSpan.FromMilliseconds(2000), 2)
.OnException <FileNotFoundException>(e => ++ exceptionCount)
.OnFailure(() => ++ failureCount)
.OnSuccess(x =>
{
++successCount;
})
.Run();
watch.Stop();
watch.Elapsed.ShouldBeGreaterThanOrEqualTo(TimeSpan.FromMilliseconds(12000));
exceptionCount.ShouldEqual(4);
failureCount.ShouldEqual(1);
successCount.ShouldEqual(0);
}
public async Task FuncAsync_Should_Be_Tried_3Times_With_ConstantInterval()
{
var exceptionCount = 0;
var failureCount = 0;
var successCount = 0;
var watch = new Stopwatch();
watch.Start();
await RetryLogic
.DoAsync <int>(this.ThrowExceptionAsync)
.Handle <FileNotFoundException>()
.AtMost(3)
.WithConstantInterval(TimeSpan.FromMilliseconds(2000))
.OnException <FileNotFoundException>(e => ++ exceptionCount)
.OnFailure(() => ++ failureCount)
.OnSuccess(x =>
{
++successCount;
})
.RunAsync();
watch.Stop();
watch.Elapsed.ShouldBeGreaterThanOrEqualTo(TimeSpan.FromMilliseconds(2000 * 2));
exceptionCount.ShouldEqual(3);
failureCount.ShouldEqual(1);
successCount.ShouldEqual(0);
}
public async Task ActionAsync_Should_Try_10_Times()
{
var exceptionCount = 0;
var failureCount = 0;
var successCount = 0;
var watch = new Stopwatch();
watch.Start();
var profile = RetryLogic
.BeginPrepare()
.Handle <FileNotFoundException>()
.AtMost(10)
.WithConstantInterval(TimeSpan.FromMilliseconds(0))
.OnException <FileNotFoundException>(e => ++ exceptionCount)
.OnFailure(() => ++ failureCount)
.EndPrepare();
await profile.DoAsync(ActionThrowExceptionAsync)
.OnSuccess(x =>
{
++successCount;
})
.RunAsync();
watch.Stop();
exceptionCount.ShouldEqual(10);
failureCount.ShouldEqual(1);
successCount.ShouldEqual(0);
}
public async Task ActionAsync_Should_Be_A_Success()
{
var exceptionCount = 0;
var failureCount = 0;
var successCount = 0;
var watch = new Stopwatch();
watch.Start();
await RetryLogic
.DoAsync(this.ActionNoExceptionAsync)
.Handle <FileNotFoundException>()
.AtMost(3)
.WithConstantInterval(TimeSpan.FromMilliseconds(2000))
.OnException <FileNotFoundException>(e => ++ exceptionCount)
.OnFailure(() => ++ failureCount)
.OnSuccess(x =>
{
++successCount;
})
.RunAsync();
watch.Stop();
exceptionCount.ShouldEqual(0);
failureCount.ShouldEqual(0);
successCount.ShouldEqual(1);
}
public void ShouldNotRetryOnNonTransientErrors(Exception error)
{
var retryLogic = new RetryLogic(TimeSpan.FromSeconds(5), null);
var work = CreateFailingWork(0, error);
var exc = Record.Exception(() => retryLogic.Retry(() => work.Work(null)));
exc.Should().Be(error);
work.Invocations.Should().Be(1);
}
public void ShouldNotRetryOnSuccess()
{
var retryLogic = new RetryLogic(TimeSpan.FromSeconds(5), null);
var work = CreateFailingWork(5);
var result = retryLogic.Retry(() => work.Work(null));
result.Should().Be(5);
work.Invocations.Should().Be(1);
}
public void ShouldRetryOnTransientErrors(Exception error)
{
var retryLogic = new RetryLogic(TimeSpan.FromSeconds(5), null);
var work = CreateFailingWork(5, error);
var result = retryLogic.Retry(() => work.Work(null));
result.Should().Be(5);
work.Invocations.Should().Be(2);
}
public SendResult Send(ElasticemailMessage msg)
{
var result = new SendResult();
try
{
string validationErrors;
if (!IsValid(msg, out validationErrors))
{
result.ResultType = ResultType.Error;
result.ErrorMessage = validationErrors;
return result;
}
var client = new WebClient();
var values = new NameValueCollection();
values.Add("api_key", _configuration.ApiKey);
values.Add("from", msg.From.Address);
values.Add("from_name", msg.From.DisplayName);
values.Add("to", string.Join(";", msg.To.Select(x => x.Address)));
values.Add("subject", msg.Subject);
values.Add(msg.IsBodyHtml ? "body_html" : "body_text", msg.Body);
if (msg.ReplyTo != null) values.Add("reply_to", msg.ReplyTo.Address);
var attachmentIds = new List<string>();
foreach (var attachment in msg.Attachments)
{
var attId = UploadAttachment(attachment.Key, attachment.Value);
attachmentIds.Add(attId);
}
if (attachmentIds.Any()) values.Add("attachments", string.Join(";", attachmentIds));
var response = new RetryLogic(_configuration.PreferredRetryStrategy).Execute(() => client.UploadValues("https://api.elasticemail.com/mailer/send", values));
var responseString = Encoding.UTF8.GetString(response);
Guid guid;
if (Guid.TryParse(responseString, out guid))
{
result.TransactionId = guid;
result.ResultType = ResultType.Success;
}
else
{
result.ErrorMessage = responseString;
result.ResultType = ResultType.Error;
}
}
catch (Exception ex)
{
result.ResultType = ResultType.Error;
result.ErrorMessage = ex.ToString();
}
return result;
}
private SqlRetryLogicBase GetRetryLogic()
{
SqlRetryLogicBase retryLogic = null;
if (!_retryLogicPool.TryTake(out retryLogic))
{
retryLogic = RetryLogic.Clone() as SqlRetryLogicBase;
}
else
{
retryLogic?.Reset();
}
return(retryLogic);
}
public void ShouldRetryAtLeastTwice()
{
var error = new TransientException("code", "message");
var logger = new Mock <IDriverLogger>();
var retryLogic = new RetryLogic(TimeSpan.FromSeconds(1), logger.Object);
var work = CreateFailingWork(TimeSpan.FromSeconds(2), 1, error);
var result = retryLogic.Retry(() => work.Work(null));
result.Should().Be(1);
logger.Verify(x => x.Warn(error,
It.Is <string>(s => s.StartsWith("Transaction failed and will be retried in"))),
Times.Once);
}
public void ShouldThrowServiceUnavailableWhenRetriesTimedOut()
{
var errorCount = 3;
var exceptions = Enumerable.Range(1, errorCount).Select(i => new TransientException($"{i}", $"{i}"))
.Cast <Exception>().ToArray();
var logger = new Mock <IDriverLogger>();
var retryLogic = new RetryLogic(TimeSpan.FromSeconds(2), logger.Object);
var work = CreateFailingWork(TimeSpan.FromSeconds(1), 1, exceptions);
var exc = Record.Exception(() => retryLogic.Retry(() => work.Work(null)));
exc.Should().BeOfType <ServiceUnavailableException>()
.Which.InnerException.Should().BeOfType <AggregateException>()
.Which.InnerExceptions.Should().BeSubsetOf(exceptions);
}
public void ShouldLogRetries(int errorCount)
{
var error = new TransientException("code", "message");
var logger = new Mock <IDriverLogger>();
var retryLogic = new RetryLogic(TimeSpan.FromMinutes(1), logger.Object);
var work = CreateFailingWork(1,
Enumerable.Range(1, errorCount).Select(x => error).Cast <Exception>().ToArray());
var result = retryLogic.Retry(() => work.Work(null));
result.Should().Be(1);
logger.Verify(x => x.Warn(error,
It.Is <string>(s => s.StartsWith("Transaction failed and will be retried in"))),
Times.Exactly(errorCount));
}
/// <summary>Executes a function and applies retry logic, if enabled.</summary>
public override async Task <TResult> ExecuteAsync <TResult>(object sender, Func <Task <TResult> > function, CancellationToken cancellationToken = default)
{
if (function == null)
{
throw SqlReliabilityUtil.ArgumentNull(nameof(function));
}
SqlRetryLogicBase retryLogic = null;
var exceptions = new List <Exception>();
retry:
try
{
TResult result = await function.Invoke();
RetryLogicPoolAdd(retryLogic);
return(result);
}
catch (Exception e)
{
if (RetryLogic.RetryCondition(sender) && RetryLogic.TransientPredicate(e))
{
retryLogic = retryLogic ?? GetRetryLogic();
SqlClientEventSource.Log.TryTraceEvent("<sc.{0}.ExecuteAsync<TResult>|INFO> Found an action eligible for the retry policy (retried attempts = {1}).",
TypeName, retryLogic.Current);
exceptions.Add(e);
if (retryLogic.TryNextInterval(out TimeSpan intervalTime))
{
// The retrying event raises on each retry.
ApplyRetryingEvent(sender, retryLogic, intervalTime, exceptions, e);
await Task.Delay(intervalTime, cancellationToken);
goto retry;
}
else
{
throw CreateException(exceptions, retryLogic);
}
}
else
{
RetryLogicPoolAdd(retryLogic);
throw;
}
}
}
public async Task Retry_With_Minimal_Settings_Should_Not_Fail()
{
var watch = new Stopwatch();
watch.Start();
Exception exception = null;
await RetryLogic.DoAsync(ActionThrowExceptionAsync)
.OnFailureContinue()
.Handle <FileNotFoundException>()
.RunAsync();
watch.Stop();
watch.Elapsed.ShouldBeGreaterThanOrEqualTo(TimeSpan.FromMilliseconds(4010));
}
/// <summary>Executes a function and applies retry logic, if enabled.</summary>
public override TResult Execute <TResult>(object sender, Func <TResult> function)
{
if (function == null)
{
throw new ArgumentNullException(nameof(function));
}
SqlRetryLogicBase retryLogic = null;
var exceptions = new List <Exception>();
retry:
try
{
TResult result = function.Invoke();
RetryLogicPoolAdd(retryLogic);
return(result);
}
catch (Exception e)
{
if (enableRetryLogic && RetryLogic.RetryCondition(sender) && RetryLogic.TransientPredicate(e))
{
retryLogic = retryLogic ?? GetRetryLogic();
SqlClientEventSource.Log.TryTraceEvent("<sc.{0}.Execute<TResult>|INFO> Found an action eligible for the retry policy (retried attempts = {1}).",
TypeName, retryLogic.Current);
exceptions.Add(e);
if (retryLogic.TryNextInterval(out TimeSpan intervalTime))
{
// The retrying event raises on each retry.
ApplyRetryingEvent(sender, retryLogic, intervalTime, exceptions, e);
Thread.Sleep(intervalTime);
goto retry;
}
else
{
throw CreateException(exceptions, retryLogic);
}
}
else
{
RetryLogicPoolAdd(retryLogic);
throw;
}
}
}
public override TResult Execute <TResult>(object sender, Func <TResult> function)
{
// Create a list to save transient exceptions to report later if necessary
IList <Exception> exceptions = new List <Exception>();
// Prepare it before reusing
RetryLogic.Reset();
// Create an infinite loop to attempt the defined maximum number of tries
do
{
try
{
// Try to invoke the function
return(function.Invoke());
}
// Catch any type of exception for further investigation
catch (Exception e)
{
// Ask the RetryLogic object if this exception is a transient error
if (RetryLogic.TransientPredicate(e))
{
// Add the exception to the list of exceptions we've retried on
exceptions.Add(e);
// Ask the RetryLogic for the next delay time before the next attempt to run the function
if (RetryLogic.TryNextInterval(out TimeSpan gapTime))
{
Console.WriteLine($"Wait for {gapTime} before next try");
// Wait before next attempt
Thread.Sleep(gapTime);
}
else
{
// Number of attempts has exceeded the maximum number of tries
throw new AggregateException("The number of retries has exceeded the maximum number of attempts.", exceptions);
}
}
else
{
// If the exception wasn't a transient failure throw the original exception
throw;
}
}
} while (true);
}
public async Task FuncAsync_Should_Be_Tried_3Times_With_ConstantInterval_IsTraced()
{
string trace = "";
var listener = new TestTraceListener();
listener.TraceWrite += (s, e) => trace += e.Message;
Trace.Listeners.Add(listener);
try
{
var exceptionCount = 0;
var failureCount = 0;
var successCount = 0;
var watch = new Stopwatch();
watch.Start();
await RetryLogic
.DoAsync <int>(this.ThrowExceptionAsync)
.Handle <FileNotFoundException>()
.WithTrace(true)
.AtMost(3)
.WithConstantInterval(TimeSpan.FromMilliseconds(2000))
.OnException <FileNotFoundException>(e => ++ exceptionCount)
.OnFailure(() => ++ failureCount)
.OnSuccess(x =>
{
++successCount;
})
.RunAsync();
watch.Stop();
watch.Elapsed.ShouldBeGreaterThanOrEqualTo(TimeSpan.FromMilliseconds(2000 * 2));
exceptionCount.ShouldEqual(3);
failureCount.ShouldEqual(1);
successCount.ShouldEqual(0);
trace.ShouldContain("Max attempt number reached");
}
finally
{
Trace.Listeners.Remove(listener);
}
}
public async Task Retry_Without_Settings_Should_Fail()
{
var watch = new Stopwatch();
watch.Start();
Exception exception = null;
try
{
await RetryLogic.DoAsync(ActionThrowExceptionAsync)
.RunAsync();
}
catch (Exception ex)
{
exception = ex;
}
exception.ShouldBeType(typeof(InvalidOperationException));
watch.Stop();
}
public async Task ActionAsync_Should_Throw_An_Exception()
{
var exceptionCount = 0;
var failureCount = 0;
var successCount = 0;
var watch = new Stopwatch();
watch.Start();
var profile = RetryLogic
.BeginPrepare()
.Handle <NullReferenceException>()
.AtMost(3)
.WithConstantInterval(TimeSpan.FromMilliseconds(2000))
.WithTrace(true)
.OnException <FileNotFoundException>(e => ++ exceptionCount)
.OnFailure(() => ++ failureCount)
.EndPrepare();
try
{
////await RetryLogic.DoAsync(ActionThrowExceptionAsync, profile)
await profile.DoAsync(ActionThrowExceptionAsync)
.OnSuccess(x =>
{
++successCount;
})
.RunAsync();
}
catch (Exception ex)
{
ex.ShouldBeType(typeof(FileNotFoundException));
}
watch.Stop();
exceptionCount.ShouldEqual(0);
failureCount.ShouldEqual(0);
successCount.ShouldEqual(0);
}
public DeliveryStatusResponse GetDeliveryStatus(Guid transactionId)
{
var response = new DeliveryStatusResponse();
try
{
var requestString = "https://api.elasticemail.com/mailer/status/" + transactionId + "?showstats=true";
var request = WebRequest.Create(requestString);
Stream stream = new RetryLogic(_configuration.PreferredRetryStrategy).Execute(() => request.GetResponse().GetResponseStream());
var xmlString = new StreamReader(stream, Encoding.UTF8).ReadToEnd();
if (xmlString.ToLower().StartsWith("no job with transactionid"))
{
response.ErrorMessage = xmlString;
response.ResultType = ResultType.Error;
}
else
{
try
{
XmlSerializer serializer = new XmlSerializer(typeof(job));
StringReader rdr = new StringReader(xmlString);
var job = (job)serializer.Deserialize(rdr);
var deliveryStatus = new DeliveryStatus();
deliveryStatus.Id = transactionId;
deliveryStatus.Recipients = job.recipients;
deliveryStatus.Delivered = job.delivered;
deliveryStatus.Failed = job.failed;
deliveryStatus.Pending = job.pending;
deliveryStatus.Status = job.status;
deliveryStatus.Clicked = job.clicked;
deliveryStatus.Opened = job.opened;
deliveryStatus.Unsubscribed = job.unsubscribed;
response.DeliveryStatus = deliveryStatus;
response.ResultType = ResultType.Success;
}
catch (Exception ex)
{
response.ErrorMessage = "Could not deserialize message: " + Environment.NewLine + ex;
response.ResultType = ResultType.Error;
}
}
}
catch (Exception ex)
{
response.ErrorMessage = ex.ToString();
response.ResultType = ResultType.Error;
}
return response;
}
