async Task BeginUploadRetry(CancellationToken cancellationToken)
{
int retryNum = 0;
var shouldRetry = RetryStrategy.GetShouldRetry();
while (shouldRetry(retryNum++, null, out TimeSpan backoffDelay))
{
Log.LogInformation($"Metric publish set to retry in {backoffDelay.Humanize()}");
await Task.Delay(backoffDelay, cancellationToken);
if (await this.TryUploadAndClear(cancellationToken))
{
// Upload succeded, end loop
return;
}
}
Log.LogInformation($"Upload retries exeeded {retryNum} allowed attempts. Deleting stored metrics.");
using (await this.scrapeUploadLock.LockAsync(cancellationToken))
{
await this.storage.RemoveAllReturnedMetricsAsync();
Log.LogInformation($"Deleted stored metrics.");
}
}
/// <summary>
/// Executes the action.
/// </summary>
/// <typeparam name="TResult">The type of the t result.</typeparam>
/// <param name="func">The function.</param>
/// <returns>TResult.</returns>
/// <exception cref="System.ArgumentNullException">func</exception>
public override TResult ExecuteAction <TResult>(Func <TResult> func)
{
//Converting func,if RetryPolicyAdapter defined
var adaptedFunction = RetryPolicyAdapter != null?RetryPolicyAdapter.AdaptExecuteAction(func) : func;
if (func == null)
{
throw new ArgumentNullException("func");
}
int retryCount = 0;
TimeSpan delay = TimeSpan.Zero;
ShouldRetry shouldRetry = RetryStrategy.GetShouldRetry();
while (true)
{
do
{
try
{
return(adaptedFunction());
}
catch (Exception ex)
{
if (!ErrorDetectionStrategy.IsTransient(ex))
{
throw;
}
if (shouldRetry(retryCount++, ex, out delay))
{
if (delay.TotalMilliseconds < 0.0)
{
delay = TimeSpan.Zero;
}
OnRetrying(retryCount, ex, delay);
}
else
{
OnRetrying(retryCount, ex, delay);
throw;
}
}
}while (retryCount <= 1 && RetryStrategy.FastFirstRetry);
Thread.Sleep(delay);
}
}
void ServiceHostFaulted(object sender, EventArgs e)
{
Console.WriteLine("Relay Echo ServiceHost faulted. {0}", statusBehavior.LastError?.Message);
serviceHost.Abort();
serviceHost.Faulted -= ServiceHostFaulted;
serviceHost = null;
var isPermanentError = statusBehavior.LastError is RelayNotFoundException ||
statusBehavior.LastError is AuthorizationFailedException ||
statusBehavior.LastError is AddressAlreadyInUseException;
// RelayNotFound indicates the Relay Service could not find an endpoint with the name/address used by this client
// AuthorizationFailed indicates that the SAS/Shared Secret key used by this client is invalid and needs to be updated
// AddressAlreadyInUseException indicates that this endpoint is in use with incompatible settings (like volatile vs persistent etc)
// If we encounter one of these conditions, retrying will not change the results. Operator/Admin intervention is required.
// Further we believe that in all other cases, the ServiceBus Client itself would retry instead of faulting the ServiceHost
// However, in case there are bugs that we are not currently aware of, restarting the ServiceHost here may overcome such conditions.
if (!isPermanentError)
{
TimeSpan waitPeriod;
var shouldRetry = retryStrategy.GetShouldRetry();
if (shouldRetry(retryCount++, statusBehavior.LastError, out waitPeriod))
{
Thread.Sleep(waitPeriod);
Open();
Console.WriteLine("Relay Echo ServiceHost recreated ");
}
}
else if (statusBehavior.LastError != null)
{
Console.WriteLine("Relay Echo Service encountered permanent error {0}", statusBehavior.LastError?.Message);
}
}
/// <summary>
/// Wrap a Transient Fault Handling Application Block retry strategy into a Microsoft.WindowsAzure.StorageClient.RetryPolicy.
/// </summary>
/// <param name="retryStrategy">The Transient Fault Handling Application Block retry strategy to wrap.</param>
/// <returns>Returns a wrapped Transient Fault Handling Application Block retry strategy into a Microsoft.WindowsAzure.StorageClient.RetryPolicy.</returns>
public static Microsoft.WindowsAzure.StorageClient.RetryPolicy AsAzureStorageClientRetryPolicy(this RetryStrategy retryStrategy)
{
Guard.ArgumentNotNull(retryStrategy, "retryStrategy");
return(() => new ShouldRetryWrapper(retryStrategy.GetShouldRetry()).ShouldRetry);
}