private void InitializeExponentialBackOff(ConfigurableHttpClient client, IFcmClientSettings settings)
{
// The Maximum Number of Retries is limited to 3 per default for a ConfigurableHttpClient. This is
// somewhat weird, because the ExponentialBackOff Algorithm is initialized with 10 Retries per default.
//
// Somehow the NumTries seems to be the limiting factor here, so it basically overrides anything you
// are going to write in the Exponential Backoff Handler.
client.MessageHandler.NumTries = settings.ExponentialBackOffSettings.MaxNumberOfRetries;
// Create the Default BackOff Algorithm:
var backoff = new ExponentialBackOff(settings.ExponentialBackOffSettings.DeltaBackOff, settings.ExponentialBackOffSettings.MaxNumberOfRetries);
// Create the Initializer. Make sure to set the Maximum Timespan between two Requests. It
// is 16 Seconds per Default:
var backoffInitializer = new BackOffHandler.Initializer(backoff)
{
MaxTimeSpan = settings.ExponentialBackOffSettings.MaxTimeSpan
};
// Now create the Handler:
var initializer = new ExponentialBackOffInitializer(ExponentialBackOffPolicy.UnsuccessfulResponse503, () => new BackOffHandler(backoffInitializer));
// And finally append the BackOff Handler, which reacts to 503 Requests:
initializer.Initialize(client);
}
internal async Task ListEventsAsync(
ListLogEntriesRequest request,
Action <EventBase> callback,
ExponentialBackOff backOff,
CancellationToken cancellationToken)
{
using (ApplicationTraceSources.Default.TraceMethod().WithParameters(request.Filter))
{
try
{
string nextPageToken = null;
do
{
request.PageToken = nextPageToken;
using (var stream = await this.service.Entries
.List(request)
.ExecuteAsStreamWithRetryAsync(backOff, cancellationToken)
.ConfigureAwait(false))
using (var reader = new JsonTextReader(new StreamReader(stream)))
{
nextPageToken = ListLogEntriesParser.Read(reader, callback);
}
}while (nextPageToken != null);
}
catch (GoogleApiException e) when(e.Error != null && e.Error.Code == 403)
{
throw new ResourceAccessDeniedException(
"You do not have sufficient permissions to view logs. " +
"You need the 'Logs Viewer' role (or an equivalent custom role) " +
"to perform this action.",
e);
}
}
}
public async static Task <Stream> ExecuteAsStreamWithRetryAsync <TResponse>(
this ClientServiceRequest <TResponse> request,
ExponentialBackOff backOff)
{
int retries = 0;
while (true)
{
try
{
return(await request.ExecuteAsStreamAsync());
}
catch (GoogleApiException e) when(e.Error != null && e.Error.Code == 429)
{
// Too many requests.
if (retries < backOff.MaxNumOfRetries)
{
TraceSources.Common.TraceWarning(
"Too many requests - backing of and retrying...", retries);
retries++;
await Task.Delay(backOff.GetNextBackOff(retries));
}
else
{
// Retried too often already.
TraceSources.Common.TraceWarning("Giving up after {0} retries", retries);
throw;
}
}
}
}
/// <summary>A helper subtest to test invalid value given to the constructor.</summary>
private void SubtestConstructor_InvalidValue(TimeSpan ts, int max = 10)
{
try
{
ExponentialBackOff backOff = new ExponentialBackOff(ts, max);
Assert.True(false, "Exception expected");
}
catch (ArgumentOutOfRangeException) { }
}
internal FactoredExponentialBackOff(
TimeSpan deltaBackOff, int maximumNumOfRetries, double factor = 1.0)
{
if (factor < 0)
{
throw new ArgumentException("Factor must not be negative.");
}
this.expBackOff = new ExponentialBackOff(deltaBackOff, maximumNumOfRetries);
this.factor = factor;
}
private async Task <T> DoRequestAsync <T>(object requestContent)
{
int retryCount = -1;
ExponentialBackOff backOff = null;
do
{
retryCount++;
if (retryCount > 0)
{
// 等待一段时间再发起重试
if (backOff == null)
{
backOff = new ExponentialBackOff(250, 2.0, disConfig.GetBackOffMaxIntervalMs(),
ExponentialBackOff.DEFAULT_MAX_ELAPSED_TIME);
}
backOff.backOff(backOff.getNextBackOff());
}
try
{
requestObs.Headers.Remove(HttpHeaderKeys.Authorization);
requestObs.Headers.Remove(HttpHeaderKeys.SdkData);
requestObs.Headers.Remove(HttpHeaderKeys.SdkShaContent);
requestObs.Headers.Remove(HttpHeaderKeys.HostHeader);
// 每次重传需要重新签名
requestObs = SignUtil.Sign(requestObs, disConfig.GetAK(), disConfig.GetSK(), disConfig.GetRegion());
return(await DoRequestAsync <T>(requestObs, requestContent));
}
catch (Exception t)
{
String errorMsg = t.Message;
int statusCode = int.Parse(errorMsg.Split('\n')[0]);
// 如果不是可以重试的异常 或者 已达到重试次数，则直接抛出异常
if (!Utils.IsRetriableSendException(statusCode) || retryCount >= disConfig.GetExceptionRetries())
{
throw new Exception(errorMsg.Substring(statusCode.ToString().Length + 1), t);
}
logger.WarnFormat("Find Retriable Exception {0}, url [{1} {2}], currRetryCount is {3}",
errorMsg.Replace("\r\n", ""), requestObs.HttpMethod, requestObs.Endpoint.Host.Trim('/') + requestObs.ResourcePath, retryCount);
}
} while (retryCount < disConfig.GetExceptionRetries());
return(default(T));
//return await DoRequestAsync<T>(requestObs, requestContent);
}
/// <summary>A helper test for testing the <c>GetNextBackOff</c> logic.</summary>
private void SubtestGetNextBackOff_MaxNumRetries(int max)
{
ExponentialBackOff backOff = new ExponentialBackOff(TimeSpan.Zero, max);
for (int i = 1; i <= 10; ++i)
{
if (i <= max)
{
Assert.NotEqual(TimeSpan.MinValue, backOff.GetNextBackOff(i));
}
else
{
Assert.Equal(TimeSpan.MinValue, backOff.GetNextBackOff(i));
}
}
}
public void GetNextBackOff_InvalidValue()
{
ExponentialBackOff backOff = new ExponentialBackOff();
try
{
backOff.GetNextBackOff(0);
Assert.True(false, "Exception expected");
}
catch (ArgumentOutOfRangeException) { }
try
{
backOff.GetNextBackOff(-2);
Assert.True(false, "Exception expected");
}
catch (ArgumentOutOfRangeException) { }
}
/// <summary>
/// Waits for a report file to generate by polling for its status using exponential backoff. In the worst case,
/// there will be 10 attempts to determine if the report is no longer processing.
/// </summary>
/// <param name="service">DfaReporting service object used to run the requests.</param>
/// <param name="userProfileId">The ID number of the DFA user profile to run this request as.</param>
/// <param name="file">The report file to poll the status of.</param>
/// <returns>The report file object, either once it is no longer processing or
/// once too much time has passed.</returns>
private static File WaitForReportRunCompletion(DfareportingService service, long userProfileId,
File file)
{
ExponentialBackOff backOff = new ExponentialBackOff();
TimeSpan interval;
file = service.Reports.Files.Get(userProfileId, file.ReportId.Value, file.Id.Value).Execute();
for (int i = 1; i <= backOff.MaxNumOfRetries; i++)
{
if (!file.Status.Equals("PROCESSING"))
{
break;
}
interval = backOff.GetNextBackOff(i);
Console.WriteLine("Polling again in {0} seconds.", interval);
Thread.Sleep(interval);
file = service.Reports.Files.Get(userProfileId, file.ReportId.Value, file.Id.Value).Execute();
}
return(file);
}
/// <summary>Test helper for testing retrying using exponential back-off.</summary>
/// <param name="retry">Index of current retry.</param>
/// <param name="delta">The delta the exponential back-off uses.
/// <seealso cref="ExponentialBackOff.DeltaBackOff"/> for more details.
/// </param>
/// <param name="epsilon">Used for checking the average result of the input retry [In milliseconds].</param>
private void SubtestGetNextBackOff(int retry, Nullable <TimeSpan> delta = null, int epsilon = 20)
{
int expectedMillis = (int)Math.Pow(2, (retry - 1)) * 1000;
ExponentialBackOff backOff = delta.HasValue ?
new ExponentialBackOff(delta.Value) : new ExponentialBackOff();
TimeSpan min = TimeSpan.FromMilliseconds(expectedMillis - backOff.DeltaBackOff.TotalMilliseconds);
TimeSpan max = TimeSpan.FromMilliseconds(expectedMillis + backOff.DeltaBackOff.TotalMilliseconds);
long total = 0;
long repeat = 1000;
for (int i = 0; i < repeat; ++i)
{
var ts = backOff.GetNextBackOff(retry);
Assert.InRange(ts, min, max);
total += (int)ts.TotalMilliseconds;
}
var average = (int)(total / repeat);
Assert.InRange(average, expectedMillis - epsilon, expectedMillis + epsilon);
}
private static CreateHttpClientArgs CreateDefaultHttpClientArgs(IFcmClientSettings settings)
{
if (settings == null)
{
throw new ArgumentNullException("settings", "Settings are needed to create the Default HttpClientArgs");
}
var args = new CreateHttpClientArgs();
// Create the Default BackOff Algorithm:
var backoff = new ExponentialBackOff(settings.ExponentialBackOffSettings.DeltaBackOff, settings.ExponentialBackOffSettings.MaxNumberOfRetries);
// Create the Initializer. Make sure to set the Maximum Timespan between two Requests. It is 16 Seconds per Default:
var backoffInitializer = new BackOffHandler.Initializer(backoff)
{
MaxTimeSpan = settings.ExponentialBackOffSettings.MaxTimeSpan
};
args.Initializers.Add(new ExponentialBackOffInitializer(ExponentialBackOffPolicy.UnsuccessfulResponse503, () => new BackOffHandler(backoffInitializer)));
return(args);
}
public static async Task ListEventsAsync(
this EntriesResource entriesResource,
ListLogEntriesRequest request,
Action <EventBase> callback,
ExponentialBackOff backOff)
{
using (TraceSources.LogAnalysis.TraceMethod().WithParameters(request.Filter))
{
string nextPageToken = null;
do
{
request.PageToken = nextPageToken;
using (var stream = await entriesResource
.List(request)
.ExecuteAsStreamWithRetryAsync(backOff))
using (var reader = new JsonTextReader(new StreamReader(stream)))
{
nextPageToken = ListLogEntriesParser.Read(reader, callback);
}
}while (nextPageToken != null);
}
}
protected PutRecordsResult InnerPutRecordsWithRetry(PutRecordsRequest putRecordsParam, PutRecordMethod putRecordMethod)
{
//数据上传的结果集
PutRecordsResult putRecordsResult = null;
//用该数组来汇总每次请求后的结果
PutRecordsResultEntry[] putRecordsResultEntryList = null;
//记录每次请求失败的下标位置
int[] retryIndex = null;
//每次需要上传的请求数据
PutRecordsRequest retryPutRecordsRequest = putRecordsParam;
int retryCount = -1;
int currentFailed = 0;
ExponentialBackOff backOff = null;
try
{
do
{
retryCount++;
if (retryCount > 0)
{
// 等待一段时间再发起重试
if (backOff == null)
{
Monitor.Enter(objlock);
logger.Info("Put records retry lock.");
backOff = new ExponentialBackOff(ExponentialBackOff.DEFAULT_INITIAL_INTERVAL,
ExponentialBackOff.DEFAULT_MULTIPLIER, _disConfig.GetBackOffMaxIntervalMs(),
ExponentialBackOff.DEFAULT_MAX_ELAPSED_TIME);
}
if (putRecordsResult != null && currentFailed != putRecordsResult.Records.Count)
{
// 部分失败则重置退避时间
backOff.resetCurrentInterval();
}
long sleepMs = backOff.getNextBackOff();
if (retryPutRecordsRequest.Records.Count > 0)
{
logger.DebugFormat(
"Put {0} records but {1} failed, will re-try after backoff {2} ms, current retry count is {3}.",
putRecordsResult != null ? putRecordsResult.Records.Count
: putRecordsParam.Records.Count,
currentFailed,
sleepMs,
retryCount);
}
backOff.backOff(sleepMs);
}
try
{
putRecordsResult = putRecordMethod(retryPutRecordsRequest);
}
catch (Exception t)
{
if (putRecordsResultEntryList != null)
{
logger.Error(t.Message, t);
break;
}
throw t;
}
if (putRecordsResult != null)
{
currentFailed = putRecordsResult.FailedRecordCount;
if (putRecordsResultEntryList == null && currentFailed == 0 || _disConfig.GetRecordsRetries() == 0)
{
// 第一次发送全部成功或者不需要重试，则直接返回结果
return(putRecordsResult);
}
if (putRecordsResultEntryList == null)
{
// 存在发送失败的情况，需要重试，则使用数组来汇总每次请求后的结果。
putRecordsResultEntryList = new PutRecordsResultEntry[putRecordsParam.Records.Count];
}
// 需要重试发送数据的原始下标
List <int> retryIndexTemp = new List <int>(currentFailed);
if (currentFailed > 0)
{
// 初始化重试发送的数据请求
retryPutRecordsRequest = new PutRecordsRequest();
retryPutRecordsRequest.StreamName = putRecordsParam.StreamName;
retryPutRecordsRequest.Records = new List <PutRecordsRequestEntry>(currentFailed);
}
// 对每条结果分析，更新结果数据
for (int i = 0; i < putRecordsResult.Records.Count; i++)
{
// 获取重试数据在原始数据中的下标位置
int originalIndex = retryIndex == null ? i : retryIndex[i];
PutRecordsResultEntry putRecordsResultEntry = putRecordsResult.Records[i];
// 对所有异常进行重试 && "DIS.4303".equals(putRecordsResultEntry.getErrorCode())
if (!string.IsNullOrEmpty(putRecordsResultEntry.ErrorCode))
{
retryIndexTemp.Add(originalIndex);
retryPutRecordsRequest.Records.Add(putRecordsParam.Records[originalIndex]);
}
putRecordsResultEntryList[originalIndex] = putRecordsResultEntry;
}
retryIndex = retryIndexTemp.Count > 0 ? retryIndexTemp.ToArray()
: new int[0];
}
} while ((retryIndex == null || retryIndex.Length > 0) && retryCount < _disConfig.GetRecordsRetries());
}
finally
{
if (retryCount > 0)
{
Monitor.Exit(objlock);
logger.Info("Put records retry unlock.");
}
}
putRecordsResult = new PutRecordsResult();
if (retryIndex == null)
{
// 不可能存在此情况，完全没有发送出去会直接抛出异常
putRecordsResult.FailedRecordCount = putRecordsParam.Records.Count;
}
else
{
putRecordsResult.FailedRecordCount = retryIndex.Length;
putRecordsResult.Records = new List <PutRecordsResultEntry>(putRecordsResultEntryList);
}
return(putRecordsResult);
}
