예제 #1
0
        private void InitializeExponentialBackOff(ConfigurableHttpClient client, IFcmClientSettings settings)
        {
            // The Maximum Number of Retries is limited to 3 per default for a ConfigurableHttpClient. This is
            // somewhat weird, because the ExponentialBackOff Algorithm is initialized with 10 Retries per default.
            //
            // Somehow the NumTries seems to be the limiting factor here, so it basically overrides anything you
            // are going to write in the Exponential Backoff Handler.
            client.MessageHandler.NumTries = settings.ExponentialBackOffSettings.MaxNumberOfRetries;

            // Create the Default BackOff Algorithm:
            var backoff = new ExponentialBackOff(settings.ExponentialBackOffSettings.DeltaBackOff, settings.ExponentialBackOffSettings.MaxNumberOfRetries);

            // Create the Initializer. Make sure to set the Maximum Timespan between two Requests. It
            // is 16 Seconds per Default:
            var backoffInitializer = new BackOffHandler.Initializer(backoff)
            {
                MaxTimeSpan = settings.ExponentialBackOffSettings.MaxTimeSpan
            };

            // Now create the Handler:
            var initializer = new ExponentialBackOffInitializer(ExponentialBackOffPolicy.UnsuccessfulResponse503, () => new BackOffHandler(backoffInitializer));

            // And finally append the BackOff Handler, which reacts to 503 Requests:
            initializer.Initialize(client);
        }
예제 #2
0
        internal async Task ListEventsAsync(
            ListLogEntriesRequest request,
            Action <EventBase> callback,
            ExponentialBackOff backOff,
            CancellationToken cancellationToken)
        {
            using (ApplicationTraceSources.Default.TraceMethod().WithParameters(request.Filter))
            {
                try
                {
                    string nextPageToken = null;
                    do
                    {
                        request.PageToken = nextPageToken;

                        using (var stream = await this.service.Entries
                                            .List(request)
                                            .ExecuteAsStreamWithRetryAsync(backOff, cancellationToken)
                                            .ConfigureAwait(false))
                            using (var reader = new JsonTextReader(new StreamReader(stream)))
                            {
                                nextPageToken = ListLogEntriesParser.Read(reader, callback);
                            }
                    }while (nextPageToken != null);
                }
                catch (GoogleApiException e) when(e.Error != null && e.Error.Code == 403)
                {
                    throw new ResourceAccessDeniedException(
                              "You do not have sufficient permissions to view logs. " +
                              "You need the 'Logs Viewer' role (or an equivalent custom role) " +
                              "to perform this action.",
                              e);
                }
            }
        }
예제 #3
0
        public async static Task <Stream> ExecuteAsStreamWithRetryAsync <TResponse>(
            this ClientServiceRequest <TResponse> request,
            ExponentialBackOff backOff)
        {
            int retries = 0;

            while (true)
            {
                try
                {
                    return(await request.ExecuteAsStreamAsync());
                }
                catch (GoogleApiException e) when(e.Error != null && e.Error.Code == 429)
                {
                    // Too many requests.
                    if (retries < backOff.MaxNumOfRetries)
                    {
                        TraceSources.Common.TraceWarning(
                            "Too many requests - backing of and retrying...", retries);

                        retries++;
                        await Task.Delay(backOff.GetNextBackOff(retries));
                    }
                    else
                    {
                        // Retried too often already.
                        TraceSources.Common.TraceWarning("Giving up after {0} retries", retries);
                        throw;
                    }
                }
            }
        }
예제 #4
0
 /// <summary>A helper subtest to test invalid value given to the constructor.</summary>
 private void SubtestConstructor_InvalidValue(TimeSpan ts, int max = 10)
 {
     try
     {
         ExponentialBackOff backOff = new ExponentialBackOff(ts, max);
         Assert.True(false, "Exception expected");
     }
     catch (ArgumentOutOfRangeException) { }
 }
예제 #5
0
            internal FactoredExponentialBackOff(
                TimeSpan deltaBackOff, int maximumNumOfRetries, double factor = 1.0)
            {
                if (factor < 0)
                {
                    throw new ArgumentException("Factor must not be negative.");
                }

                this.expBackOff = new ExponentialBackOff(deltaBackOff, maximumNumOfRetries);
                this.factor     = factor;
            }
        private async Task <T> DoRequestAsync <T>(object requestContent)
        {
            int retryCount             = -1;
            ExponentialBackOff backOff = null;

            do
            {
                retryCount++;
                if (retryCount > 0)
                {
                    // 等待一段时间再发起重试
                    if (backOff == null)
                    {
                        backOff = new ExponentialBackOff(250, 2.0, disConfig.GetBackOffMaxIntervalMs(),
                                                         ExponentialBackOff.DEFAULT_MAX_ELAPSED_TIME);
                    }
                    backOff.backOff(backOff.getNextBackOff());
                }

                try
                {
                    requestObs.Headers.Remove(HttpHeaderKeys.Authorization);
                    requestObs.Headers.Remove(HttpHeaderKeys.SdkData);
                    requestObs.Headers.Remove(HttpHeaderKeys.SdkShaContent);
                    requestObs.Headers.Remove(HttpHeaderKeys.HostHeader);
                    // 每次重传需要重新签名
                    requestObs = SignUtil.Sign(requestObs, disConfig.GetAK(), disConfig.GetSK(), disConfig.GetRegion());
                    return(await DoRequestAsync <T>(requestObs, requestContent));
                }
                catch (Exception t)
                {
                    String errorMsg   = t.Message;
                    int    statusCode = int.Parse(errorMsg.Split('\n')[0]);
                    // 如果不是可以重试的异常 或者 已达到重试次数,则直接抛出异常
                    if (!Utils.IsRetriableSendException(statusCode) || retryCount >= disConfig.GetExceptionRetries())
                    {
                        throw new Exception(errorMsg.Substring(statusCode.ToString().Length + 1), t);
                    }

                    logger.WarnFormat("Find Retriable Exception {0}, url [{1} {2}], currRetryCount is {3}",
                                      errorMsg.Replace("\r\n", ""), requestObs.HttpMethod, requestObs.Endpoint.Host.Trim('/') + requestObs.ResourcePath, retryCount);
                }
            } while (retryCount < disConfig.GetExceptionRetries());

            return(default(T));

            //return await DoRequestAsync<T>(requestObs, requestContent);
        }
예제 #7
0
        /// <summary>A helper test for testing the <c>GetNextBackOff</c> logic.</summary>
        private void SubtestGetNextBackOff_MaxNumRetries(int max)
        {
            ExponentialBackOff backOff = new ExponentialBackOff(TimeSpan.Zero, max);

            for (int i = 1; i <= 10; ++i)
            {
                if (i <= max)
                {
                    Assert.NotEqual(TimeSpan.MinValue, backOff.GetNextBackOff(i));
                }
                else
                {
                    Assert.Equal(TimeSpan.MinValue, backOff.GetNextBackOff(i));
                }
            }
        }
예제 #8
0
        public void GetNextBackOff_InvalidValue()
        {
            ExponentialBackOff backOff = new ExponentialBackOff();

            try
            {
                backOff.GetNextBackOff(0);
                Assert.True(false, "Exception expected");
            }
            catch (ArgumentOutOfRangeException) { }

            try
            {
                backOff.GetNextBackOff(-2);
                Assert.True(false, "Exception expected");
            }
            catch (ArgumentOutOfRangeException) { }
        }
예제 #9
0
        /// <summary>
        /// Waits for a report file to generate by polling for its status using exponential backoff. In the worst case,
        /// there will be 10 attempts to determine if the report is no longer processing.
        /// </summary>
        /// <param name="service">DfaReporting service object used to run the requests.</param>
        /// <param name="userProfileId">The ID number of the DFA user profile to run this request as.</param>
        /// <param name="file">The report file to poll the status of.</param>
        /// <returns>The report file object, either once it is no longer processing or
        ///     once too much time has passed.</returns>
        private static File WaitForReportRunCompletion(DfareportingService service, long userProfileId,
                                                       File file)
        {
            ExponentialBackOff backOff = new ExponentialBackOff();
            TimeSpan           interval;

            file = service.Reports.Files.Get(userProfileId, file.ReportId.Value, file.Id.Value).Execute();

            for (int i = 1; i <= backOff.MaxNumOfRetries; i++)
            {
                if (!file.Status.Equals("PROCESSING"))
                {
                    break;
                }

                interval = backOff.GetNextBackOff(i);
                Console.WriteLine("Polling again in {0} seconds.", interval);
                Thread.Sleep(interval);
                file = service.Reports.Files.Get(userProfileId, file.ReportId.Value, file.Id.Value).Execute();
            }
            return(file);
        }
예제 #10
0
        /// <summary>Test helper for testing retrying using exponential back-off.</summary>
        /// <param name="retry">Index of current retry.</param>
        /// <param name="delta">The delta the exponential back-off uses.
        /// <seealso cref="ExponentialBackOff.DeltaBackOff"/> for more details.
        /// </param>
        /// <param name="epsilon">Used for checking the average result of the input retry [In milliseconds].</param>
        private void SubtestGetNextBackOff(int retry, Nullable <TimeSpan> delta = null, int epsilon = 20)
        {
            int expectedMillis         = (int)Math.Pow(2, (retry - 1)) * 1000;
            ExponentialBackOff backOff = delta.HasValue ?
                                         new ExponentialBackOff(delta.Value) : new ExponentialBackOff();

            TimeSpan min    = TimeSpan.FromMilliseconds(expectedMillis - backOff.DeltaBackOff.TotalMilliseconds);
            TimeSpan max    = TimeSpan.FromMilliseconds(expectedMillis + backOff.DeltaBackOff.TotalMilliseconds);
            long     total  = 0;
            long     repeat = 1000;

            for (int i = 0; i < repeat; ++i)
            {
                var ts = backOff.GetNextBackOff(retry);
                Assert.InRange(ts, min, max);
                total += (int)ts.TotalMilliseconds;
            }

            var average = (int)(total / repeat);

            Assert.InRange(average, expectedMillis - epsilon, expectedMillis + epsilon);
        }
예제 #11
0
        private static CreateHttpClientArgs CreateDefaultHttpClientArgs(IFcmClientSettings settings)
        {
            if (settings == null)
            {
                throw new ArgumentNullException("settings", "Settings are needed to create the Default HttpClientArgs");
            }

            var args = new CreateHttpClientArgs();

            // Create the Default BackOff Algorithm:
            var backoff = new ExponentialBackOff(settings.ExponentialBackOffSettings.DeltaBackOff, settings.ExponentialBackOffSettings.MaxNumberOfRetries);

            // Create the Initializer. Make sure to set the Maximum Timespan between two Requests. It is 16 Seconds per Default:
            var backoffInitializer = new BackOffHandler.Initializer(backoff)
            {
                MaxTimeSpan = settings.ExponentialBackOffSettings.MaxTimeSpan
            };

            args.Initializers.Add(new ExponentialBackOffInitializer(ExponentialBackOffPolicy.UnsuccessfulResponse503, () => new BackOffHandler(backoffInitializer)));

            return(args);
        }
        public static async Task ListEventsAsync(
            this EntriesResource entriesResource,
            ListLogEntriesRequest request,
            Action <EventBase> callback,
            ExponentialBackOff backOff)
        {
            using (TraceSources.LogAnalysis.TraceMethod().WithParameters(request.Filter))
            {
                string nextPageToken = null;
                do
                {
                    request.PageToken = nextPageToken;

                    using (var stream = await entriesResource
                                        .List(request)
                                        .ExecuteAsStreamWithRetryAsync(backOff))
                        using (var reader = new JsonTextReader(new StreamReader(stream)))
                        {
                            nextPageToken = ListLogEntriesParser.Read(reader, callback);
                        }
                }while (nextPageToken != null);
            }
        }
예제 #13
0
        protected PutRecordsResult InnerPutRecordsWithRetry(PutRecordsRequest putRecordsParam, PutRecordMethod putRecordMethod)
        {
            //数据上传的结果集
            PutRecordsResult putRecordsResult = null;

            //用该数组来汇总每次请求后的结果
            PutRecordsResultEntry[] putRecordsResultEntryList = null;

            //记录每次请求失败的下标位置
            int[] retryIndex = null;

            //每次需要上传的请求数据
            PutRecordsRequest retryPutRecordsRequest = putRecordsParam;

            int retryCount             = -1;
            int currentFailed          = 0;
            ExponentialBackOff backOff = null;

            try
            {
                do
                {
                    retryCount++;
                    if (retryCount > 0)
                    {
                        // 等待一段时间再发起重试
                        if (backOff == null)
                        {
                            Monitor.Enter(objlock);
                            logger.Info("Put records retry lock.");
                            backOff = new ExponentialBackOff(ExponentialBackOff.DEFAULT_INITIAL_INTERVAL,
                                                             ExponentialBackOff.DEFAULT_MULTIPLIER, _disConfig.GetBackOffMaxIntervalMs(),
                                                             ExponentialBackOff.DEFAULT_MAX_ELAPSED_TIME);
                        }

                        if (putRecordsResult != null && currentFailed != putRecordsResult.Records.Count)
                        {
                            // 部分失败则重置退避时间
                            backOff.resetCurrentInterval();
                        }

                        long sleepMs = backOff.getNextBackOff();

                        if (retryPutRecordsRequest.Records.Count > 0)
                        {
                            logger.DebugFormat(
                                "Put {0} records but {1} failed, will re-try after backoff {2} ms, current retry count is {3}.",
                                putRecordsResult != null ? putRecordsResult.Records.Count
                                    : putRecordsParam.Records.Count,
                                currentFailed,
                                sleepMs,
                                retryCount);
                        }

                        backOff.backOff(sleepMs);
                    }

                    try
                    {
                        putRecordsResult = putRecordMethod(retryPutRecordsRequest);
                    }
                    catch (Exception t)
                    {
                        if (putRecordsResultEntryList != null)
                        {
                            logger.Error(t.Message, t);
                            break;
                        }
                        throw t;
                    }

                    if (putRecordsResult != null)
                    {
                        currentFailed = putRecordsResult.FailedRecordCount;

                        if (putRecordsResultEntryList == null && currentFailed == 0 || _disConfig.GetRecordsRetries() == 0)
                        {
                            // 第一次发送全部成功或者不需要重试,则直接返回结果
                            return(putRecordsResult);
                        }

                        if (putRecordsResultEntryList == null)
                        {
                            // 存在发送失败的情况,需要重试,则使用数组来汇总每次请求后的结果。
                            putRecordsResultEntryList = new PutRecordsResultEntry[putRecordsParam.Records.Count];
                        }

                        // 需要重试发送数据的原始下标
                        List <int> retryIndexTemp = new List <int>(currentFailed);

                        if (currentFailed > 0)
                        {
                            // 初始化重试发送的数据请求
                            retryPutRecordsRequest            = new PutRecordsRequest();
                            retryPutRecordsRequest.StreamName = putRecordsParam.StreamName;
                            retryPutRecordsRequest.Records    = new List <PutRecordsRequestEntry>(currentFailed);
                        }

                        // 对每条结果分析,更新结果数据
                        for (int i = 0; i < putRecordsResult.Records.Count; i++)
                        {
                            // 获取重试数据在原始数据中的下标位置
                            int originalIndex = retryIndex == null ? i : retryIndex[i];
                            PutRecordsResultEntry putRecordsResultEntry = putRecordsResult.Records[i];
                            // 对所有异常进行重试 && "DIS.4303".equals(putRecordsResultEntry.getErrorCode())
                            if (!string.IsNullOrEmpty(putRecordsResultEntry.ErrorCode))
                            {
                                retryIndexTemp.Add(originalIndex);
                                retryPutRecordsRequest.Records.Add(putRecordsParam.Records[originalIndex]);
                            }
                            putRecordsResultEntryList[originalIndex] = putRecordsResultEntry;
                        }
                        retryIndex = retryIndexTemp.Count > 0 ? retryIndexTemp.ToArray()
                            : new int[0];
                    }
                } while ((retryIndex == null || retryIndex.Length > 0) && retryCount < _disConfig.GetRecordsRetries());
            }
            finally
            {
                if (retryCount > 0)
                {
                    Monitor.Exit(objlock);
                    logger.Info("Put records retry unlock.");
                }
            }
            putRecordsResult = new PutRecordsResult();
            if (retryIndex == null)
            {
                // 不可能存在此情况,完全没有发送出去会直接抛出异常
                putRecordsResult.FailedRecordCount = putRecordsParam.Records.Count;
            }
            else
            {
                putRecordsResult.FailedRecordCount = retryIndex.Length;
                putRecordsResult.Records           = new List <PutRecordsResultEntry>(putRecordsResultEntryList);
            }

            return(putRecordsResult);
        }