public void NullChecking()
{
Assert.Throws<ArgumentNullException>(() => new RetrySettings(null, s_sampleBackoff, Expiration.None).ToString());
Assert.Throws<ArgumentNullException>(() => new RetrySettings(s_sampleBackoff, null, Expiration.None).ToString());
Assert.Throws<ArgumentNullException>(() => new RetrySettings(s_sampleBackoff, s_sampleBackoff, null).ToString());
// No exceptions here...
var settings = new RetrySettings(s_sampleBackoff, s_sampleBackoff, Expiration.None, null);
settings = new RetrySettings(s_sampleBackoff, s_sampleBackoff, Expiration.None, null, null);
}
public Task InitialBackoffOverrideRespected()
{
var settings = RetrySettings.FromConstantBackoff(
maxAttempts: 4, backoff: FiveSeconds, retryFilter: ex => true, backoffJitter: RetrySettings.NoJitter);
var scheduler = new FakeScheduler();
var sequence = RetryAttempt.CreateRetrySequence(settings, scheduler, initialBackoffOverride: OneSecond);
// Should attempt at T=0, T=1, T=6, T=11.
return(AssertAttemptsAsync(sequence, scheduler, () => new Exception(), 0, 1, 6, 11));
}
public Task MaxAttemptsRespected()
{
var settings = RetrySettings.FromConstantBackoff(
maxAttempts: 4, backoff: OneSecond, retryFilter: ex => true, backoffJitter: RetrySettings.NoJitter);
var scheduler = new FakeScheduler();
var sequence = RetryAttempt.CreateRetrySequence(settings, scheduler);
// Should attempt at T=0, T=1, T=2, T=3, then stop because we're only allowed four attempts.
return(AssertAttemptsAsync(sequence, scheduler, () => new Exception(), 0, 1, 2, 3));
}
public void FromConstantBackoff_DefaultJitter()
{
var backoff = TimeSpan.FromSeconds(3);
var filter = RetrySettings.FilterForStatusCodes(StatusCode.Aborted);
var settings = RetrySettings.FromConstantBackoff(10, backoff, filter);
Assert.Equal(10, settings.MaxAttempts);
Assert.Equal(backoff, settings.InitialBackoff);
Assert.Equal(backoff, settings.MaxBackoff);
Assert.Same(filter, settings.RetryFilter);
Assert.Same(RetrySettings.RandomJitter, settings.BackoffJitter);
}
public RowAsyncEnumerator(
BigtableClientImpl client,
ReadRowsRequest originalRequest,
CallSettings callSettings,
RetrySettings retrySettings)
{
_client = client;
_callSettings = callSettings;
_retrySettings = retrySettings;
_requestManager = new BigtableReadRowsRequestManager(originalRequest);
}
private async Task DoBodyProccessAsync(BodySetting bodySetting, RetrySettings retrySettings_4FailedItems)
{
if (bodySetting == null)
{
return;
}
var bodyResult1 = await DoBodyAsync(bodySetting);
var bodyResult2 = bodyResult1;
await SaveBodyAsync(bodySetting, bodyResult2, retrySettings_4FailedItems);
}
internal ReadRowsStream(
BigtableClientImpl client,
ReadRowsRequest originalRequest,
CallSettings callSettings,
RetrySettings retrySettings,
BigtableServiceApiClient.ReadRowsStream underlyingStream)
{
_client = client;
_originalRequest = originalRequest;
_callSettings = callSettings;
_retrySettings = retrySettings;
_underlyingStream = underlyingStream;
}
private static AsyncRetryPolicy GetPolicy(
RetrySettings settings,
ILogger logger)
{
var policyBuilder = Policy
.Handle <ApiException>(ex => settings.RetryTypes.Any(t => ex.GetType() == t));
return(policyBuilder
.WaitAndRetryAsync(
settings.MaxRetryCount,
retryAttempt => TimeSpan.FromSeconds(Math.Pow(retryAttempt, settings.Exponent) * settings.Multiplier),
(exception, timespan, context) => OnRetryCallback(exception, timespan, context, logger)));
}
public Task PredicateRespected()
{
int count = 0;
Func <Exception> exceptionProvider = () => ++ count == 3 ? new Exception() : new RpcException(Status.DefaultCancelled);
var settings = RetrySettings.FromExponentialBackoff(
maxAttempts: 4, initialBackoff: OneSecond, maxBackoff: FiveSeconds,
backoffMultiplier: 1, retryFilter: ex => ex is RpcException, backoffJitter: RetrySettings.NoJitter);
var scheduler = new FakeScheduler();
var sequence = RetryAttempt.CreateRetrySequence(settings, scheduler);
return(AssertAttemptsAsync(sequence, scheduler, exceptionProvider, 0, 1, 2));
}
public Task DeadlineRespected()
{
var settings = RetrySettings.FromExponentialBackoff(
maxAttempts: 10, initialBackoff: OneSecond, maxBackoff: FiveSeconds,
backoffMultiplier: 2, retryFilter: ex => true, backoffJitter: RetrySettings.NoJitter);
var scheduler = new FakeScheduler();
var deadline = scheduler.Clock.GetCurrentDateTimeUtc() + FiveSeconds;
var sequence = RetryAttempt.CreateRetrySequence(settings, scheduler, deadline, scheduler.Clock);
// Should attempt at T=0, T=1, T=3, then stop because the next attempt would be after the deadline.
return(AssertAttemptsAsync(sequence, scheduler, () => new Exception(), 0, 1, 3));
}
public void RetrySettingsTiming()
{
// Sample: RetrySettingsTiming
// Each delay is double the previous one, with a maximum of 5s.
// The first delay is 1s, then 2s, then 4s, then 5s, then 5s, etc.
// Only aborted RPCs are retried in this example.
RetrySettings settings = RetrySettings.FromExponentialBackoff(
maxAttempts: 10,
initialBackoff: TimeSpan.FromSeconds(1),
maxBackoff: TimeSpan.FromSeconds(5),
backoffMultiplier: 2.0,
retryFilter: RetrySettings.FilterForStatusCodes(StatusCode.Aborted));
// End sample
}
private IRestClient GetRestClientWithRetries()
{
// configure to retry 3 times with 100ms between attempts, and no backoff.
var retrySettings = new RetrySettings(
RetryCount,
RetryExponent,
RetryMultiplier,
new[] { typeof(ServiceUnavailableException) });
return(new ResilientRestClient(
retrySettings,
this.mockRestClient.Object,
MockLogger.Object));
}
public Task JitterRespected()
{
var settings = RetrySettings.FromExponentialBackoff(
maxAttempts: 6, initialBackoff: TimeSpan.FromSeconds(2), maxBackoff: TimeSpan.FromSeconds(10),
backoffMultiplier: 2, retryFilter: ex => true, backoffJitter: new HalvingJitter());
var scheduler = new FakeScheduler();
var sequence = RetryAttempt.CreateRetrySequence(settings, scheduler);
// Sequence of theoretical backoffs is 2, 4, 8, 10, 10, 10
// Sequence of jittered backoffs is 1, 2, 4, 5, 5.
return(AssertAttemptsAsync(sequence, scheduler, () => new Exception(), 0, 1, 3, 7, 12, 17));
}
public void FromExponentialBackoff_SpecificJitter()
{
var initialBackoff = TimeSpan.FromSeconds(3);
var maxBackoff = TimeSpan.FromSeconds(5);
var filter = RetrySettings.FilterForStatusCodes(StatusCode.Aborted);
var settings = RetrySettings.FromExponentialBackoff(10, initialBackoff, maxBackoff, 1.5, filter, RetrySettings.NoJitter);
Assert.Equal(10, settings.MaxAttempts);
Assert.Equal(initialBackoff, settings.InitialBackoff);
Assert.Equal(maxBackoff, settings.MaxBackoff);
Assert.Equal(1.5, settings.BackoffMultiplier);
Assert.Same(filter, settings.RetryFilter);
Assert.Same(RetrySettings.NoJitter, settings.BackoffJitter);
}
public RowAsyncEnumerator(
BigtableClientImpl client,
ReadRowsRequest originalRequest,
CallSettings callSettings,
RetrySettings retrySettings,
BigtableServiceApiClient.ReadRowsStream stream)
{
_client = client;
_callSettings = callSettings;
_retrySettings = retrySettings;
_stream = stream;
_requestManager = new BigtableReadRowsRequestManager(originalRequest);
}
/// <summary>
/// Вычислить задержку для указанного номера повторения.
/// </summary>
/// <param name="retrySettings">Настройки повторной обработки сообщений.</param>
/// <param name="retryNumber">Номер повторной обработки.</param>
/// <returns>Временной интервал.</returns>
public TimeSpan Compute(RetrySettings retrySettings, int retryNumber)
{
// для больших значений retryNumber, Math.Pow быстро возвращает значение бесконечности,
// поэтому дополнительных проверок делать не нужно.
var newDelayTimeInSeconds = Math.Pow(2, retryNumber);
// Если Delay превысил максимальное время выполнения, ограничиваем максимальным значением.
if (newDelayTimeInSeconds > retrySettings.RetryMaxDelayInSeconds)
{
newDelayTimeInSeconds = retrySettings.RetryMaxDelayInSeconds;
}
return(TimeSpan.FromSeconds(newDelayTimeInSeconds));
}
// TODO: This is a modified/simplified version of ApiCallRetryExtensions.WithRetry from Google.Api.Gax.Grpc. Can we combine them somehow?
internal static async Task RetryOperationUntilCompleted(
Func <Task <bool> > fn,
IClock clock,
IScheduler scheduler,
CallSettings callSettings,
RetrySettings retrySettings)
{
DateTime?overallDeadline = retrySettings.TotalExpiration.CalculateDeadline(clock);
TimeSpan retryDelay = retrySettings.RetryBackoff.Delay;
TimeSpan callTimeout = retrySettings.TimeoutBackoff.Delay;
while (true)
{
DateTime attemptDeadline = clock.GetCurrentDateTimeUtc() + callTimeout;
// Note: this handles a null total deadline due to "<" returning false if overallDeadline is null.
DateTime combinedDeadline = overallDeadline < attemptDeadline ? overallDeadline.Value : attemptDeadline;
CallSettings attemptCallSettings = callSettings.WithCallTiming(CallTiming.FromDeadline(combinedDeadline));
TimeSpan actualDelay = retrySettings.DelayJitter.GetDelay(retryDelay);
DateTime expectedRetryTime;
try
{
bool isResponseOk = await fn().ConfigureAwait(false);
if (isResponseOk)
{
return;
}
expectedRetryTime = clock.GetCurrentDateTimeUtc() + actualDelay;
if (expectedRetryTime > overallDeadline)
{
// TODO: Can we get this string from somewhere?
throw new RpcException(new Status(StatusCode.DeadlineExceeded, "Deadline Exceeded"));
}
}
catch (RpcException e) when(retrySettings.RetryFilter(e))
{
expectedRetryTime = clock.GetCurrentDateTimeUtc() + actualDelay;
if (expectedRetryTime > overallDeadline)
{
throw;
}
}
await scheduler.Delay(actualDelay, callSettings.CancellationToken.GetValueOrDefault()).ConfigureAwait(false);
retryDelay = retrySettings.RetryBackoff.NextDelay(retryDelay);
callTimeout = retrySettings.TimeoutBackoff.NextDelay(callTimeout);
}
}
public void NextBackoff(double current, double expectedNext)
{
var settings = new RetrySettings(
maxAttempts: 5,
initialBackoff: TimeSpan.FromSeconds(1.0),
maxBackoff: TimeSpan.FromSeconds(5.0),
backoffMultiplier: 2.0,
e => true,
RetrySettings.NoJitter);
var expected = TimeSpan.FromSeconds(expectedNext);
var actual = settings.NextBackoff(TimeSpan.FromSeconds(current));
Assert.Equal(expected, actual);
}
public void WithExpiration_SettingsWithRetry()
{
var token = new CancellationTokenSource().Token;
var backoffSettings = new BackoffSettings(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10), 1.5);
var retry = new RetrySettings(backoffSettings, backoffSettings, Expiration.FromTimeout(TimeSpan.FromSeconds(5)));
var originalTiming = CallTiming.FromRetry(retry);
CallSettings settings = CallSettings.FromCancellationToken(token).WithCallTiming(originalTiming);
Expiration expiration = Expiration.FromTimeout(TimeSpan.FromSeconds(1));
var result = settings.WithExpiration(expiration);
Assert.Same(expiration, result.Timing.Retry.TotalExpiration);
Assert.Same(backoffSettings, result.Timing.Retry.RetryBackoff);
Assert.Same(backoffSettings, result.Timing.Retry.TimeoutBackoff);
Assert.Equal(token, result.CancellationToken);
}
public void WithNewExpiration()
{
var retryBackoff = new BackoffSettings(TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(3));
var timeoutBackoff = new BackoffSettings(TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(3));
var original = new RetrySettings(retryBackoff, timeoutBackoff, Expiration.FromTimeout(TimeSpan.FromSeconds(5)),
e => false, RetrySettings.NoJitter);
var newExpiration = Expiration.FromDeadline(DateTime.UtcNow);
var newSettings = original.WithTotalExpiration(newExpiration);
Assert.Same(original.RetryBackoff, newSettings.RetryBackoff);
Assert.Same(original.TimeoutBackoff, newSettings.TimeoutBackoff);
Assert.Same(original.RetryFilter, newSettings.RetryFilter);
Assert.Same(original.DelayJitter, newSettings.DelayJitter);
Assert.Same(newExpiration, newSettings.TotalExpiration);
}
/// <summary>
/// Constructor with complete control that does not perform any validation.
/// </summary>
internal ResultStream(
SpannerClient client,
ReadOrQueryRequest request,
Session session,
CallSettings callSettings,
int maxBufferSize,
RetrySettings retrySettings)
{
_buffer = new LinkedList <PartialResultSet>();
_client = GaxPreconditions.CheckNotNull(client, nameof(client));
_request = GaxPreconditions.CheckNotNull(request, nameof(request));
_session = GaxPreconditions.CheckNotNull(session, nameof(session));
_callSettings = callSettings;
_maxBufferSize = GaxPreconditions.CheckArgumentRange(maxBufferSize, nameof(maxBufferSize), 1, 10_000);
_retrySettings = GaxPreconditions.CheckNotNull(retrySettings, nameof(retrySettings));
}
/// <summary>
/// Установить информацию о повторной обработке в текущий скоуп логгера.
/// </summary>
/// <param name="properties">Метаданные сообщения.</param>
/// <param name="subscriberSettings">Настройки повтора подписчика.</param>
/// <param name="loggingScope">Скоуп.</param>
/// <returns>Номер повторной попытки, признак попытки последней обработки.</returns>
internal static (int retryNumber, bool isLastRetry) EnsureRetryInfo(
this IBasicProperties properties,
RetrySettings subscriberSettings,
Dictionary <string, object?> loggingScope
)
{
var isLastRetry = properties.IsLastRetry(subscriberSettings, out var retryCount);
loggingScope["RetryNumber"] = retryCount;
if (isLastRetry)
{
loggingScope["IsLastRetry"] = true;
}
return(retryCount, isLastRetry);
}
public override void SetUp()
{
_latched = false;
_unlatched = false;
_pingFails = false;
theSettings = new RetrySettings();
theRetryAgent = new LightweightRetryAgent(this, theSettings);
for (int i = 0; i < batches.Length; i++)
{
batches[i] = batchForEnvelopes(15);
}
_enqueued.Clear();
}
public LogUploader(
LoggingServiceV2Client client,
IScheduler scheduler,
IClock clock,
ILogQueue logQ,
LogEntry logsLostWarningTemplate,
int maxUploadBatchSize,
RetrySettings serverErrorRetrySettings)
{
_client = client;
_scheduler = scheduler;
_clock = clock;
_logQ = logQ;
_logsLostWarningTemplate = logsLostWarningTemplate;
_maxUploadBatchSize = maxUploadBatchSize;
_serverErrorRetrySettings = serverErrorRetrySettings;
_uploaderTaskCancellation = new CancellationTokenSource();
_uploaderTask = Task.Run(() => RunUploader(_uploaderTaskCancellation.Token));
}
internal WatchStream(IScheduler scheduler, IWatchState state, Target target, FirestoreDb db, CancellationToken cancellationToken)
{
_scheduler = scheduler;
_state = state;
_target = target;
_db = db;
_callbackCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
_networkCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(_callbackCancellationTokenSource.Token);
_listenCallSettings = CallSettings.FromHeader(FirestoreClientImpl.ResourcePrefixHeader, db.RootPath);
// TODO: Make these configurable?
_backoffSettings = RetrySettings.FromExponentialBackoff(
maxAttempts: int.MaxValue,
initialBackoff: TimeSpan.FromSeconds(1),
maxBackoff: TimeSpan.FromSeconds(30),
backoffMultiplier: 2.0,
retryFilter: _ => false, // Ignored
backoffJitter: RetrySettings.RandomJitter);
}
/// <summary>
/// Получить, является ли текущая попытка обработки последней.
/// </summary>
/// <param name="properties">Метаданные сообщения.</param>
/// <param name="retrySettings">Настройки повтора подписчика.</param>
/// <param name="retryCount">Количество повторных попыток.</param>
/// <returns>Признак попытки последней обработки.</returns>
internal static bool IsLastRetry(
this IBasicProperties properties,
RetrySettings retrySettings,
out int retryCount
)
{
retryCount = properties.GetRetryNumber();
if (!retrySettings.IsEnabled)
{
return(true);
}
if (retrySettings.DoInfinityRetries)
{
return(false);
}
return(retryCount >= retrySettings.RetryCount);
}
/// <summary>
/// Specifies the current constraint must be retried a certain amount of time
/// </summary>
public static IResolveConstraint Retry(this IResolveConstraint source, int count = -1, int timeGap = -1)
{
if (count <= 0)
{
count = DefaultRetryCount;
}
if (timeGap <= 0)
{
timeGap = DefaultRetryTimeGap;
}
var settings = new RetrySettings(retryCount: count, timeGap: timeGap);
if (source is IConstraint constraint)
{
return(new RetryableWithResolveConstraintImpl(wrapped: constraint, retrySettings: settings));
}
return(new RetryableResolveConstraintImpl(wrapped: source, retrySettings: settings));
}
public SubscriberSettings Create(MqConnectionSettings mqConnectionSettings)
{
return(new SubscriberSettings(
mqConnectionSettings,
SubscriberName,
QueueName,
(Bindings ?? Enumerable.Empty <ExchangeBindingDto>()).Select(b => b.Create()),
Arguments,
UseModelTypeAsSuffix,
ConsumerName,
Durable,
Exclusive,
AutoDelete,
AutoAck,
UseDeadLetter,
UsePublisherConfirms,
TracingSettings?.Create(),
RetrySettings?.Create(),
ScalingSettings?.Create()
));
}
internal static async Task RetryOperationUntilCompleted(
Func <CallSettings, Task <bool> > fn,
IClock clock,
IScheduler scheduler,
CallSettings callSettings,
RetrySettings retrySettings)
{
DateTime?overallDeadline = callSettings.Expiration.CalculateDeadline(clock);
// Every attempt should use the same deadline, calculated from the start of the call.
if (callSettings.Expiration?.Type == ExpirationType.Timeout)
{
callSettings = callSettings.WithDeadline(overallDeadline.Value);
}
var deadlineException = new RpcException(new Status(StatusCode.DeadlineExceeded, "Deadline Exceeded"));
foreach (var attempt in RetryAttempt.CreateRetrySequence(retrySettings, scheduler, overallDeadline, clock))
{
try
{
bool isResponseOk = await fn(callSettings).ConfigureAwait(false);
if (isResponseOk)
{
return;
}
if (!attempt.ShouldRetry(deadlineException))
{
throw deadlineException;
}
}
catch (RpcException e) when(attempt.ShouldRetry(e))
{
// We back off below...
}
await attempt.BackoffAsync(callSettings.CancellationToken.GetValueOrDefault()).ConfigureAwait(false);
}
throw new InvalidOperationException("Bug in GAX retry handling: finished sequence of attempts");
}
internal static async Task <TResponse> Retry <TRequest, TResponse>(
Func <TRequest, CallSettings, Task <TResponse> > fn,
TRequest request, CallSettings callSettings, IClock clock, IScheduler scheduler)
{
RetrySettings retrySettings = callSettings.Timing?.Retry;
if (retrySettings == null)
{
return(await fn(request, callSettings).ConfigureAwait(false));
}
DateTime?overallDeadline = retrySettings.TotalExpiration.CalculateDeadline(clock);
TimeSpan retryDelay = retrySettings.RetryBackoff.Delay;
TimeSpan callTimeout = retrySettings.TimeoutBackoff.Delay;
while (true)
{
DateTime attemptDeadline = clock.GetCurrentDateTimeUtc() + callTimeout;
// Note: this handles a null total deadline due to "<" returning false if overallDeadline is null.
DateTime combinedDeadline = overallDeadline < attemptDeadline ? overallDeadline.Value : attemptDeadline;
CallSettings attemptCallSettings = callSettings.WithCallTiming(CallTiming.FromDeadline(combinedDeadline));
try
{
return(await fn(request, attemptCallSettings).ConfigureAwait(false));
}
catch (RpcException e) when(retrySettings.RetryFilter(e))
{
TimeSpan actualDelay = retrySettings.DelayJitter.GetDelay(retryDelay);
DateTime expectedRetryTime = clock.GetCurrentDateTimeUtc() + actualDelay;
if (expectedRetryTime > overallDeadline)
{
throw;
}
await scheduler.Delay(actualDelay, callSettings.CancellationToken.GetValueOrDefault()).ConfigureAwait(false);
retryDelay = retrySettings.RetryBackoff.NextDelay(retryDelay);
callTimeout = retrySettings.TimeoutBackoff.NextDelay(callTimeout);
}
}
}
public async Task <Geeklist> GetGeeklist(int id, RetrySettings retrySettings, bool getComments = false)
{
string uri = $"{Endpoints.GET_GEEKLIST}/{id}";
if (getComments)
{
uri = $"{uri}?comments=1";
}
HttpResponseMessage resp = await xmlRestClient.GetWithRetryAsync(uri, retrySettings);
if (resp.StatusCode != HttpStatusCode.OK)
{
return(new Geeklist {
StatusCode = (int)resp.StatusCode
});
}
_GeeklistResult result = await resp.Content.DeserializeXml <_GeeklistResult>();
return(result.ConvertToGeeklist());
}
