public async Task FirstCallSucceeds(bool async) { var name = "name"; // Copied from request to response var scheduler = new FakeScheduler(); var time0 = scheduler.Clock.GetCurrentDateTimeUtc(); var server = new Server(0, TimeSpan.FromTicks(300), scheduler); var retrySettings = new RetrySettings( maxAttempts: 5, initialBackoff: TimeSpan.FromSeconds(1), maxBackoff: TimeSpan.FromSeconds(5), backoffMultiplier: 2.0, retryFilter: NotFoundFilter, backoffJitter: RetrySettings.NoJitter); await scheduler.RunAsync(async() => { var callSettings = CallSettings.FromRetry(retrySettings); var request = new SimpleRequest { Name = name }; var result = await Call(async, scheduler, server, request, callSettings); Assert.Equal(name, result.Name); }); server.AssertCallTimes(time0); // Time of last action was when the call returned Assert.Equal(300, scheduler.Clock.GetCurrentDateTimeUtc().Ticks); }
public async Task RetryFilter_EventualSuccess(bool async, StatusCode failureCode, StatusCode[] filterCodes) { var callDuration = 100; var failures = 1; var scheduler = new FakeScheduler(); var server = new Server(failures, callDuration, scheduler, failureCode); // We're not really interested in the timing in this test. var retrySettings = new RetrySettings( retryBackoff: ConstantBackoff, timeoutBackoff: ConstantBackoff, delayJitter: RetrySettings.NoJitter, totalExpiration: Expiration.FromTimeout(TimeSpan.FromSeconds(1)), retryFilter: RetrySettings.FilterForStatusCodes(filterCodes)); await scheduler.RunAsync(async() => { var callSettings = CallSettings.FromCallTiming(CallTiming.FromRetry(retrySettings)); var retryingCallable = server.Callable.WithRetry(scheduler.Clock, scheduler); await Call(async, retryingCallable, new SimpleRequest { Name = "irrelevant" }, callSettings); }); Assert.True(server.CallTimes.Count() > 1); }
private Task <SimpleResponse> Call( bool async, FakeScheduler scheduler, Server server, SimpleRequest request, CallSettings callSettings) { var retryingCallable = server.Callable.WithRetry(scheduler.Clock, scheduler, NullLogger.Instance); return(Call(async, retryingCallable, request, callSettings)); }
public async Task FirstCallSucceeds(bool async, bool serverStreaming) { var name = "name"; // Copied from request to response var scheduler = new FakeScheduler(); var time0 = scheduler.Clock.GetCurrentDateTimeUtc(); var server = new Server(0, TimeSpan.FromTicks(300), scheduler); var retrySettings = new RetrySettings( retryBackoff: DoublingBackoff, timeoutBackoff: ConstantBackoff, totalExpiration: Expiration.FromTimeout(TimeSpan.FromSeconds(1)), retryFilter: null, delayJitter: RetrySettings.NoJitter); await scheduler.RunAsync(async() => { var callSettings = CallSettings.FromCallTiming(CallTiming.FromRetry(retrySettings)); var request = new SimpleRequest { Name = name }; var result = await Call(async, serverStreaming, scheduler, server, request, callSettings); Assert.Equal(name, result.Name); }); server.AssertCallTimes(time0); // Time of last action was when the call returned Assert.Equal(300, scheduler.Clock.GetCurrentDateTimeUtc().Ticks); }
internal Server(int failuresToReturn, TimeSpan callDuration, FakeScheduler scheduler, StatusCode failureCode = StatusCode.NotFound) { _callDuration = callDuration; _failuresToReturn = failuresToReturn; _failureCode = failureCode; _scheduler = scheduler; }
public async Task RetryFilter_EventualFailure(bool async, StatusCode failureCode, StatusCode[] filterCodes) { var callDuration = TimeSpan.FromTicks(100); var failures = 1; var scheduler = new FakeScheduler(); var server = new Server(failures, callDuration, scheduler, failureCode); // We're not really interested in the timing in this test. var retrySettings = new RetrySettings( maxAttempts: 5, initialBackoff: TimeSpan.Zero, maxBackoff: TimeSpan.Zero, backoffMultiplier: 1.0, retryFilter: RetrySettings.FilterForStatusCodes(filterCodes), backoffJitter: RetrySettings.NoJitter); var task = scheduler.RunAsync(async() => { var callSettings = CallSettings.FromRetry(retrySettings); var request = new SimpleRequest { Name = "irrelevant" }; await Call(async, scheduler, server, request, callSettings); }); await Assert.ThrowsAsync <RpcException>(() => task); Assert.Equal(1, server.CallTimes.Count()); }
public async Task ExponentialTimeouts(bool async, bool serverStreaming) { var callDuration = TimeSpan.FromTicks(300); var failures = 2; var scheduler = new FakeScheduler(); var time0 = scheduler.Clock.GetCurrentDateTimeUtc(); var server = new Server(failures, callDuration, scheduler); var callable = server.Callable; var retrySettings = new RetrySettings( retryBackoff: ConstantBackoff, // 1500 ticks always timeoutBackoff: DoublingBackoff, // 1000, then 2000, then 4000 totalExpiration: Expiration.FromTimeout(TimeSpan.FromTicks(4500)), retryFilter: null, delayJitter: RetrySettings.NoJitter); await scheduler.RunAsync(async() => { // Expiration truncates the third timeout. We expect: // Call 1: t=0, deadline=1000, completes at 300 // Call 2: t=1800, deadline=3800 (2000+1800), completes at 2100 // Call 3, t=3600, deadline=4500 (would be 7600, but overall deadline truncates), completes at 3900 (with success) var callSettings = CallSettings.FromCallTiming(CallTiming.FromRetry(retrySettings)); var request = new SimpleRequest { Name = "irrelevant" }; await Call(async, serverStreaming, scheduler, server, request, callSettings); }); server.AssertCallTimes(time0, time0 + TimeSpan.FromTicks(1800), time0 + TimeSpan.FromTicks(3600)); server.AssertDeadlines(time0 + TimeSpan.FromTicks(1000), time0 + TimeSpan.FromTicks(3800), time0 + TimeSpan.FromTicks(4500)); Assert.Equal(3900L, scheduler.Clock.GetCurrentDateTimeUtc().Ticks); }
internal Server(int failuresToReturn, long tickCallDuration, FakeScheduler scheduler, StatusCode failureCode = StatusCode.NotFound) { callDuration = TimeSpan.FromTicks(tickCallDuration); this.failuresToReturn = failuresToReturn; this.failureCode = failureCode; this.scheduler = scheduler; }
public void Should_collect_and_process_data_from_telemetry_collector_on_each_scheduler_tick() { //Arrange var fakeTelemetryProcessor = new FakeTelemetryProcessor(); var fakeTelemetryCollector = new FakeTelemetryCollector(); fakeTelemetryCollector.ExptectedTelemetryData = new[] { new TelemetryData { Kind = "Pressure", Value = "10 psi" }, new TelemetryData { Kind = "Temperature", Value = "70 °F" }, }; Func <string, ITelemetryProcessor> processorFactory = (string telemetryType) => { return(fakeTelemetryProcessor); }; var fakeScheduler = new FakeScheduler(); //Act var telemetryClient = new TelemetryClient(fakeTelemetryCollector, fakeScheduler, processorFactory); telemetryClient.Start(); fakeScheduler.RaiseElaspedEvent(); //Assert Assert.That(() => fakeTelemetryProcessor.ReceivedTelemetryData.Count(data => data.Kind == "Pressure" && data.Value == "10 psi") == 1); Assert.That(() => fakeTelemetryProcessor.ReceivedTelemetryData.Count(data => data.Kind == "Temperature" && data.Value == "70 °F") == 1); }
public async Task CallSettingsDeadlineIsObserved(bool async, bool serverStreaming) { var callDuration = TimeSpan.FromTicks(300); var failures = 4; // Fifth call would succeed, but we won't get that far. var scheduler = new FakeScheduler(); var time0 = scheduler.Clock.GetCurrentDateTimeUtc(); var server = new Server(failures, callDuration, scheduler); var callable = server.Callable; var retrySettings = new RetrySettings( retryBackoff: DoublingBackoff, timeoutBackoff: ConstantBackoff, totalExpiration: Expiration.FromTimeout(TimeSpan.FromTicks(2500)), retryFilter: null, delayJitter: RetrySettings.NoJitter); var task = scheduler.RunAsync(async() => { // Expiration makes it fail while waiting to make third call var callSettings = CallSettings.FromCallTiming(CallTiming.FromRetry(retrySettings)); var request = new SimpleRequest { Name = "irrelevant" }; await Call(async, serverStreaming, scheduler, server, request, callSettings); }); await Assert.ThrowsAsync <RpcException>(() => task); var firstCall = time0; var secondCall = firstCall + callDuration + TimeSpan.FromTicks(1000); server.AssertCallTimes(firstCall, secondCall); // We fail immediately when we work out that we would time out before we make the third // call - so this is before the actual total timeout. Assert.Equal((secondCall + callDuration).Ticks, scheduler.Clock.GetCurrentDateTimeUtc().Ticks); }
public async Task RetryFilter_EventualFailure(bool async, bool serverStreaming, StatusCode failureCode, StatusCode[] filterCodes) { var callDuration = TimeSpan.FromTicks(100); var failures = 1; var scheduler = new FakeScheduler(); var server = new Server(failures, callDuration, scheduler, failureCode); // We're not really interested in the timing in this test. var retrySettings = new RetrySettings( retryBackoff: ConstantBackoff, timeoutBackoff: ConstantBackoff, delayJitter: RetrySettings.NoJitter, totalExpiration: Expiration.FromTimeout(TimeSpan.FromSeconds(1)), retryFilter: RetrySettings.FilterForStatusCodes(filterCodes)); var task = scheduler.RunAsync(async() => { var callSettings = CallSettings.FromCallTiming(CallTiming.FromRetry(retrySettings)); var request = new SimpleRequest { Name = "irrelevant" }; await Call(async, serverStreaming, scheduler, server, request, callSettings); }); await Assert.ThrowsAsync <RpcException>(() => task); Assert.Equal(1, server.CallTimes.Count()); }
public async Task MultipleCallsEventualSuccess(bool async, bool serverStreaming) { var callDuration = TimeSpan.FromTicks(300); var failures = 4; // Fifth call will succeed var name = "name"; // Copied from request to response var scheduler = new FakeScheduler(); var time0 = scheduler.Clock.GetCurrentDateTimeUtc(); var server = new Server(failures, callDuration, scheduler); var retrySettings = new RetrySettings( retryBackoff: DoublingBackoff, timeoutBackoff: ConstantBackoff, totalExpiration: Expiration.FromTimeout(TimeSpan.FromSeconds(1)), retryFilter: null, delayJitter: RetrySettings.NoJitter); await scheduler.RunAsync(async() => { var callSettings = CallSettings.FromCallTiming(CallTiming.FromRetry(retrySettings)); var request = new SimpleRequest { Name = name }; var result = await Call(async, serverStreaming, scheduler, server, request, callSettings); Assert.Equal(name, result.Name); }); var firstCall = time0; var secondCall = firstCall + callDuration + TimeSpan.FromTicks(1000); // Delay for 1000 ticks var thirdCall = secondCall + callDuration + TimeSpan.FromTicks(2000); // Delay for 2000 ticks var fourthCall = thirdCall + callDuration + TimeSpan.FromTicks(4000); // Delay for 4000 ticks var fifthCall = fourthCall + callDuration + TimeSpan.FromTicks(5000); // Delay for 5000 ticks, as that's the max server.AssertCallTimes(firstCall, secondCall, thirdCall, fourthCall, fifthCall); // Time of last action was when the call returned Assert.Equal(fifthCall + callDuration, scheduler.Clock.GetCurrentDateTimeUtc()); }
public async Task RetryCancellation(bool serverStreaming, [CombinatorialValues(1500, 3500)] int delayMs) { // Note: Cannot test cancellation during wait for response header, due to FakeScheduler shortcomings. var async = true; var scheduler = new FakeScheduler(); var time0 = scheduler.Clock.GetCurrentDateTimeUtc(); var server = new Server(10, TimeSpan.FromSeconds(1), scheduler); var retrySettings = new RetrySettings( retryBackoff: new BackoffSettings(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(1), 1.0), timeoutBackoff: new BackoffSettings(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(1), 1.0), delayJitter: RetrySettings.NoJitter, totalExpiration: Expiration.FromTimeout(TimeSpan.FromSeconds(10)), retryFilter: RetrySettings.DefaultFilter); var delay = TimeSpan.FromMilliseconds(delayMs); Task task = scheduler.RunAsync(async() => { var cts = new CancellationTokenSource(); var unused = Task.Run(async() => { await scheduler.Delay(delay); cts.Cancel(); }); var callSettings = CallSettings.FromCallTiming(CallTiming.FromRetry(retrySettings)).WithCancellationToken(cts.Token); var request = new SimpleRequest { Name = "irrelevant" }; await Call(async, serverStreaming, scheduler, server, request, callSettings); }); await Assert.ThrowsAsync <TaskCanceledException>(() => task); Assert.Equal(time0 + delay, scheduler.Clock.GetCurrentDateTimeUtc()); }
public async Task RetryFilter_EventualSuccess(bool async, bool serverStreaming) { StatusCode failureCode = StatusCode.NotFound; StatusCode[] filterCodes = new[] { StatusCode.NotFound, StatusCode.DeadlineExceeded }; var callDuration = TimeSpan.FromTicks(100); var failures = 1; var scheduler = new FakeScheduler(); var server = new Server(failures, callDuration, scheduler, failureCode); // We're not really interested in the timing in this test. var retrySettings = new RetrySettings( retryBackoff: ConstantBackoff, timeoutBackoff: ConstantBackoff, delayJitter: RetrySettings.NoJitter, totalExpiration: Expiration.FromTimeout(TimeSpan.FromSeconds(1)), retryFilter: RetrySettings.FilterForStatusCodes(filterCodes)); await scheduler.RunAsync(async() => { var callSettings = CallSettings.FromCallTiming(CallTiming.FromRetry(retrySettings)); var request = new SimpleRequest { Name = "irrelevant" }; await Call(async, serverStreaming, scheduler, server, request, callSettings); }); Assert.True(server.CallTimes.Count() > 1); }
public async Task RetryCancellation([CombinatorialValues(1500, 3500)] int delayMs) { // Note: Cannot test cancellation during wait for response header, due to FakeScheduler shortcomings. var async = true; var scheduler = new FakeScheduler(); var time0 = scheduler.Clock.GetCurrentDateTimeUtc(); var server = new Server(10, TimeSpan.FromSeconds(1), scheduler); var retrySettings = ConstantBackoff(5, TimeSpan.FromSeconds(1), NotFoundFilter); var delay = TimeSpan.FromMilliseconds(delayMs); Task task = scheduler.RunAsync(async() => { var cts = new CancellationTokenSource(); var unused = Task.Run(async() => { await scheduler.Delay(delay); cts.Cancel(); }); var callSettings = CallSettings.FromRetry(retrySettings).WithCancellationToken(cts.Token); var request = new SimpleRequest { Name = "irrelevant" }; await Call(async, scheduler, server, request, callSettings); }); await Assert.ThrowsAsync <TaskCanceledException>(() => task); Assert.Equal(time0 + delay, scheduler.Clock.GetCurrentDateTimeUtc()); }
public async Task SimpleDelay() { var scheduler = new FakeScheduler(); await scheduler.RunAsync(() => scheduler.Delay(TimeSpan.FromTicks(1000))); Assert.Equal(1000, scheduler.Clock.GetCurrentDateTimeUtc().Ticks); }
public FakeOperationsClient(FakeScheduler scheduler, CallSettings baseCallSettings, Action <CallSettings> callSettingsValidation = null) { FakeScheduler = scheduler; _baseCallSettings = baseCallSettings; _callSettingsValidation = callSettingsValidation; operations = new Dictionary <string, Tuple <DateTime, Operation> >(); }
public Callee(FakeScheduler scheduler, int initialDelayMs = InitialDelayBackoffMs, int backoffMultiplier = BackoffMultiplier, int jitterMultiplier = JitterMultiplier) { _scheduler = scheduler; _initialDelayTicks = initialDelayMs * TimeSpan.TicksPerMillisecond; _backoffMultiplier = backoffMultiplier; _jitterMultiplier = jitterMultiplier; }
public void Setup() { _scheduler = new FakeScheduler() { UtcNow = new DateTime(2000, 1, 1) }; _deliveredMessages = new List <string>(); _service = new TimerService(_scheduler); }
internal TestSequence(CancellationToken initialCancellationToken = default) { _streams = new List <FakeWatchStream>(); _client = new FakeFirestoreListenClient(_streams, "projects/project/databases/db"); _watchState = new FakeWatchState(TriggerCompletion); var db = FirestoreDb.Create("project", "db", _client); var target = WatchStream.CreateTarget(db.Document("col/doc")); _scheduler = new FakeScheduler(); WatchStream = new WatchStream(_scheduler, _watchState, target, db, initialCancellationToken); }
public Task InitialBackoffOverrideRespected() { var settings = RetrySettings.FromConstantBackoff( maxAttempts: 4, backoff: FiveSeconds, retryFilter: ex => true, backoffJitter: RetrySettings.NoJitter); var scheduler = new FakeScheduler(); var sequence = RetryAttempt.CreateRetrySequence(settings, scheduler, initialBackoffOverride: OneSecond); // Should attempt at T=0, T=1, T=6, T=11. return(AssertAttemptsAsync(sequence, scheduler, () => new Exception(), 0, 1, 6, 11)); }
public Task MaxAttemptsRespected() { var settings = RetrySettings.FromConstantBackoff( maxAttempts: 4, backoff: OneSecond, retryFilter: ex => true, backoffJitter: RetrySettings.NoJitter); var scheduler = new FakeScheduler(); var sequence = RetryAttempt.CreateRetrySequence(settings, scheduler); // Should attempt at T=0, T=1, T=2, T=3, then stop because we're only allowed four attempts. return(AssertAttemptsAsync(sequence, scheduler, () => new Exception(), 0, 1, 2, 3)); }
public Task PredicateRespected() { int count = 0; Func <Exception> exceptionProvider = () => ++ count == 3 ? new Exception() : new RpcException(Status.DefaultCancelled); var settings = RetrySettings.FromExponentialBackoff( maxAttempts: 4, initialBackoff: OneSecond, maxBackoff: FiveSeconds, backoffMultiplier: 1, retryFilter: ex => ex is RpcException, backoffJitter: RetrySettings.NoJitter); var scheduler = new FakeScheduler(); var sequence = RetryAttempt.CreateRetrySequence(settings, scheduler); return(AssertAttemptsAsync(sequence, scheduler, exceptionProvider, 0, 1, 2)); }
public Task DeadlineRespected() { var settings = RetrySettings.FromExponentialBackoff( maxAttempts: 10, initialBackoff: OneSecond, maxBackoff: FiveSeconds, backoffMultiplier: 2, retryFilter: ex => true, backoffJitter: RetrySettings.NoJitter); var scheduler = new FakeScheduler(); var deadline = scheduler.Clock.GetCurrentDateTimeUtc() + FiveSeconds; var sequence = RetryAttempt.CreateRetrySequence(settings, scheduler, deadline, scheduler.Clock); // Should attempt at T=0, T=1, T=3, then stop because the next attempt would be after the deadline. return(AssertAttemptsAsync(sequence, scheduler, () => new Exception(), 0, 1, 3)); }
private Task <SimpleResponse> Call( bool async, bool serverStreaming, FakeScheduler scheduler, Server server, SimpleRequest request, CallSettings callSettings) { if (serverStreaming) { var retryingCallable = server.ServerStreamingCallable.WithRetry(scheduler.Clock, scheduler); return(Call(async, retryingCallable, request, callSettings)); } else { var retryingCallable = server.Callable.WithRetry(scheduler.Clock, scheduler); return(Call(async, retryingCallable, request, callSettings)); } }
public Task JitterRespected() { var settings = RetrySettings.FromExponentialBackoff( maxAttempts: 6, initialBackoff: TimeSpan.FromSeconds(2), maxBackoff: TimeSpan.FromSeconds(10), backoffMultiplier: 2, retryFilter: ex => true, backoffJitter: new HalvingJitter()); var scheduler = new FakeScheduler(); var sequence = RetryAttempt.CreateRetrySequence(settings, scheduler); // Sequence of theoretical backoffs is 2, 4, 8, 10, 10, 10 // Sequence of jittered backoffs is 1, 2, 4, 5, 5. return(AssertAttemptsAsync(sequence, scheduler, () => new Exception(), 0, 1, 3, 7, 12, 17)); }
public async Task CancelDelay() { var scheduler = new FakeScheduler(); var time0 = scheduler.Clock.GetCurrentDateTimeUtc(); var cts = new CancellationTokenSource(); Task task = scheduler.RunAsync(async() => { var unused = Task.Run(async() => { await scheduler.Delay(TimeSpan.FromSeconds(1)); cts.Cancel(); }); await scheduler.Delay(TimeSpan.FromSeconds(2), cts.Token); }); await Assert.ThrowsAsync <TaskCanceledException>(() => task); Assert.True(cts.IsCancellationRequested); Assert.Equal(time0 + TimeSpan.FromSeconds(1), scheduler.Clock.GetCurrentDateTimeUtc()); }
public void Should_start_scheduler_when_started() { //Arrange var fakeTelemetryProcessor = new FakeTelemetryProcessor(); var fakeTelemetryCollector = new FakeTelemetryCollector(); Func <string, ITelemetryProcessor> processorFactory = (string telemetryType) => { return(fakeTelemetryProcessor); }; var fakeScheduler = new FakeScheduler(); //Act var telemetryClient = new TelemetryClient(fakeTelemetryCollector, fakeScheduler, processorFactory); telemetryClient.Start(); //Assert Assert.True(fakeScheduler.Started); }
public async Task CallSettingsDeadlineIsObserved(bool async) { var callDuration = TimeSpan.FromTicks(300); var failures = 4; // Fifth call would succeed, but we won't get that far. var scheduler = new FakeScheduler(); var time0 = scheduler.Clock.GetCurrentDateTimeUtc(); var server = new Server(failures, callDuration, scheduler); var callable = server.Callable; var timeout = TimeSpan.FromTicks(2500); var retrySettings = new RetrySettings( maxAttempts: 5, initialBackoff: TimeSpan.FromTicks(1000), maxBackoff: TimeSpan.FromTicks(5000), backoffMultiplier: 2.0, retryFilter: NotFoundFilter, backoffJitter: RetrySettings.NoJitter); var task = scheduler.RunAsync(async() => { // Expiration makes it fail while waiting to make third call var callSettings = CallSettings.FromRetry(retrySettings).WithTimeout(timeout); var request = new SimpleRequest { Name = "irrelevant" }; await Call(async, scheduler, server, request, callSettings); }); await Assert.ThrowsAsync <RpcException>(() => task); var firstCall = time0; var secondCall = firstCall + callDuration + TimeSpan.FromTicks(1000); server.AssertCallTimes(firstCall, secondCall); // We use the same deadline for all calls. server.AssertDeadlines(time0 + timeout, time0 + timeout); // We fail immediately when we work out that we would time out before we make the third // call - so this is before the actual total timeout. Assert.Equal((secondCall + callDuration).Ticks, scheduler.Clock.GetCurrentDateTimeUtc().Ticks); }
public void Should_collect_data_from_telemetry_collector_on_each_scheduler_tick() { //Arrange var fakeTelemetryProcessor = new FakeTelemetryProcessor(); var fakeTelemetryCollector = new FakeTelemetryCollector(); fakeTelemetryCollector.ExptectedTelemetryData = Enumerable.Empty <TelemetryData>(); Func <string, ITelemetryProcessor> processorFactory = (string telemetryType) => { return(fakeTelemetryProcessor); }; var fakeScheduler = new FakeScheduler(); //Act var telemetryClient = new TelemetryClient(fakeTelemetryCollector, fakeScheduler, processorFactory); telemetryClient.Start(); fakeScheduler.RaiseElaspedEvent(); //Assert Assert.IsTrue(fakeTelemetryCollector.CollectWasInvoked); }
public void SynchronousSleep() { var scheduler = new FakeScheduler(); scheduler.Run(() => scheduler.Sleep(TimeSpan.FromTicks(1000))); Assert.Equal(1000, scheduler.Clock.GetCurrentDateTimeUtc().Ticks); }