public async Task RetryCancellation([CombinatorialValues(1500, 3500)] int delayMs) { // Note: Cannot test cancellation during wait for response header, due to FakeScheduler shortcomings. var async = true; var scheduler = new FakeScheduler(); var time0 = scheduler.Clock.GetCurrentDateTimeUtc(); var server = new Server(10, TimeSpan.FromSeconds(1), scheduler); var retrySettings = ConstantBackoff(5, TimeSpan.FromSeconds(1), NotFoundFilter); var delay = TimeSpan.FromMilliseconds(delayMs); Task task = scheduler.RunAsync(async() => { var cts = new CancellationTokenSource(); var unused = Task.Run(async() => { await scheduler.Delay(delay); cts.Cancel(); }); var callSettings = CallSettings.FromRetry(retrySettings).WithCancellationToken(cts.Token); var request = new SimpleRequest { Name = "irrelevant" }; await Call(async, scheduler, server, request, callSettings); }); await Assert.ThrowsAsync <TaskCanceledException>(() => task); Assert.Equal(time0 + delay, scheduler.Clock.GetCurrentDateTimeUtc()); }
public async Task FirstCallSucceeds(bool async) { var name = "name"; // Copied from request to response var scheduler = new FakeScheduler(); var time0 = scheduler.Clock.GetCurrentDateTimeUtc(); var server = new Server(0, TimeSpan.FromTicks(300), scheduler); var retrySettings = new RetrySettings( maxAttempts: 5, initialBackoff: TimeSpan.FromSeconds(1), maxBackoff: TimeSpan.FromSeconds(5), backoffMultiplier: 2.0, retryFilter: NotFoundFilter, backoffJitter: RetrySettings.NoJitter); await scheduler.RunAsync(async() => { var callSettings = CallSettings.FromRetry(retrySettings); var request = new SimpleRequest { Name = name }; var result = await Call(async, scheduler, server, request, callSettings); Assert.Equal(name, result.Name); }); server.AssertCallTimes(time0); // Time of last action was when the call returned Assert.Equal(300, scheduler.Clock.GetCurrentDateTimeUtc().Ticks); }
public async Task PublishMessageWithRetrySettingsAsync(string projectId, string topicId, string messageText) { TopicName topicName = TopicName.FromProjectTopic(projectId, topicId); // Retry settings control how the publisher handles retry-able failures var maxAttempts = 3; var initialBackoff = TimeSpan.FromMilliseconds(110); // default: 100 ms var maxBackoff = TimeSpan.FromSeconds(70); // default : 60 seconds var backoffMultiplier = 1.3; // default: 1.0 var totalTimeout = TimeSpan.FromSeconds(100); // default: 600 seconds var publisher = await PublisherClient.CreateAsync(topicName, clientCreationSettings : new PublisherClient.ClientCreationSettings( publisherServiceApiSettings: new PublisherServiceApiSettings { PublishSettings = CallSettings.FromRetry(RetrySettings.FromExponentialBackoff( maxAttempts: maxAttempts, initialBackoff: initialBackoff, maxBackoff: maxBackoff, backoffMultiplier: backoffMultiplier, retryFilter: RetrySettings.FilterForStatusCodes(StatusCode.Unavailable))) .WithTimeout(totalTimeout) } )).ConfigureAwait(false); string message = await publisher.PublishAsync(messageText); Console.WriteLine($"Published message {message}"); }
public async Task RetryFilter_EventualFailure(bool async, StatusCode failureCode, StatusCode[] filterCodes) { var callDuration = TimeSpan.FromTicks(100); var failures = 1; var scheduler = new FakeScheduler(); var server = new Server(failures, callDuration, scheduler, failureCode); // We're not really interested in the timing in this test. var retrySettings = new RetrySettings( maxAttempts: 5, initialBackoff: TimeSpan.Zero, maxBackoff: TimeSpan.Zero, backoffMultiplier: 1.0, retryFilter: RetrySettings.FilterForStatusCodes(filterCodes), backoffJitter: RetrySettings.NoJitter); var task = scheduler.RunAsync(async() => { var callSettings = CallSettings.FromRetry(retrySettings); var request = new SimpleRequest { Name = "irrelevant" }; await Call(async, scheduler, server, request, callSettings); }); await Assert.ThrowsAsync <RpcException>(() => task); Assert.Equal(1, server.CallTimes.Count()); }
public void FromRetry() { Assert.Null(CallSettings.FromRetry(null)); var retry = new RetrySettings(5, TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(5), 2.0, RetrySettings.FilterForStatusCodes(), RetrySettings.RandomJitter); var settings = CallSettings.FromRetry(retry); Assert.Same(retry, settings.Retry); }
public void FailWithRetry() { var apiCall = ApiBidirectionalStreamingCall.Create <int, int>( callOptions => null, CallSettings.FromRetry(new RetrySettings(5, TimeSpan.Zero, TimeSpan.Zero, 1.0, e => false, RetrySettings.RandomJitter)), new BidirectionalStreamingSettings(100), new FakeClock()); Assert.Throws <InvalidOperationException>(() => apiCall.Call(null)); }
public async Task FailWithRetry() { var apiCall = ApiServerStreamingCall.Create <int, int>( (request, callOptions) => null, CallSettings.FromRetry(new RetrySettings(5, TimeSpan.Zero, TimeSpan.Zero, 1.0, e => false, RetrySettings.RandomJitter)), new FakeClock()); await Assert.ThrowsAsync <InvalidOperationException>(() => apiCall.CallAsync(0, null)); Assert.Throws <InvalidOperationException>(() => apiCall.Call(0, null)); }
public async Task CallSettingsDeadlineIsObserved(bool async) { var callDuration = TimeSpan.FromTicks(300); var failures = 4; // Fifth call would succeed, but we won't get that far. var scheduler = new FakeScheduler(); var time0 = scheduler.Clock.GetCurrentDateTimeUtc(); var server = new Server(failures, callDuration, scheduler); var callable = server.Callable; var timeout = TimeSpan.FromTicks(2500); var retrySettings = new RetrySettings( maxAttempts: 5, initialBackoff: TimeSpan.FromTicks(1000), maxBackoff: TimeSpan.FromTicks(5000), backoffMultiplier: 2.0, retryFilter: NotFoundFilter, backoffJitter: RetrySettings.NoJitter); var task = scheduler.RunAsync(async() => { // Expiration makes it fail while waiting to make third call var callSettings = CallSettings.FromRetry(retrySettings).WithTimeout(timeout); var request = new SimpleRequest { Name = "irrelevant" }; await Call(async, scheduler, server, request, callSettings); }); await Assert.ThrowsAsync <RpcException>(() => task); var firstCall = time0; var secondCall = firstCall + callDuration + TimeSpan.FromTicks(1000); server.AssertCallTimes(firstCall, secondCall); // We use the same deadline for all calls. server.AssertDeadlines(time0 + timeout, time0 + timeout); // We fail immediately when we work out that we would time out before we make the third // call - so this is before the actual total timeout. Assert.Equal((secondCall + callDuration).Ticks, scheduler.Clock.GetCurrentDateTimeUtc().Ticks); }
private FirestoreDb(string projectId, string databaseId, FirestoreClient client, Action <string> warningLogger, SerializationContext serializationContext) { ProjectId = GaxPreconditions.CheckNotNull(projectId, nameof(projectId)); DatabaseId = GaxPreconditions.CheckNotNull(databaseId, nameof(databaseId)); Client = GaxPreconditions.CheckNotNull(client, nameof(client)); // TODO: Investigate using DatabaseName and DocumentPathName. RootPath = $"projects/{ProjectId}/databases/{DatabaseId}"; DocumentsPath = $"{RootPath}/documents"; WarningLogger = warningLogger; // TODO: Validate these settings, and potentially make them configurable var batchGetRetry = RetrySettings.FromExponentialBackoff( maxAttempts: int.MaxValue, initialBackoff: TimeSpan.FromMilliseconds(500), maxBackoff: TimeSpan.FromSeconds(5), backoffMultiplier: 2.0, retryFilter: RetrySettings.FilterForStatusCodes(StatusCode.Unavailable)); _batchGetCallSettings = CallSettings.FromRetry(batchGetRetry).WithTimeout(TimeSpan.FromMinutes(10)); SerializationContext = GaxPreconditions.CheckNotNull(serializationContext, nameof(serializationContext)); }
public async Task MaxAttemptsObserved(bool async) { var callDuration = TimeSpan.FromTicks(300); var failures = 4; // Fifth call would succeed, but we won't get that far. var scheduler = new FakeScheduler(); var time0 = scheduler.Clock.GetCurrentDateTimeUtc(); var server = new Server(failures, callDuration, scheduler); var callable = server.Callable; var retrySettings = ConstantBackoff(failures, TimeSpan.Zero, NotFoundFilter); var task = scheduler.RunAsync(async() => { // MaxAttempts makes the overall operation fail on the 4th RPC. var callSettings = CallSettings.FromRetry(retrySettings); var request = new SimpleRequest { Name = "irrelevant" }; await Call(async, scheduler, server, request, callSettings); }); await Assert.ThrowsAsync <RpcException>(() => task); Assert.Equal(4, server.CallTimes.Count); }
public async Task RetryFilter_EventualSuccess(bool async) { StatusCode failureCode = StatusCode.NotFound; StatusCode[] filterCodes = new[] { StatusCode.NotFound, StatusCode.DeadlineExceeded }; var callDuration = TimeSpan.FromTicks(100); var failures = 1; var scheduler = new FakeScheduler(); var server = new Server(failures, callDuration, scheduler, failureCode); // We're not really interested in the timing in this test. var retrySettings = ConstantBackoff(5, TimeSpan.Zero, RetrySettings.FilterForStatusCodes(filterCodes)); await scheduler.RunAsync(async() => { var callSettings = CallSettings.FromRetry(retrySettings); var request = new SimpleRequest { Name = "irrelevant" }; await Call(async, scheduler, server, request, callSettings); }); Assert.True(server.CallTimes.Count() > 1); }
private FirestoreDb(string projectId, string databaseId, FirestoreClient client, Action <string> warningLogger, SerializationContext serializationContext) { ProjectId = GaxPreconditions.CheckNotNull(projectId, nameof(projectId)); DatabaseId = GaxPreconditions.CheckNotNull(databaseId, nameof(databaseId)); Client = GaxPreconditions.CheckNotNull(client, nameof(client)); // TODO: Investigate using DatabaseName and DocumentPathName. RootPath = $"projects/{ProjectId}/databases/{DatabaseId}"; DocumentsPath = $"{RootPath}/documents"; WarningLogger = warningLogger; // TODO: Potentially make these configurable. // The retry settings are taken from firestore_grpc_service_config.json. var batchGetRetry = RetrySettings.FromExponentialBackoff( maxAttempts: 5, initialBackoff: TimeSpan.FromMilliseconds(100), maxBackoff: TimeSpan.FromSeconds(60), backoffMultiplier: 1.3, retryFilter: RetrySettings.FilterForStatusCodes(StatusCode.Unavailable, StatusCode.Internal, StatusCode.DeadlineExceeded)); _batchGetCallSettings = CallSettings.FromRetry(batchGetRetry).WithTimeout(TimeSpan.FromMinutes(10)); SerializationContext = GaxPreconditions.CheckNotNull(serializationContext, nameof(serializationContext)); }
public async Task MultipleCallsEventualSuccess(bool async) { var callDuration = TimeSpan.FromTicks(300); var failures = 4; // Fifth call will succeed var name = "name"; // Copied from request to response var scheduler = new FakeScheduler(); var time0 = scheduler.Clock.GetCurrentDateTimeUtc(); var server = new Server(failures, callDuration, scheduler); var retrySettings = new RetrySettings( maxAttempts: 5, initialBackoff: TimeSpan.FromTicks(1000), maxBackoff: TimeSpan.FromTicks(5000), backoffMultiplier: 2.0, retryFilter: NotFoundFilter, backoffJitter: RetrySettings.NoJitter); await scheduler.RunAsync(async() => { var callSettings = CallSettings.FromRetry(retrySettings); var request = new SimpleRequest { Name = name }; var result = await Call(async, scheduler, server, request, callSettings); Assert.Equal(name, result.Name); }); var firstCall = time0; var secondCall = firstCall + callDuration + TimeSpan.FromTicks(1000); // Delay for 1000 ticks var thirdCall = secondCall + callDuration + TimeSpan.FromTicks(2000); // Delay for 2000 ticks var fourthCall = thirdCall + callDuration + TimeSpan.FromTicks(4000); // Delay for 4000 ticks var fifthCall = fourthCall + callDuration + TimeSpan.FromTicks(5000); // Delay for 5000 ticks, as that's the max server.AssertCallTimes(firstCall, secondCall, thirdCall, fourthCall, fifthCall); // Time of last action was when the call returned Assert.Equal(fifthCall + callDuration, scheduler.Clock.GetCurrentDateTimeUtc()); }
private async Task RunBulkMessagingImpl( TopicName topicName, SubscriptionName subscriptionName, int messageCount, int minMessageSize, int maxMessageSize, int maxMessagesInFlight, int initialNackCount, TimeSpan?timeouts = null, int?cancelAfterRecvCount = null, TimeSpan?interPublishDelay = null, TimeSpan?debugOutputPeriod = null, int?publisherChannelCount = null, int?clientCount = null) { // Force messages to be at least 4 bytes long, so an int ID can be used. minMessageSize = Math.Max(4, minMessageSize); // Create PublisherClient and SubscriberClient var publisher = await PublisherClient.CreateAsync(topicName, clientCreationSettings : new PublisherClient.ClientCreationSettings( clientCount: publisherChannelCount, publisherServiceApiSettings: timeouts == null ? null : new PublisherServiceApiSettings { PublishSettings = CallSettings .FromRetry(RetrySettings.FromExponentialBackoff( maxAttempts: int.MaxValue, initialBackoff: TimeSpan.FromMilliseconds(100), maxBackoff: TimeSpan.FromSeconds(6), backoffMultiplier: 1.3, retryFilter: RetrySettings.FilterForStatusCodes(StatusCode.Unavailable))) .WithTimeout(timeouts.Value) } )).ConfigureAwait(false); var subscriber = await SubscriberClient.CreateAsync(subscriptionName, clientCreationSettings : new SubscriberClient.ClientCreationSettings(clientCount: clientCount), settings : new SubscriberClient.Settings { AckDeadline = timeouts, FlowControlSettings = new FlowControlSettings(maxMessagesInFlight, null) }).ConfigureAwait(false); Console.WriteLine("Topic, Subscription, Publisher and Subscriber all created"); // Subscribe int recvCount = 0; // Count of received messages int dupCount = 0; // Count of duplicate messages long recvSum = 0L; // Sum of bytes of received messages var recvedIds = new ConcurrentDictionary <int, bool>(); var nackedIds = new HashSet <int>(); Task subTask = subscriber.StartAsync((msg, ct) => { int id = BitConverter.ToInt32(msg.Data.ToArray(), 0); lock (nackedIds) { if (nackedIds.Count < initialNackCount) { if (nackedIds.Add(id)) { // This ID not already nacked Interlocked.Increment(ref recvCount); return(Task.FromResult(SubscriberClient.Reply.Nack)); } } } bool wasAdded = recvedIds.TryAdd(id, false); if (wasAdded) { var localRecvCount = Interlocked.Increment(ref recvCount); Interlocked.Add(ref recvSum, msg.Data.Sum(x => (long)x)); if (localRecvCount == cancelAfterRecvCount || localRecvCount >= messageCount + initialNackCount) { // Test finished, so stop subscriber Console.WriteLine("All msgs received, stopping subscriber."); Task unused = subscriber.StopAsync(TimeSpan.FromSeconds(15)); } } else { Interlocked.Add(ref dupCount, 1); } // ACK all messages return(Task.FromResult(SubscriberClient.Reply.Ack)); }); // Publish var rnd = new Random(1234); var activePubs = new HashSet <Task>(); int sentCount = 0; long sentSum = 0L; // Sum of bytes of sent messages // Watchdog to report progress and fail test on deadlock CancellationTokenSource watchdogCts = new CancellationTokenSource(); Task.Run(async() => { var debugOutputPeriod1 = debugOutputPeriod ?? TimeSpan.FromSeconds(1); int prevSentCount = -1; int prevRecvCount = -1; int noProgressCount = 0; while (true) { await Task.Delay(debugOutputPeriod1, watchdogCts.Token).ConfigureAwait(false); var localSentCount = Interlocked.Add(ref sentCount, 0); var localRecvCount = Interlocked.Add(ref recvCount, 0); var localDupCount = Interlocked.Add(ref dupCount, 0); if (prevSentCount == localSentCount && prevRecvCount == localRecvCount) { if (noProgressCount > 60) { // Deadlock, shutdown subscriber, and cancel Console.WriteLine("Deadlock detected. Cancelling test"); subscriber.StopAsync(new CancellationToken(true)); watchdogCts.Cancel(); break; } noProgressCount += 1; } else { noProgressCount = 0; } prevSentCount = localSentCount; prevRecvCount = localRecvCount; Console.WriteLine($"[{DateTime.Now}] Sent: {localSentCount} (in-flight: {activePubs.Locked(() => activePubs.Count)}); Recv: {localRecvCount} (dups: {localDupCount})"); } }); for (int i = 0; i < messageCount; i++) { if (watchdogCts.IsCancellationRequested) { Assert.True(false, "Test cancelled by watchdog"); } if (subTask.IsCompleted) { break; } if (i > 0 && interPublishDelay is TimeSpan delay) { await Task.Delay(delay, watchdogCts.Token).ConfigureAwait(false); } var msgSize = rnd.Next(minMessageSize, maxMessageSize + 1); var msg = new byte[msgSize]; rnd.NextBytes(msg); // Insert an int ID into message Array.Copy(BitConverter.GetBytes(i), msg, 4); sentSum += msg.Sum(x => (long)x); // Send message, and record Task var pubTask = publisher.PublishAsync(msg); Interlocked.Increment(ref sentCount); activePubs.Locked(() => activePubs.Add(pubTask)); // Remove Task from active when the message has been sent to server pubTask.ContinueWith(t => activePubs.Locked(() => activePubs.Remove(pubTask))); // If too many messages are currently in flight, wait a bit while (activePubs.Locked(() => activePubs.Count) >= maxMessagesInFlight) { await Task.Delay(TimeSpan.FromMilliseconds(1)).ConfigureAwait(false); } } Console.WriteLine("Publishing complete"); // Wait for all messages to be sent to server await Task.WhenAll(activePubs.Locked(() => activePubs.ToArray())).ConfigureAwait(false); Console.WriteLine("Publishing completed sending to server"); // Wait for subscriber to finish shutdown await subTask.ConfigureAwait(false); watchdogCts.Cancel(); Console.WriteLine("Subscriber finished shutdown"); Console.WriteLine($"Sent: {sentCount}; Recv: {recvCount}"); if (cancelAfterRecvCount is int cancelAfter) { Assert.True(recvCount >= cancelAfter && recvCount <= cancelAfter + maxMessagesInFlight, $"Incorrect recvCount: {recvCount}"); } else { // Check that all messages are correctly received. Assert.Equal(messageCount + initialNackCount, recvCount); // This isn't foolproof (we can get to the right sum with wrong values) but it's a pretty strong indicator. Assert.Equal(sentSum, recvSum); } Console.WriteLine("Test complete."); }