Exemple #1
0
        public async Task RetryCancellation([CombinatorialValues(1500, 3500)] int delayMs)
        {
            // Note: Cannot test cancellation during wait for response header, due to FakeScheduler shortcomings.
            var  async         = true;
            var  scheduler     = new FakeScheduler();
            var  time0         = scheduler.Clock.GetCurrentDateTimeUtc();
            var  server        = new Server(10, TimeSpan.FromSeconds(1), scheduler);
            var  retrySettings = ConstantBackoff(5, TimeSpan.FromSeconds(1), NotFoundFilter);
            var  delay         = TimeSpan.FromMilliseconds(delayMs);
            Task task          = scheduler.RunAsync(async() =>
            {
                var cts    = new CancellationTokenSource();
                var unused = Task.Run(async() =>
                {
                    await scheduler.Delay(delay);
                    cts.Cancel();
                });
                var callSettings = CallSettings.FromRetry(retrySettings).WithCancellationToken(cts.Token);
                var request      = new SimpleRequest {
                    Name = "irrelevant"
                };
                await Call(async, scheduler, server, request, callSettings);
            });
            await Assert.ThrowsAsync <TaskCanceledException>(() => task);

            Assert.Equal(time0 + delay, scheduler.Clock.GetCurrentDateTimeUtc());
        }
Exemple #2
0
        public async Task FirstCallSucceeds(bool async)
        {
            var name          = "name"; // Copied from request to response
            var scheduler     = new FakeScheduler();
            var time0         = scheduler.Clock.GetCurrentDateTimeUtc();
            var server        = new Server(0, TimeSpan.FromTicks(300), scheduler);
            var retrySettings = new RetrySettings(
                maxAttempts: 5,
                initialBackoff: TimeSpan.FromSeconds(1),
                maxBackoff: TimeSpan.FromSeconds(5),
                backoffMultiplier: 2.0,
                retryFilter: NotFoundFilter,
                backoffJitter: RetrySettings.NoJitter);

            await scheduler.RunAsync(async() =>
            {
                var callSettings = CallSettings.FromRetry(retrySettings);
                var request      = new SimpleRequest {
                    Name = name
                };
                var result = await Call(async, scheduler, server, request, callSettings);
                Assert.Equal(name, result.Name);
            });

            server.AssertCallTimes(time0);
            // Time of last action was when the call returned
            Assert.Equal(300, scheduler.Clock.GetCurrentDateTimeUtc().Ticks);
        }
Exemple #3
0
    public async Task PublishMessageWithRetrySettingsAsync(string projectId, string topicId, string messageText)
    {
        TopicName topicName = TopicName.FromProjectTopic(projectId, topicId);
        // Retry settings control how the publisher handles retry-able failures
        var maxAttempts       = 3;
        var initialBackoff    = TimeSpan.FromMilliseconds(110); // default: 100 ms
        var maxBackoff        = TimeSpan.FromSeconds(70);       // default : 60 seconds
        var backoffMultiplier = 1.3;                            // default: 1.0
        var totalTimeout      = TimeSpan.FromSeconds(100);      // default: 600 seconds

        var publisher = await PublisherClient.CreateAsync(topicName,
                                                          clientCreationSettings : new PublisherClient.ClientCreationSettings(
                                                              publisherServiceApiSettings: new PublisherServiceApiSettings
        {
            PublishSettings = CallSettings.FromRetry(RetrySettings.FromExponentialBackoff(
                                                         maxAttempts: maxAttempts,
                                                         initialBackoff: initialBackoff,
                                                         maxBackoff: maxBackoff,
                                                         backoffMultiplier: backoffMultiplier,
                                                         retryFilter: RetrySettings.FilterForStatusCodes(StatusCode.Unavailable)))
                              .WithTimeout(totalTimeout)
        }
                                                              )).ConfigureAwait(false);

        string message = await publisher.PublishAsync(messageText);

        Console.WriteLine($"Published message {message}");
    }
Exemple #4
0
        public async Task RetryFilter_EventualFailure(bool async, StatusCode failureCode, StatusCode[] filterCodes)
        {
            var callDuration = TimeSpan.FromTicks(100);
            var failures     = 1;
            var scheduler    = new FakeScheduler();
            var server       = new Server(failures, callDuration, scheduler, failureCode);
            // We're not really interested in the timing in this test.
            var retrySettings = new RetrySettings(
                maxAttempts: 5,
                initialBackoff: TimeSpan.Zero,
                maxBackoff: TimeSpan.Zero,
                backoffMultiplier: 1.0,
                retryFilter: RetrySettings.FilterForStatusCodes(filterCodes),
                backoffJitter: RetrySettings.NoJitter);

            var task = scheduler.RunAsync(async() =>
            {
                var callSettings = CallSettings.FromRetry(retrySettings);
                var request      = new SimpleRequest {
                    Name = "irrelevant"
                };
                await Call(async, scheduler, server, request, callSettings);
            });
            await Assert.ThrowsAsync <RpcException>(() => task);

            Assert.Equal(1, server.CallTimes.Count());
        }
Exemple #5
0
        public void FromRetry()
        {
            Assert.Null(CallSettings.FromRetry(null));
            var retry    = new RetrySettings(5, TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(5), 2.0, RetrySettings.FilterForStatusCodes(), RetrySettings.RandomJitter);
            var settings = CallSettings.FromRetry(retry);

            Assert.Same(retry, settings.Retry);
        }
        public void FailWithRetry()
        {
            var apiCall = ApiBidirectionalStreamingCall.Create <int, int>(
                callOptions => null,
                CallSettings.FromRetry(new RetrySettings(5, TimeSpan.Zero, TimeSpan.Zero, 1.0, e => false, RetrySettings.RandomJitter)),
                new BidirectionalStreamingSettings(100),
                new FakeClock());

            Assert.Throws <InvalidOperationException>(() => apiCall.Call(null));
        }
        public async Task FailWithRetry()
        {
            var apiCall = ApiServerStreamingCall.Create <int, int>(
                (request, callOptions) => null,
                CallSettings.FromRetry(new RetrySettings(5, TimeSpan.Zero, TimeSpan.Zero, 1.0, e => false, RetrySettings.RandomJitter)),
                new FakeClock());
            await Assert.ThrowsAsync <InvalidOperationException>(() => apiCall.CallAsync(0, null));

            Assert.Throws <InvalidOperationException>(() => apiCall.Call(0, null));
        }
Exemple #8
0
        public async Task CallSettingsDeadlineIsObserved(bool async)
        {
            var callDuration  = TimeSpan.FromTicks(300);
            var failures      = 4; // Fifth call would succeed, but we won't get that far.
            var scheduler     = new FakeScheduler();
            var time0         = scheduler.Clock.GetCurrentDateTimeUtc();
            var server        = new Server(failures, callDuration, scheduler);
            var callable      = server.Callable;
            var timeout       = TimeSpan.FromTicks(2500);
            var retrySettings = new RetrySettings(
                maxAttempts: 5,
                initialBackoff: TimeSpan.FromTicks(1000),
                maxBackoff: TimeSpan.FromTicks(5000),
                backoffMultiplier: 2.0,
                retryFilter: NotFoundFilter,
                backoffJitter: RetrySettings.NoJitter);

            var task = scheduler.RunAsync(async() =>
            {
                // Expiration makes it fail while waiting to make third call
                var callSettings = CallSettings.FromRetry(retrySettings).WithTimeout(timeout);
                var request      = new SimpleRequest {
                    Name = "irrelevant"
                };
                await Call(async, scheduler, server, request, callSettings);
            });
            await Assert.ThrowsAsync <RpcException>(() => task);

            var firstCall  = time0;
            var secondCall = firstCall + callDuration + TimeSpan.FromTicks(1000);

            server.AssertCallTimes(firstCall, secondCall);
            // We use the same deadline for all calls.
            server.AssertDeadlines(time0 + timeout, time0 + timeout);

            // We fail immediately when we work out that we would time out before we make the third
            // call - so this is before the actual total timeout.
            Assert.Equal((secondCall + callDuration).Ticks, scheduler.Clock.GetCurrentDateTimeUtc().Ticks);
        }
Exemple #9
0
        private FirestoreDb(string projectId, string databaseId, FirestoreClient client, Action <string> warningLogger, SerializationContext serializationContext)
        {
            ProjectId  = GaxPreconditions.CheckNotNull(projectId, nameof(projectId));
            DatabaseId = GaxPreconditions.CheckNotNull(databaseId, nameof(databaseId));
            Client     = GaxPreconditions.CheckNotNull(client, nameof(client));
            // TODO: Investigate using DatabaseName and DocumentPathName.
            RootPath      = $"projects/{ProjectId}/databases/{DatabaseId}";
            DocumentsPath = $"{RootPath}/documents";
            WarningLogger = warningLogger;

            // TODO: Validate these settings, and potentially make them configurable
            var batchGetRetry = RetrySettings.FromExponentialBackoff(
                maxAttempts: int.MaxValue,
                initialBackoff: TimeSpan.FromMilliseconds(500),
                maxBackoff: TimeSpan.FromSeconds(5),
                backoffMultiplier: 2.0,
                retryFilter: RetrySettings.FilterForStatusCodes(StatusCode.Unavailable));

            _batchGetCallSettings = CallSettings.FromRetry(batchGetRetry).WithTimeout(TimeSpan.FromMinutes(10));

            SerializationContext = GaxPreconditions.CheckNotNull(serializationContext, nameof(serializationContext));
        }
Exemple #10
0
        public async Task MaxAttemptsObserved(bool async)
        {
            var callDuration  = TimeSpan.FromTicks(300);
            var failures      = 4; // Fifth call would succeed, but we won't get that far.
            var scheduler     = new FakeScheduler();
            var time0         = scheduler.Clock.GetCurrentDateTimeUtc();
            var server        = new Server(failures, callDuration, scheduler);
            var callable      = server.Callable;
            var retrySettings = ConstantBackoff(failures, TimeSpan.Zero, NotFoundFilter);

            var task = scheduler.RunAsync(async() =>
            {
                // MaxAttempts makes the overall operation fail on the 4th RPC.
                var callSettings = CallSettings.FromRetry(retrySettings);
                var request      = new SimpleRequest {
                    Name = "irrelevant"
                };
                await Call(async, scheduler, server, request, callSettings);
            });
            await Assert.ThrowsAsync <RpcException>(() => task);

            Assert.Equal(4, server.CallTimes.Count);
        }
Exemple #11
0
        public async Task RetryFilter_EventualSuccess(bool async)
        {
            StatusCode failureCode = StatusCode.NotFound;

            StatusCode[] filterCodes  = new[] { StatusCode.NotFound, StatusCode.DeadlineExceeded };
            var          callDuration = TimeSpan.FromTicks(100);
            var          failures     = 1;
            var          scheduler    = new FakeScheduler();
            var          server       = new Server(failures, callDuration, scheduler, failureCode);
            // We're not really interested in the timing in this test.
            var retrySettings = ConstantBackoff(5, TimeSpan.Zero, RetrySettings.FilterForStatusCodes(filterCodes));

            await scheduler.RunAsync(async() =>
            {
                var callSettings = CallSettings.FromRetry(retrySettings);
                var request      = new SimpleRequest {
                    Name = "irrelevant"
                };
                await Call(async, scheduler, server, request, callSettings);
            });

            Assert.True(server.CallTimes.Count() > 1);
        }
        private FirestoreDb(string projectId, string databaseId, FirestoreClient client, Action <string> warningLogger, SerializationContext serializationContext)
        {
            ProjectId  = GaxPreconditions.CheckNotNull(projectId, nameof(projectId));
            DatabaseId = GaxPreconditions.CheckNotNull(databaseId, nameof(databaseId));
            Client     = GaxPreconditions.CheckNotNull(client, nameof(client));
            // TODO: Investigate using DatabaseName and DocumentPathName.
            RootPath      = $"projects/{ProjectId}/databases/{DatabaseId}";
            DocumentsPath = $"{RootPath}/documents";
            WarningLogger = warningLogger;

            // TODO: Potentially make these configurable.
            // The retry settings are taken from firestore_grpc_service_config.json.
            var batchGetRetry = RetrySettings.FromExponentialBackoff(
                maxAttempts: 5,
                initialBackoff: TimeSpan.FromMilliseconds(100),
                maxBackoff: TimeSpan.FromSeconds(60),
                backoffMultiplier: 1.3,
                retryFilter: RetrySettings.FilterForStatusCodes(StatusCode.Unavailable, StatusCode.Internal, StatusCode.DeadlineExceeded));

            _batchGetCallSettings = CallSettings.FromRetry(batchGetRetry).WithTimeout(TimeSpan.FromMinutes(10));

            SerializationContext = GaxPreconditions.CheckNotNull(serializationContext, nameof(serializationContext));
        }
Exemple #13
0
        public async Task MultipleCallsEventualSuccess(bool async)
        {
            var callDuration  = TimeSpan.FromTicks(300);
            var failures      = 4;      // Fifth call will succeed
            var name          = "name"; // Copied from request to response
            var scheduler     = new FakeScheduler();
            var time0         = scheduler.Clock.GetCurrentDateTimeUtc();
            var server        = new Server(failures, callDuration, scheduler);
            var retrySettings = new RetrySettings(
                maxAttempts: 5,
                initialBackoff: TimeSpan.FromTicks(1000),
                maxBackoff: TimeSpan.FromTicks(5000),
                backoffMultiplier: 2.0,
                retryFilter: NotFoundFilter,
                backoffJitter: RetrySettings.NoJitter);

            await scheduler.RunAsync(async() =>
            {
                var callSettings = CallSettings.FromRetry(retrySettings);
                var request      = new SimpleRequest {
                    Name = name
                };
                var result = await Call(async, scheduler, server, request, callSettings);
                Assert.Equal(name, result.Name);
            });

            var firstCall  = time0;
            var secondCall = firstCall + callDuration + TimeSpan.FromTicks(1000);  // Delay for 1000 ticks
            var thirdCall  = secondCall + callDuration + TimeSpan.FromTicks(2000); // Delay for 2000 ticks
            var fourthCall = thirdCall + callDuration + TimeSpan.FromTicks(4000);  // Delay for 4000 ticks
            var fifthCall  = fourthCall + callDuration + TimeSpan.FromTicks(5000); // Delay for 5000 ticks, as that's the max

            server.AssertCallTimes(firstCall, secondCall, thirdCall, fourthCall, fifthCall);
            // Time of last action was when the call returned
            Assert.Equal(fifthCall + callDuration, scheduler.Clock.GetCurrentDateTimeUtc());
        }
        private async Task RunBulkMessagingImpl(
            TopicName topicName, SubscriptionName subscriptionName,
            int messageCount, int minMessageSize, int maxMessageSize, int maxMessagesInFlight, int initialNackCount,
            TimeSpan?timeouts          = null, int?cancelAfterRecvCount  = null, TimeSpan?interPublishDelay = null,
            TimeSpan?debugOutputPeriod = null, int?publisherChannelCount = null, int?clientCount            = null)
        {
            // Force messages to be at least 4 bytes long, so an int ID can be used.
            minMessageSize = Math.Max(4, minMessageSize);

            // Create PublisherClient and SubscriberClient
            var publisher = await PublisherClient.CreateAsync(topicName,
                                                              clientCreationSettings : new PublisherClient.ClientCreationSettings(
                                                                  clientCount: publisherChannelCount,
                                                                  publisherServiceApiSettings: timeouts == null ? null : new PublisherServiceApiSettings
            {
                PublishSettings = CallSettings
                                  .FromRetry(RetrySettings.FromExponentialBackoff(
                                                 maxAttempts: int.MaxValue,
                                                 initialBackoff: TimeSpan.FromMilliseconds(100),
                                                 maxBackoff: TimeSpan.FromSeconds(6),
                                                 backoffMultiplier: 1.3,
                                                 retryFilter: RetrySettings.FilterForStatusCodes(StatusCode.Unavailable)))
                                  .WithTimeout(timeouts.Value)
            }
                                                                  )).ConfigureAwait(false);

            var subscriber = await SubscriberClient.CreateAsync(subscriptionName,
                                                                clientCreationSettings : new SubscriberClient.ClientCreationSettings(clientCount: clientCount),
                                                                settings : new SubscriberClient.Settings
            {
                AckDeadline         = timeouts,
                FlowControlSettings = new FlowControlSettings(maxMessagesInFlight, null)
            }).ConfigureAwait(false);

            Console.WriteLine("Topic, Subscription, Publisher and Subscriber all created");

            // Subscribe
            int  recvCount = 0;  // Count of received messages
            int  dupCount  = 0;  // Count of duplicate messages
            long recvSum   = 0L; // Sum of bytes of received messages
            var  recvedIds = new ConcurrentDictionary <int, bool>();
            var  nackedIds = new HashSet <int>();
            Task subTask   = subscriber.StartAsync((msg, ct) =>
            {
                int id = BitConverter.ToInt32(msg.Data.ToArray(), 0);
                lock (nackedIds)
                {
                    if (nackedIds.Count < initialNackCount)
                    {
                        if (nackedIds.Add(id))
                        {
                            // This ID not already nacked
                            Interlocked.Increment(ref recvCount);
                            return(Task.FromResult(SubscriberClient.Reply.Nack));
                        }
                    }
                }
                bool wasAdded = recvedIds.TryAdd(id, false);
                if (wasAdded)
                {
                    var localRecvCount = Interlocked.Increment(ref recvCount);
                    Interlocked.Add(ref recvSum, msg.Data.Sum(x => (long)x));
                    if (localRecvCount == cancelAfterRecvCount || localRecvCount >= messageCount + initialNackCount)
                    {
                        // Test finished, so stop subscriber
                        Console.WriteLine("All msgs received, stopping subscriber.");
                        Task unused = subscriber.StopAsync(TimeSpan.FromSeconds(15));
                    }
                }
                else
                {
                    Interlocked.Add(ref dupCount, 1);
                }
                // ACK all messages
                return(Task.FromResult(SubscriberClient.Reply.Ack));
            });

            // Publish
            var  rnd        = new Random(1234);
            var  activePubs = new HashSet <Task>();
            int  sentCount  = 0;
            long sentSum    = 0L; // Sum of bytes of sent messages

            // Watchdog to report progress and fail test on deadlock
            CancellationTokenSource watchdogCts = new CancellationTokenSource();

            Task.Run(async() =>
            {
                var debugOutputPeriod1 = debugOutputPeriod ?? TimeSpan.FromSeconds(1);
                int prevSentCount      = -1;
                int prevRecvCount      = -1;
                int noProgressCount    = 0;
                while (true)
                {
                    await Task.Delay(debugOutputPeriod1, watchdogCts.Token).ConfigureAwait(false);
                    var localSentCount = Interlocked.Add(ref sentCount, 0);
                    var localRecvCount = Interlocked.Add(ref recvCount, 0);
                    var localDupCount  = Interlocked.Add(ref dupCount, 0);
                    if (prevSentCount == localSentCount && prevRecvCount == localRecvCount)
                    {
                        if (noProgressCount > 60)
                        {
                            // Deadlock, shutdown subscriber, and cancel
                            Console.WriteLine("Deadlock detected. Cancelling test");
                            subscriber.StopAsync(new CancellationToken(true));
                            watchdogCts.Cancel();
                            break;
                        }
                        noProgressCount += 1;
                    }
                    else
                    {
                        noProgressCount = 0;
                    }
                    prevSentCount = localSentCount;
                    prevRecvCount = localRecvCount;
                    Console.WriteLine($"[{DateTime.Now}] Sent: {localSentCount} (in-flight: {activePubs.Locked(() => activePubs.Count)}); Recv: {localRecvCount} (dups: {localDupCount})");
                }
            });

            for (int i = 0; i < messageCount; i++)
            {
                if (watchdogCts.IsCancellationRequested)
                {
                    Assert.True(false, "Test cancelled by watchdog");
                }
                if (subTask.IsCompleted)
                {
                    break;
                }
                if (i > 0 && interPublishDelay is TimeSpan delay)
                {
                    await Task.Delay(delay, watchdogCts.Token).ConfigureAwait(false);
                }
                var msgSize = rnd.Next(minMessageSize, maxMessageSize + 1);
                var msg     = new byte[msgSize];
                rnd.NextBytes(msg);
                // Insert an int ID into message
                Array.Copy(BitConverter.GetBytes(i), msg, 4);
                sentSum += msg.Sum(x => (long)x);
                // Send message, and record Task
                var pubTask = publisher.PublishAsync(msg);
                Interlocked.Increment(ref sentCount);
                activePubs.Locked(() => activePubs.Add(pubTask));
                // Remove Task from active when the message has been sent to server
                pubTask.ContinueWith(t => activePubs.Locked(() => activePubs.Remove(pubTask)));
                // If too many messages are currently in flight, wait a bit
                while (activePubs.Locked(() => activePubs.Count) >= maxMessagesInFlight)
                {
                    await Task.Delay(TimeSpan.FromMilliseconds(1)).ConfigureAwait(false);
                }
            }
            Console.WriteLine("Publishing complete");
            // Wait for all messages to be sent to server
            await Task.WhenAll(activePubs.Locked(() => activePubs.ToArray())).ConfigureAwait(false);

            Console.WriteLine("Publishing completed sending to server");

            // Wait for subscriber to finish shutdown
            await subTask.ConfigureAwait(false);

            watchdogCts.Cancel();
            Console.WriteLine("Subscriber finished shutdown");
            Console.WriteLine($"Sent: {sentCount}; Recv: {recvCount}");

            if (cancelAfterRecvCount is int cancelAfter)
            {
                Assert.True(recvCount >= cancelAfter && recvCount <= cancelAfter + maxMessagesInFlight, $"Incorrect recvCount: {recvCount}");
            }
            else
            {
                // Check that all messages are correctly received.
                Assert.Equal(messageCount + initialNackCount, recvCount);
                // This isn't foolproof (we can get to the right sum with wrong values) but it's a pretty strong indicator.
                Assert.Equal(sentSum, recvSum);
            }
            Console.WriteLine("Test complete.");
        }