private async Task RunBulkMessaging( int messageCount, int minMessageSize, int maxMessageSize, int maxMessagesInFlight, int initialNackCount, TimeSpan?timeouts = null, int?cancelAfterRecvCount = null, TimeSpan?interPublishDelay = null, TimeSpan?debugOutputPeriod = null, int?publisherChannelCount = null, int?clientCount = null) { // Force messages to be at least 4 bytes long, so an int ID can be used. minMessageSize = Math.Max(4, minMessageSize); var topicId = _fixture.CreateTopicId(); var subscriptionId = _fixture.CreateSubscriptionId(); Console.WriteLine("BulkMessaging test"); Console.WriteLine($"ProjectID: {_fixture.ProjectId}"); Console.WriteLine($"Topic ID: {topicId}"); Console.WriteLine($"Subscription ID: {subscriptionId}"); Console.WriteLine($"{messageCount} messages; of size [{minMessageSize}, {maxMessageSize}]; " + $"max in-flight: {maxMessagesInFlight}, initialNacks: {initialNackCount}, " + $"cancelAfterRecvCount: {cancelAfterRecvCount}, interPublishDelay: {interPublishDelay}, " + $"clientCount: {clientCount?.ToString() ?? $"UsingDefault:{Environment.ProcessorCount}"}"); var topicName = new TopicName(_fixture.ProjectId, topicId); var subscriptionName = new SubscriptionName(_fixture.ProjectId, subscriptionId); await CreateTopicAndSubscription(topicName, subscriptionName); await RunBulkMessagingImpl(topicName, subscriptionName, messageCount, minMessageSize, maxMessageSize, maxMessagesInFlight, initialNackCount, timeouts, cancelAfterRecvCount, interPublishDelay, debugOutputPeriod, publisherChannelCount, clientCount); }
private async Task RunBulkMessaging( int messageCount, int minMessageSize, int maxMessageSize, int maxMessagesInFlight, int initialNackCount, TimeSpan?timeouts = null, int?cancelAfterRecvCount = null) { // Force messages to be at least 4 bytes long, so an int ID can be used. minMessageSize = Math.Max(4, minMessageSize); var topicId = _fixture.CreateTopicId(); var subscriptionId = _fixture.CreateSubscriptionId(); Console.WriteLine("BulkMessaging test"); Console.WriteLine($"{messageCount} messages; of size [{minMessageSize}, {maxMessageSize}]; " + $"max in-flight: {maxMessagesInFlight}, initialNacks: {initialNackCount}, cancelAfterRecvCount: {cancelAfterRecvCount}"); // Create topic var topicName = new TopicName(_fixture.ProjectId, topicId); var publisher = await PublisherClient.CreateAsync().ConfigureAwait(false); await publisher.CreateTopicAsync(topicName).ConfigureAwait(false); // Subscribe to the topic var subscriber = await SubscriberClient.CreateAsync().ConfigureAwait(false); var subscriptionName = new SubscriptionName(_fixture.ProjectId, subscriptionId); await subscriber.CreateSubscriptionAsync(subscriptionName, topicName, null, 60).ConfigureAwait(false); // Create SimplePublisher and SimpleSubscriber var simplePublisher = await SimplePublisher.CreateAsync(topicName, clientCreationSettings : timeouts == null?null : new SimplePublisher.ClientCreationSettings( publisherSettings: new PublisherSettings { PublishSettings = CallSettings.FromCallTiming(CallTiming.FromRetry(new RetrySettings( retryBackoff: PublisherSettings.GetMessagingRetryBackoff(), timeoutBackoff: new BackoffSettings(timeouts.Value, timeouts.Value, 1.0), totalExpiration: Expiration.FromTimeout(timeouts.Value), retryFilter: PublisherSettings.NonIdempotentRetryFilter ))) } )).ConfigureAwait(false); var simpleSubscriber = await SimpleSubscriber.CreateAsync(subscriptionName, settings : new SimpleSubscriber.Settings { StreamAckDeadline = timeouts, FlowControlSettings = new FlowControlSettings(maxMessagesInFlight, null) }).ConfigureAwait(false); Console.WriteLine("Topic, Subscription, SimplePublisher and SimpleSubscriber all created"); // Subscribe object recvLock = new object(); int recvCount = 0; // Count of received messages int dupCount = 0; // Count of duplicate messages long recvSum = 0L; // Sum of bytes of received messages var recvedIds = new ConcurrentDictionary <int, bool>(); var nackedIds = new HashSet <int>(); Task subTask = simpleSubscriber.StartAsync((msg, ct) => { int id = BitConverter.ToInt32(msg.Data.ToArray(), 0); lock (nackedIds) { if (nackedIds.Count < initialNackCount) { if (nackedIds.Add(id)) { // This ID not already nacked Interlocked.Increment(ref recvCount); return(Task.FromResult(SimpleSubscriber.Reply.Nack)); } } } bool wasAdded = recvedIds.TryAdd(id, false); if (wasAdded) { var localRecvCount = Interlocked.Increment(ref recvCount); Interlocked.Add(ref recvSum, msg.Data.Sum(x => (long)x)); if (localRecvCount == cancelAfterRecvCount || localRecvCount >= messageCount + initialNackCount) { // Test finished, so stop subscriber Task unused = simpleSubscriber.StopAsync(TimeSpan.FromSeconds(15)); } } else { Interlocked.Add(ref dupCount, 1); } // ACK all messages return(Task.FromResult(SimpleSubscriber.Reply.Ack)); }); // Publish var rnd = new Random(1234); var activePubs = new HashSet <Task>(); int sentCount = 0; long sentSum = 0L; // Sum of bytes of sent messages // Watchdog to report progress and fail test on deadlock CancellationTokenSource watchdogCts = new CancellationTokenSource(); Task.Run(async() => { int prevSentCount = -1; int prevRecvCount = -1; int noProgressCount = 0; while (!watchdogCts.IsCancellationRequested) { await Task.Delay(TimeSpan.FromSeconds(1), watchdogCts.Token).ConfigureAwait(false); var localSentCount = Interlocked.Add(ref sentCount, 0); var localRecvCount = Interlocked.Add(ref recvCount, 0); var localDupCount = Interlocked.Add(ref dupCount, 0); if (prevSentCount == localSentCount && prevRecvCount == localRecvCount) { if (noProgressCount > 100) { // Deadlock, shutdown subscriber, and cancel Console.WriteLine("Deadlock detected. Cancelling test"); simpleSubscriber.StopAsync(new CancellationToken(true)); watchdogCts.Cancel(); break; } noProgressCount += 1; } else { noProgressCount = 0; } prevSentCount = localSentCount; prevRecvCount = localRecvCount; Console.WriteLine($"Sent: {localSentCount} (in-flight: {activePubs.Locked(() => activePubs.Count)}); Recv: {localRecvCount} (dups: {localDupCount})"); } }); for (int i = 0; i < messageCount; i++) { if (watchdogCts.IsCancellationRequested) { Assert.True(false, "Test cancelled by watchdog"); } if (subTask.IsCompleted) { break; } var msgSize = rnd.Next(minMessageSize, maxMessageSize + 1); var msg = new byte[msgSize]; rnd.NextBytes(msg); // Insert an int ID into message Array.Copy(BitConverter.GetBytes(i), msg, 4); sentSum += msg.Sum(x => (long)x); // Send message, and record Task var pubTask = simplePublisher.PublishAsync(msg); Interlocked.Increment(ref sentCount); activePubs.Locked(() => activePubs.Add(pubTask)); // Remove Task from active when the message has been sent to server pubTask.ContinueWith(t => activePubs.Locked(() => activePubs.Remove(pubTask))); // If too many messages are currently in flight, wait a bit while (activePubs.Locked(() => activePubs.Count) >= maxMessagesInFlight) { await Task.Delay(TimeSpan.FromMilliseconds(1)).ConfigureAwait(false); } } Console.WriteLine("Publishing complete"); // Wait for all messages to be sent to server await Task.WhenAll(activePubs.Locked(() => activePubs.ToArray())).ConfigureAwait(false); Console.WriteLine("Publishing completed sending to server"); // Wait for subscriber to finish shutdown await subTask.ConfigureAwait(false); watchdogCts.Cancel(); Console.WriteLine("Subscriber finished shutdown"); Console.WriteLine($"Sent: {sentCount}; Recv: {recvCount}"); if (cancelAfterRecvCount is int cancelAfter) { Assert.True(recvCount >= cancelAfter && recvCount <= cancelAfter + maxMessagesInFlight, $"Incorrect recvCount: {recvCount}"); } else { // Check that all messages are correctly received. Assert.Equal(messageCount + initialNackCount, recvCount); // This isn't foolproof (we can get to the right sum with wrong values) but it's a pretty strong indicator. Assert.Equal(sentSum, recvSum); } }