public async Task MultipleCallsEventualSuccess(bool async, bool serverStreaming)
{
var callDuration = TimeSpan.FromTicks(300);
var failures = 4; // Fifth call will succeed
var name = "name"; // Copied from request to response
var scheduler = new FakeScheduler();
var time0 = scheduler.Clock.GetCurrentDateTimeUtc();
var server = new Server(failures, callDuration, scheduler);
var retrySettings = new RetrySettings(
retryBackoff: DoublingBackoff,
timeoutBackoff: ConstantBackoff,
totalExpiration: Expiration.FromTimeout(TimeSpan.FromSeconds(1)),
retryFilter: null,
delayJitter: RetrySettings.NoJitter);
await scheduler.RunAsync(async() =>
{
var callSettings = CallSettings.FromCallTiming(CallTiming.FromRetry(retrySettings));
var request = new SimpleRequest {
Name = name
};
var result = await Call(async, serverStreaming, scheduler, server, request, callSettings);
Assert.Equal(name, result.Name);
});
var firstCall = time0;
var secondCall = firstCall + callDuration + TimeSpan.FromTicks(1000); // Delay for 1000 ticks
var thirdCall = secondCall + callDuration + TimeSpan.FromTicks(2000); // Delay for 2000 ticks
var fourthCall = thirdCall + callDuration + TimeSpan.FromTicks(4000); // Delay for 4000 ticks
var fifthCall = fourthCall + callDuration + TimeSpan.FromTicks(5000); // Delay for 5000 ticks, as that's the max
server.AssertCallTimes(firstCall, secondCall, thirdCall, fourthCall, fifthCall);
// Time of last action was when the call returned
Assert.Equal(fifthCall + callDuration, scheduler.Clock.GetCurrentDateTimeUtc());
}
public async Task RetryFilter_EventualSuccess(bool async, StatusCode failureCode, StatusCode[] filterCodes)
{
var callDuration = 100;
var failures = 1;
var scheduler = new FakeScheduler();
var server = new Server(failures, callDuration, scheduler, failureCode);
// We're not really interested in the timing in this test.
var retrySettings = new RetrySettings(
retryBackoff: ConstantBackoff,
timeoutBackoff: ConstantBackoff,
delayJitter: RetrySettings.NoJitter,
totalExpiration: Expiration.FromTimeout(TimeSpan.FromSeconds(1)),
retryFilter: RetrySettings.FilterForStatusCodes(filterCodes));
await scheduler.RunAsync(async() =>
{
var callSettings = CallSettings.FromCallTiming(CallTiming.FromRetry(retrySettings));
var retryingCallable = server.Callable.WithRetry(scheduler.Clock, scheduler);
await Call(async, retryingCallable, new SimpleRequest {
Name = "irrelevant"
}, callSettings);
});
Assert.True(server.CallTimes.Count() > 1);
}
public async Task ExponentialTimeouts(bool async, bool serverStreaming)
{
var callDuration = TimeSpan.FromTicks(300);
var failures = 2;
var scheduler = new FakeScheduler();
var time0 = scheduler.Clock.GetCurrentDateTimeUtc();
var server = new Server(failures, callDuration, scheduler);
var callable = server.Callable;
var retrySettings = new RetrySettings(
retryBackoff: ConstantBackoff, // 1500 ticks always
timeoutBackoff: DoublingBackoff, // 1000, then 2000, then 4000
totalExpiration: Expiration.FromTimeout(TimeSpan.FromTicks(4500)),
retryFilter: null,
delayJitter: RetrySettings.NoJitter);
await scheduler.RunAsync(async() =>
{
// Expiration truncates the third timeout. We expect:
// Call 1: t=0, deadline=1000, completes at 300
// Call 2: t=1800, deadline=3800 (2000+1800), completes at 2100
// Call 3, t=3600, deadline=4500 (would be 7600, but overall deadline truncates), completes at 3900 (with success)
var callSettings = CallSettings.FromCallTiming(CallTiming.FromRetry(retrySettings));
var request = new SimpleRequest {
Name = "irrelevant"
};
await Call(async, serverStreaming, scheduler, server, request, callSettings);
});
server.AssertCallTimes(time0, time0 + TimeSpan.FromTicks(1800), time0 + TimeSpan.FromTicks(3600));
server.AssertDeadlines(time0 + TimeSpan.FromTicks(1000), time0 + TimeSpan.FromTicks(3800), time0 + TimeSpan.FromTicks(4500));
Assert.Equal(3900L, scheduler.Clock.GetCurrentDateTimeUtc().Ticks);
}
public async Task RetryFilter_EventualSuccess(bool async, bool serverStreaming)
{
StatusCode failureCode = StatusCode.NotFound;
StatusCode[] filterCodes = new[] { StatusCode.NotFound, StatusCode.DeadlineExceeded };
var callDuration = TimeSpan.FromTicks(100);
var failures = 1;
var scheduler = new FakeScheduler();
var server = new Server(failures, callDuration, scheduler, failureCode);
// We're not really interested in the timing in this test.
var retrySettings = new RetrySettings(
retryBackoff: ConstantBackoff,
timeoutBackoff: ConstantBackoff,
delayJitter: RetrySettings.NoJitter,
totalExpiration: Expiration.FromTimeout(TimeSpan.FromSeconds(1)),
retryFilter: RetrySettings.FilterForStatusCodes(filterCodes));
await scheduler.RunAsync(async() =>
{
var callSettings = CallSettings.FromCallTiming(CallTiming.FromRetry(retrySettings));
var request = new SimpleRequest {
Name = "irrelevant"
};
await Call(async, serverStreaming, scheduler, server, request, callSettings);
});
Assert.True(server.CallTimes.Count() > 1);
}
/// <summary>
/// Creates a service context that binds the service, callsettings and the client.
/// </summary>
/// <param name="config">The configuration.</param>
/// <returns>The service context.</returns>
private GoogleAdsServiceContext CreateServiceContext(GoogleAdsConfig config)
{
GoogleAdsServiceContext serviceContext = new GoogleAdsServiceContext();
CallSettings callSettings = CallSettings.FromCallTiming(
CallTiming.FromRetry(new RetrySettings(
retryBackoff : backoffSettings,
timeoutBackoff : backoffSettings,
totalExpiration : Expiration.FromTimeout(TimeSpan.FromMilliseconds(
config.Timeout)),
retryFilter : retryFilter
)))
.WithHeader(GoogleAdsConfig.DEVELOPER_TOKEN_KEYNAME, config.DeveloperToken)
.WithResponseMetadataHandler(delegate(Metadata metadata)
{
GoogleAdsResponseMetadata responseMetadata = new GoogleAdsResponseMetadata(metadata);
serviceContext.OnResponseMetadataReceived(responseMetadata);
});
if (!string.IsNullOrEmpty(config.LoginCustomerId))
{
callSettings.WithHeader("login-customer-id", config.LoginCustomerId);
}
serviceContext.CallSettings = callSettings;
return(serviceContext);
}
public async Task CallSettingsDeadlineIsObserved(bool async, bool serverStreaming)
{
var callDuration = TimeSpan.FromTicks(300);
var failures = 4; // Fifth call would succeed, but we won't get that far.
var scheduler = new FakeScheduler();
var time0 = scheduler.Clock.GetCurrentDateTimeUtc();
var server = new Server(failures, callDuration, scheduler);
var callable = server.Callable;
var retrySettings = new RetrySettings(
retryBackoff: DoublingBackoff,
timeoutBackoff: ConstantBackoff,
totalExpiration: Expiration.FromTimeout(TimeSpan.FromTicks(2500)),
retryFilter: null,
delayJitter: RetrySettings.NoJitter);
var task = scheduler.RunAsync(async() =>
{
// Expiration makes it fail while waiting to make third call
var callSettings = CallSettings.FromCallTiming(CallTiming.FromRetry(retrySettings));
var request = new SimpleRequest {
Name = "irrelevant"
};
await Call(async, serverStreaming, scheduler, server, request, callSettings);
});
await Assert.ThrowsAsync <RpcException>(() => task);
var firstCall = time0;
var secondCall = firstCall + callDuration + TimeSpan.FromTicks(1000);
server.AssertCallTimes(firstCall, secondCall);
// We fail immediately when we work out that we would time out before we make the third
// call - so this is before the actual total timeout.
Assert.Equal((secondCall + callDuration).Ticks, scheduler.Clock.GetCurrentDateTimeUtc().Ticks);
}
public async Task RetryCancellation(bool serverStreaming, [CombinatorialValues(1500, 3500)] int delayMs)
{
// Note: Cannot test cancellation during wait for response header, due to FakeScheduler shortcomings.
var async = true;
var scheduler = new FakeScheduler();
var time0 = scheduler.Clock.GetCurrentDateTimeUtc();
var server = new Server(10, TimeSpan.FromSeconds(1), scheduler);
var retrySettings = new RetrySettings(
retryBackoff: new BackoffSettings(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(1), 1.0),
timeoutBackoff: new BackoffSettings(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(1), 1.0),
delayJitter: RetrySettings.NoJitter,
totalExpiration: Expiration.FromTimeout(TimeSpan.FromSeconds(10)),
retryFilter: RetrySettings.DefaultFilter);
var delay = TimeSpan.FromMilliseconds(delayMs);
Task task = scheduler.RunAsync(async() =>
{
var cts = new CancellationTokenSource();
var unused = Task.Run(async() =>
{
await scheduler.Delay(delay);
cts.Cancel();
});
var callSettings = CallSettings.FromCallTiming(CallTiming.FromRetry(retrySettings)).WithCancellationToken(cts.Token);
var request = new SimpleRequest {
Name = "irrelevant"
};
await Call(async, serverStreaming, scheduler, server, request, callSettings);
});
await Assert.ThrowsAsync <TaskCanceledException>(() => task);
Assert.Equal(time0 + delay, scheduler.Clock.GetCurrentDateTimeUtc());
}
public async Task FirstCallSucceeds(bool async, bool serverStreaming)
{
var name = "name"; // Copied from request to response
var scheduler = new FakeScheduler();
var time0 = scheduler.Clock.GetCurrentDateTimeUtc();
var server = new Server(0, TimeSpan.FromTicks(300), scheduler);
var retrySettings = new RetrySettings(
retryBackoff: DoublingBackoff,
timeoutBackoff: ConstantBackoff,
totalExpiration: Expiration.FromTimeout(TimeSpan.FromSeconds(1)),
retryFilter: null,
delayJitter: RetrySettings.NoJitter);
await scheduler.RunAsync(async() =>
{
var callSettings = CallSettings.FromCallTiming(CallTiming.FromRetry(retrySettings));
var request = new SimpleRequest {
Name = name
};
var result = await Call(async, serverStreaming, scheduler, server, request, callSettings);
Assert.Equal(name, result.Name);
});
server.AssertCallTimes(time0);
// Time of last action was when the call returned
Assert.Equal(300, scheduler.Clock.GetCurrentDateTimeUtc().Ticks);
}
public async Task RetryFilter_EventualFailure(bool async, bool serverStreaming, StatusCode failureCode, StatusCode[] filterCodes)
{
var callDuration = TimeSpan.FromTicks(100);
var failures = 1;
var scheduler = new FakeScheduler();
var server = new Server(failures, callDuration, scheduler, failureCode);
// We're not really interested in the timing in this test.
var retrySettings = new RetrySettings(
retryBackoff: ConstantBackoff,
timeoutBackoff: ConstantBackoff,
delayJitter: RetrySettings.NoJitter,
totalExpiration: Expiration.FromTimeout(TimeSpan.FromSeconds(1)),
retryFilter: RetrySettings.FilterForStatusCodes(filterCodes));
var task = scheduler.RunAsync(async() =>
{
var callSettings = CallSettings.FromCallTiming(CallTiming.FromRetry(retrySettings));
var request = new SimpleRequest {
Name = "irrelevant"
};
await Call(async, serverStreaming, scheduler, server, request, callSettings);
});
await Assert.ThrowsAsync <RpcException>(() => task);
Assert.Equal(1, server.CallTimes.Count());
}
public void FailWithRetry()
{
var apiCall = ApiServerStreamingCall.Create <int, int>(
(request, callOptions) => null,
CallSettings.FromCallTiming(CallTiming.FromRetry(new RetrySettings(
new BackoffSettings(TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(100), 2.0),
new BackoffSettings(TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(100), 2.0),
Expiration.FromTimeout(TimeSpan.FromSeconds(100))))),
new FakeClock());
Assert.Throws <InvalidOperationException>(() => apiCall.Call(0, null));
}
public void WithEarlierDeadline_DeadlineIsLaterThanExistingRetryTotalExpiration()
{
var backoffSettings = new BackoffSettings(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(2), 2.0);
// Use a cancellation token to emphasize that it's not just the timing.
var token = new CancellationTokenSource().Token;
var timing = CallTiming.FromRetry(new RetrySettings(backoffSettings, backoffSettings, Expiration.FromDeadline(new DateTime(100L, DateTimeKind.Utc))));
CallSettings settings = CallSettings.FromCancellationToken(token)
.WithCallTiming(timing);
DateTime?deadline = new DateTime(200L, DateTimeKind.Utc);
Assert.Same(settings, settings.WithEarlierDeadline(deadline, new FakeClock()));
}
public StackDriverLogger(string projectId)
{
_projectId = projectId;
_retryAWhile = CallSettings.FromCallTiming(CallTiming.FromRetry(new RetrySettings(
new BackoffSettings(TimeSpan.FromSeconds(3), TimeSpan.FromSeconds(12), 2.0),
new BackoffSettings(TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(120)),
Google.Api.Gax.Expiration.FromTimeout(TimeSpan.FromSeconds(180)),
(Grpc.Core.RpcException e) =>
new[] { Grpc.Core.StatusCode.Internal, Grpc.Core.StatusCode.DeadlineExceeded }
.Contains(e.Status.StatusCode)
)));
}
/// <inheritdoc />
public async Task <SpannerClient> CreateClientAsync(ServiceEndpoint endpoint, ITokenAccess credential, IDictionary additionalOptions)
{
ChannelCredentials channelCredentials;
var allowImmediateTimeout = false;
if (additionalOptions.Contains(nameof(SpannerSettings.AllowImmediateTimeouts)))
{
allowImmediateTimeout = Convert.ToBoolean(additionalOptions[nameof(SpannerSettings.AllowImmediateTimeouts)]);
}
if (credential == null)
{
channelCredentials = await CreateDefaultChannelCredentialsAsync().ConfigureAwait(false);
}
else
{
channelCredentials = credential.ToChannelCredentials();
}
var channel = new Channel(
endpoint.Host,
endpoint.Port,
channelCredentials);
Logger.LogPerformanceCounterFn("SpannerClient.RawCreateCount", x => x + 1);
//Pull the timeout from spanner options.
//The option must be set before OpenAsync is called.
var idempotentCallSettings = CallSettings.FromCallTiming(
CallTiming.FromRetry(
new RetrySettings(
SpannerSettings.GetDefaultRetryBackoff(),
SpannerSettings.GetDefaultTimeoutBackoff(),
SpannerSettings.ConvertTimeoutToExpiration(SpannerOptions.Instance.Timeout, allowImmediateTimeout),
SpannerSettings.IdempotentRetryFilter
)));
return(SpannerClient.Create(
channel, new SpannerSettings
{
CreateSessionSettings = idempotentCallSettings,
GetSessionSettings = idempotentCallSettings,
DeleteSessionSettings = idempotentCallSettings,
ExecuteSqlSettings = idempotentCallSettings,
ReadSettings = idempotentCallSettings,
BeginTransactionSettings = idempotentCallSettings,
CommitSettings = idempotentCallSettings,
RollbackSettings = idempotentCallSettings,
AllowImmediateTimeouts = allowImmediateTimeout
}));
}
public void WithExpiration_SettingsWithRetry()
{
var token = new CancellationTokenSource().Token;
var backoffSettings = new BackoffSettings(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10), 1.5);
var retry = new RetrySettings(backoffSettings, backoffSettings, Expiration.FromTimeout(TimeSpan.FromSeconds(5)));
var originalTiming = CallTiming.FromRetry(retry);
CallSettings settings = CallSettings.FromCancellationToken(token).WithCallTiming(originalTiming);
Expiration expiration = Expiration.FromTimeout(TimeSpan.FromSeconds(1));
var result = settings.WithExpiration(expiration);
Assert.Same(expiration, result.Timing.Retry.TotalExpiration);
Assert.Same(backoffSettings, result.Timing.Retry.RetryBackoff);
Assert.Same(backoffSettings, result.Timing.Retry.TimeoutBackoff);
Assert.Equal(token, result.CancellationToken);
}
partial void OnConstruction()
{
var originalMutateRowsSettings = MutateRowsSettings;
GaxPreconditions.CheckState(
originalMutateRowsSettings.Timing != null &&
originalMutateRowsSettings.Timing.Type == CallTimingType.Expiration,
"The default MutateRowsSettings are not in the expected state");
MutateRowsSettings = originalMutateRowsSettings.WithCallTiming(
CallTiming.FromRetry(new RetrySettings(
retryBackoff: GetDefaultRetryBackoff(),
timeoutBackoff: GetDefaultTimeoutBackoff(),
totalExpiration: originalMutateRowsSettings.Timing.Expiration,
retryFilter: IdempotentRetryFilter
)));
}
private FirestoreDb(string projectId, string databaseId, FirestoreClient client, Action <string> warningLogger)
{
ProjectId = GaxPreconditions.CheckNotNull(projectId, nameof(projectId));
DatabaseId = GaxPreconditions.CheckNotNull(databaseId, nameof(databaseId));
Client = GaxPreconditions.CheckNotNull(client, nameof(client));
// TODO: Investigate using DatabaseName and DocumentPathName.
RootPath = $"projects/{ProjectId}/databases/{DatabaseId}";
DocumentsPath = $"{RootPath}/documents";
WarningLogger = warningLogger;
// TODO: Validate these settings, and potentially make them configurable
_batchGetCallSettings = CallSettings.FromCallTiming(CallTiming.FromRetry(new RetrySettings(
retryBackoff: new BackoffSettings(TimeSpan.FromMilliseconds(500), TimeSpan.FromSeconds(5), 2.0),
timeoutBackoff: new BackoffSettings(TimeSpan.FromMinutes(1), TimeSpan.FromMinutes(3), 2.0),
Expiration.FromTimeout(TimeSpan.FromMinutes(10)),
RetrySettings.FilterForStatusCodes(StatusCode.Unavailable))));
}
// [START retry]
/// <summary>
/// Creates new CallSettings that will retry an RPC that fails.
/// </summary>
/// <param name="tryCount">
/// How many times to try the RPC before giving up?
/// </param>
/// <param name="finalStatusCodes">
/// Which status codes should we *not* retry?
/// </param>
/// <returns>
/// A CallSettings instance.
/// </returns>
CallSettings newRetryCallSettings(int tryCount,
params StatusCode[] finalStatusCodes)
{
// Initialize values for backoff settings to be used
// by the CallSettings for RPC retries
var backoff = new BackoffSettings(
delay: TimeSpan.FromMilliseconds(500),
maxDelay: TimeSpan.FromSeconds(3), delayMultiplier: 2);
return(new CallSettings(null, null,
CallTiming.FromRetry(new RetrySettings(backoff, backoff,
Google.Api.Gax.Expiration.None,
(RpcException e) => (
StatusCode.OK != e.Status.StatusCode &&
!finalStatusCodes.Contains(e.Status.StatusCode) &&
--tryCount > 0),
RetrySettings.NoJitter)),
metadata => metadata.Add("ClientVersion", "1.0.0"), null, null));
}
// [START retry]
/// <summary>
/// Creates new CallSettings that will retry an RPC that fails.
/// </summary>
/// <param name="tryCount">
/// How many times to try the RPC before giving up?
/// </param>
/// <param name="finalStatusCodes">
/// Which status codes should we *not* retry?
/// </param>
/// <returns>
/// A CallSettings instance.
/// </returns>
CallSettings newRetryCallSettings(int tryCount,
params StatusCode[] finalStatusCodes)
{
var backoff = new BackoffSettings()
{
Delay = TimeSpan.FromMilliseconds(500),
DelayMultiplier = 2,
MaxDelay = TimeSpan.FromSeconds(3)
};
return(new CallSettings()
{
Timing = CallTiming.FromRetry(new RetrySettings()
{
RetryBackoff = backoff,
TimeoutBackoff = backoff,
RetryFilter = (RpcException e) => (
StatusCode.OK != e.Status.StatusCode &&
!finalStatusCodes.Contains(e.Status.StatusCode) &&
--tryCount > 0),
DelayJitter = RetrySettings.NoJitter,
})
});
}
private async Task RunBulkMessaging(
int messageCount, int minMessageSize, int maxMessageSize, int maxMessagesInFlight, int initialNackCount,
TimeSpan?timeouts = null, int?cancelAfterRecvCount = null)
{
// Force messages to be at least 4 bytes long, so an int ID can be used.
minMessageSize = Math.Max(4, minMessageSize);
var topicId = _fixture.CreateTopicId();
var subscriptionId = _fixture.CreateSubscriptionId();
Console.WriteLine("BulkMessaging test");
Console.WriteLine($"{messageCount} messages; of size [{minMessageSize}, {maxMessageSize}]; " +
$"max in-flight: {maxMessagesInFlight}, initialNacks: {initialNackCount}, cancelAfterRecvCount: {cancelAfterRecvCount}");
// Create topic
var topicName = new TopicName(_fixture.ProjectId, topicId);
var publisher = await PublisherClient.CreateAsync().ConfigureAwait(false);
await publisher.CreateTopicAsync(topicName).ConfigureAwait(false);
// Subscribe to the topic
var subscriber = await SubscriberClient.CreateAsync().ConfigureAwait(false);
var subscriptionName = new SubscriptionName(_fixture.ProjectId, subscriptionId);
await subscriber.CreateSubscriptionAsync(subscriptionName, topicName, null, 60).ConfigureAwait(false);
// Create SimplePublisher and SimpleSubscriber
var simplePublisher = await SimplePublisher.CreateAsync(topicName, clientCreationSettings : timeouts == null?null :
new SimplePublisher.ClientCreationSettings(
publisherSettings: new PublisherSettings
{
PublishSettings = CallSettings.FromCallTiming(CallTiming.FromRetry(new RetrySettings(
retryBackoff: PublisherSettings.GetMessagingRetryBackoff(),
timeoutBackoff: new BackoffSettings(timeouts.Value, timeouts.Value, 1.0),
totalExpiration: Expiration.FromTimeout(timeouts.Value),
retryFilter: PublisherSettings.NonIdempotentRetryFilter
)))
}
)).ConfigureAwait(false);
var simpleSubscriber = await SimpleSubscriber.CreateAsync(subscriptionName,
settings : new SimpleSubscriber.Settings
{
StreamAckDeadline = timeouts,
FlowControlSettings = new FlowControlSettings(maxMessagesInFlight, null)
}).ConfigureAwait(false);
Console.WriteLine("Topic, Subscription, SimplePublisher and SimpleSubscriber all created");
// Subscribe
object recvLock = new object();
int recvCount = 0; // Count of received messages
int dupCount = 0; // Count of duplicate messages
long recvSum = 0L; // Sum of bytes of received messages
var recvedIds = new ConcurrentDictionary <int, bool>();
var nackedIds = new HashSet <int>();
Task subTask = simpleSubscriber.StartAsync((msg, ct) =>
{
int id = BitConverter.ToInt32(msg.Data.ToArray(), 0);
lock (nackedIds)
{
if (nackedIds.Count < initialNackCount)
{
if (nackedIds.Add(id))
{
// This ID not already nacked
Interlocked.Increment(ref recvCount);
return(Task.FromResult(SimpleSubscriber.Reply.Nack));
}
}
}
bool wasAdded = recvedIds.TryAdd(id, false);
if (wasAdded)
{
var localRecvCount = Interlocked.Increment(ref recvCount);
Interlocked.Add(ref recvSum, msg.Data.Sum(x => (long)x));
if (localRecvCount == cancelAfterRecvCount || localRecvCount >= messageCount + initialNackCount)
{
// Test finished, so stop subscriber
Task unused = simpleSubscriber.StopAsync(TimeSpan.FromSeconds(15));
}
}
else
{
Interlocked.Add(ref dupCount, 1);
}
// ACK all messages
return(Task.FromResult(SimpleSubscriber.Reply.Ack));
});
// Publish
var rnd = new Random(1234);
var activePubs = new HashSet <Task>();
int sentCount = 0;
long sentSum = 0L; // Sum of bytes of sent messages
// Watchdog to report progress and fail test on deadlock
CancellationTokenSource watchdogCts = new CancellationTokenSource();
Task.Run(async() =>
{
int prevSentCount = -1;
int prevRecvCount = -1;
int noProgressCount = 0;
while (!watchdogCts.IsCancellationRequested)
{
await Task.Delay(TimeSpan.FromSeconds(1), watchdogCts.Token).ConfigureAwait(false);
var localSentCount = Interlocked.Add(ref sentCount, 0);
var localRecvCount = Interlocked.Add(ref recvCount, 0);
var localDupCount = Interlocked.Add(ref dupCount, 0);
if (prevSentCount == localSentCount && prevRecvCount == localRecvCount)
{
if (noProgressCount > 100)
{
// Deadlock, shutdown subscriber, and cancel
Console.WriteLine("Deadlock detected. Cancelling test");
simpleSubscriber.StopAsync(new CancellationToken(true));
watchdogCts.Cancel();
break;
}
noProgressCount += 1;
}
else
{
noProgressCount = 0;
}
prevSentCount = localSentCount;
prevRecvCount = localRecvCount;
Console.WriteLine($"Sent: {localSentCount} (in-flight: {activePubs.Locked(() => activePubs.Count)}); Recv: {localRecvCount} (dups: {localDupCount})");
}
});
for (int i = 0; i < messageCount; i++)
{
if (watchdogCts.IsCancellationRequested)
{
Assert.True(false, "Test cancelled by watchdog");
}
if (subTask.IsCompleted)
{
break;
}
var msgSize = rnd.Next(minMessageSize, maxMessageSize + 1);
var msg = new byte[msgSize];
rnd.NextBytes(msg);
// Insert an int ID into message
Array.Copy(BitConverter.GetBytes(i), msg, 4);
sentSum += msg.Sum(x => (long)x);
// Send message, and record Task
var pubTask = simplePublisher.PublishAsync(msg);
Interlocked.Increment(ref sentCount);
activePubs.Locked(() => activePubs.Add(pubTask));
// Remove Task from active when the message has been sent to server
pubTask.ContinueWith(t => activePubs.Locked(() => activePubs.Remove(pubTask)));
// If too many messages are currently in flight, wait a bit
while (activePubs.Locked(() => activePubs.Count) >= maxMessagesInFlight)
{
await Task.Delay(TimeSpan.FromMilliseconds(1)).ConfigureAwait(false);
}
}
Console.WriteLine("Publishing complete");
// Wait for all messages to be sent to server
await Task.WhenAll(activePubs.Locked(() => activePubs.ToArray())).ConfigureAwait(false);
Console.WriteLine("Publishing completed sending to server");
// Wait for subscriber to finish shutdown
await subTask.ConfigureAwait(false);
watchdogCts.Cancel();
Console.WriteLine("Subscriber finished shutdown");
Console.WriteLine($"Sent: {sentCount}; Recv: {recvCount}");
if (cancelAfterRecvCount is int cancelAfter)
{
Assert.True(recvCount >= cancelAfter && recvCount <= cancelAfter + maxMessagesInFlight, $"Incorrect recvCount: {recvCount}");
}
else
{
// Check that all messages are correctly received.
Assert.Equal(messageCount + initialNackCount, recvCount);
// This isn't foolproof (we can get to the right sum with wrong values) but it's a pretty strong indicator.
Assert.Equal(sentSum, recvSum);
}
}