public async Task RetryCancellation([CombinatorialValues(1500, 3500)] int delayMs)
{
// Note: Cannot test cancellation during wait for response header, due to FakeScheduler shortcomings.
var async = true;
var scheduler = new FakeScheduler();
var time0 = scheduler.Clock.GetCurrentDateTimeUtc();
var server = new Server(10, TimeSpan.FromSeconds(1), scheduler);
var retrySettings = ConstantBackoff(5, TimeSpan.FromSeconds(1), NotFoundFilter);
var delay = TimeSpan.FromMilliseconds(delayMs);
Task task = scheduler.RunAsync(async() =>
{
var cts = new CancellationTokenSource();
var unused = Task.Run(async() =>
{
await scheduler.Delay(delay);
cts.Cancel();
});
var callSettings = CallSettings.FromRetry(retrySettings).WithCancellationToken(cts.Token);
var request = new SimpleRequest {
Name = "irrelevant"
};
await Call(async, scheduler, server, request, callSettings);
});
await Assert.ThrowsAsync <TaskCanceledException>(() => task);
Assert.Equal(time0 + delay, scheduler.Clock.GetCurrentDateTimeUtc());
}
public async Task FirstCallSucceeds(bool async)
{
var name = "name"; // Copied from request to response
var scheduler = new FakeScheduler();
var time0 = scheduler.Clock.GetCurrentDateTimeUtc();
var server = new Server(0, TimeSpan.FromTicks(300), scheduler);
var retrySettings = new RetrySettings(
maxAttempts: 5,
initialBackoff: TimeSpan.FromSeconds(1),
maxBackoff: TimeSpan.FromSeconds(5),
backoffMultiplier: 2.0,
retryFilter: NotFoundFilter,
backoffJitter: RetrySettings.NoJitter);
await scheduler.RunAsync(async() =>
{
var callSettings = CallSettings.FromRetry(retrySettings);
var request = new SimpleRequest {
Name = name
};
var result = await Call(async, scheduler, server, request, callSettings);
Assert.Equal(name, result.Name);
});
server.AssertCallTimes(time0);
// Time of last action was when the call returned
Assert.Equal(300, scheduler.Clock.GetCurrentDateTimeUtc().Ticks);
}
public async Task PublishMessageWithRetrySettingsAsync(string projectId, string topicId, string messageText)
{
TopicName topicName = TopicName.FromProjectTopic(projectId, topicId);
// Retry settings control how the publisher handles retry-able failures
var maxAttempts = 3;
var initialBackoff = TimeSpan.FromMilliseconds(110); // default: 100 ms
var maxBackoff = TimeSpan.FromSeconds(70); // default : 60 seconds
var backoffMultiplier = 1.3; // default: 1.0
var totalTimeout = TimeSpan.FromSeconds(100); // default: 600 seconds
var publisher = await PublisherClient.CreateAsync(topicName,
clientCreationSettings : new PublisherClient.ClientCreationSettings(
publisherServiceApiSettings: new PublisherServiceApiSettings
{
PublishSettings = CallSettings.FromRetry(RetrySettings.FromExponentialBackoff(
maxAttempts: maxAttempts,
initialBackoff: initialBackoff,
maxBackoff: maxBackoff,
backoffMultiplier: backoffMultiplier,
retryFilter: RetrySettings.FilterForStatusCodes(StatusCode.Unavailable)))
.WithTimeout(totalTimeout)
}
)).ConfigureAwait(false);
string message = await publisher.PublishAsync(messageText);
Console.WriteLine($"Published message {message}");
}
public async Task RetryFilter_EventualFailure(bool async, StatusCode failureCode, StatusCode[] filterCodes)
{
var callDuration = TimeSpan.FromTicks(100);
var failures = 1;
var scheduler = new FakeScheduler();
var server = new Server(failures, callDuration, scheduler, failureCode);
// We're not really interested in the timing in this test.
var retrySettings = new RetrySettings(
maxAttempts: 5,
initialBackoff: TimeSpan.Zero,
maxBackoff: TimeSpan.Zero,
backoffMultiplier: 1.0,
retryFilter: RetrySettings.FilterForStatusCodes(filterCodes),
backoffJitter: RetrySettings.NoJitter);
var task = scheduler.RunAsync(async() =>
{
var callSettings = CallSettings.FromRetry(retrySettings);
var request = new SimpleRequest {
Name = "irrelevant"
};
await Call(async, scheduler, server, request, callSettings);
});
await Assert.ThrowsAsync <RpcException>(() => task);
Assert.Equal(1, server.CallTimes.Count());
}
public void FromRetry()
{
Assert.Null(CallSettings.FromRetry(null));
var retry = new RetrySettings(5, TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(5), 2.0, RetrySettings.FilterForStatusCodes(), RetrySettings.RandomJitter);
var settings = CallSettings.FromRetry(retry);
Assert.Same(retry, settings.Retry);
}
public void FailWithRetry()
{
var apiCall = ApiBidirectionalStreamingCall.Create <int, int>(
callOptions => null,
CallSettings.FromRetry(new RetrySettings(5, TimeSpan.Zero, TimeSpan.Zero, 1.0, e => false, RetrySettings.RandomJitter)),
new BidirectionalStreamingSettings(100),
new FakeClock());
Assert.Throws <InvalidOperationException>(() => apiCall.Call(null));
}
public async Task FailWithRetry()
{
var apiCall = ApiServerStreamingCall.Create <int, int>(
(request, callOptions) => null,
CallSettings.FromRetry(new RetrySettings(5, TimeSpan.Zero, TimeSpan.Zero, 1.0, e => false, RetrySettings.RandomJitter)),
new FakeClock());
await Assert.ThrowsAsync <InvalidOperationException>(() => apiCall.CallAsync(0, null));
Assert.Throws <InvalidOperationException>(() => apiCall.Call(0, null));
}
public async Task CallSettingsDeadlineIsObserved(bool async)
{
var callDuration = TimeSpan.FromTicks(300);
var failures = 4; // Fifth call would succeed, but we won't get that far.
var scheduler = new FakeScheduler();
var time0 = scheduler.Clock.GetCurrentDateTimeUtc();
var server = new Server(failures, callDuration, scheduler);
var callable = server.Callable;
var timeout = TimeSpan.FromTicks(2500);
var retrySettings = new RetrySettings(
maxAttempts: 5,
initialBackoff: TimeSpan.FromTicks(1000),
maxBackoff: TimeSpan.FromTicks(5000),
backoffMultiplier: 2.0,
retryFilter: NotFoundFilter,
backoffJitter: RetrySettings.NoJitter);
var task = scheduler.RunAsync(async() =>
{
// Expiration makes it fail while waiting to make third call
var callSettings = CallSettings.FromRetry(retrySettings).WithTimeout(timeout);
var request = new SimpleRequest {
Name = "irrelevant"
};
await Call(async, scheduler, server, request, callSettings);
});
await Assert.ThrowsAsync <RpcException>(() => task);
var firstCall = time0;
var secondCall = firstCall + callDuration + TimeSpan.FromTicks(1000);
server.AssertCallTimes(firstCall, secondCall);
// We use the same deadline for all calls.
server.AssertDeadlines(time0 + timeout, time0 + timeout);
// We fail immediately when we work out that we would time out before we make the third
// call - so this is before the actual total timeout.
Assert.Equal((secondCall + callDuration).Ticks, scheduler.Clock.GetCurrentDateTimeUtc().Ticks);
}
private FirestoreDb(string projectId, string databaseId, FirestoreClient client, Action <string> warningLogger, SerializationContext serializationContext)
{
ProjectId = GaxPreconditions.CheckNotNull(projectId, nameof(projectId));
DatabaseId = GaxPreconditions.CheckNotNull(databaseId, nameof(databaseId));
Client = GaxPreconditions.CheckNotNull(client, nameof(client));
// TODO: Investigate using DatabaseName and DocumentPathName.
RootPath = $"projects/{ProjectId}/databases/{DatabaseId}";
DocumentsPath = $"{RootPath}/documents";
WarningLogger = warningLogger;
// TODO: Validate these settings, and potentially make them configurable
var batchGetRetry = RetrySettings.FromExponentialBackoff(
maxAttempts: int.MaxValue,
initialBackoff: TimeSpan.FromMilliseconds(500),
maxBackoff: TimeSpan.FromSeconds(5),
backoffMultiplier: 2.0,
retryFilter: RetrySettings.FilterForStatusCodes(StatusCode.Unavailable));
_batchGetCallSettings = CallSettings.FromRetry(batchGetRetry).WithTimeout(TimeSpan.FromMinutes(10));
SerializationContext = GaxPreconditions.CheckNotNull(serializationContext, nameof(serializationContext));
}
public async Task MaxAttemptsObserved(bool async)
{
var callDuration = TimeSpan.FromTicks(300);
var failures = 4; // Fifth call would succeed, but we won't get that far.
var scheduler = new FakeScheduler();
var time0 = scheduler.Clock.GetCurrentDateTimeUtc();
var server = new Server(failures, callDuration, scheduler);
var callable = server.Callable;
var retrySettings = ConstantBackoff(failures, TimeSpan.Zero, NotFoundFilter);
var task = scheduler.RunAsync(async() =>
{
// MaxAttempts makes the overall operation fail on the 4th RPC.
var callSettings = CallSettings.FromRetry(retrySettings);
var request = new SimpleRequest {
Name = "irrelevant"
};
await Call(async, scheduler, server, request, callSettings);
});
await Assert.ThrowsAsync <RpcException>(() => task);
Assert.Equal(4, server.CallTimes.Count);
}
public async Task RetryFilter_EventualSuccess(bool async)
{
StatusCode failureCode = StatusCode.NotFound;
StatusCode[] filterCodes = new[] { StatusCode.NotFound, StatusCode.DeadlineExceeded };
var callDuration = TimeSpan.FromTicks(100);
var failures = 1;
var scheduler = new FakeScheduler();
var server = new Server(failures, callDuration, scheduler, failureCode);
// We're not really interested in the timing in this test.
var retrySettings = ConstantBackoff(5, TimeSpan.Zero, RetrySettings.FilterForStatusCodes(filterCodes));
await scheduler.RunAsync(async() =>
{
var callSettings = CallSettings.FromRetry(retrySettings);
var request = new SimpleRequest {
Name = "irrelevant"
};
await Call(async, scheduler, server, request, callSettings);
});
Assert.True(server.CallTimes.Count() > 1);
}
private FirestoreDb(string projectId, string databaseId, FirestoreClient client, Action <string> warningLogger, SerializationContext serializationContext)
{
ProjectId = GaxPreconditions.CheckNotNull(projectId, nameof(projectId));
DatabaseId = GaxPreconditions.CheckNotNull(databaseId, nameof(databaseId));
Client = GaxPreconditions.CheckNotNull(client, nameof(client));
// TODO: Investigate using DatabaseName and DocumentPathName.
RootPath = $"projects/{ProjectId}/databases/{DatabaseId}";
DocumentsPath = $"{RootPath}/documents";
WarningLogger = warningLogger;
// TODO: Potentially make these configurable.
// The retry settings are taken from firestore_grpc_service_config.json.
var batchGetRetry = RetrySettings.FromExponentialBackoff(
maxAttempts: 5,
initialBackoff: TimeSpan.FromMilliseconds(100),
maxBackoff: TimeSpan.FromSeconds(60),
backoffMultiplier: 1.3,
retryFilter: RetrySettings.FilterForStatusCodes(StatusCode.Unavailable, StatusCode.Internal, StatusCode.DeadlineExceeded));
_batchGetCallSettings = CallSettings.FromRetry(batchGetRetry).WithTimeout(TimeSpan.FromMinutes(10));
SerializationContext = GaxPreconditions.CheckNotNull(serializationContext, nameof(serializationContext));
}
public async Task MultipleCallsEventualSuccess(bool async)
{
var callDuration = TimeSpan.FromTicks(300);
var failures = 4; // Fifth call will succeed
var name = "name"; // Copied from request to response
var scheduler = new FakeScheduler();
var time0 = scheduler.Clock.GetCurrentDateTimeUtc();
var server = new Server(failures, callDuration, scheduler);
var retrySettings = new RetrySettings(
maxAttempts: 5,
initialBackoff: TimeSpan.FromTicks(1000),
maxBackoff: TimeSpan.FromTicks(5000),
backoffMultiplier: 2.0,
retryFilter: NotFoundFilter,
backoffJitter: RetrySettings.NoJitter);
await scheduler.RunAsync(async() =>
{
var callSettings = CallSettings.FromRetry(retrySettings);
var request = new SimpleRequest {
Name = name
};
var result = await Call(async, scheduler, server, request, callSettings);
Assert.Equal(name, result.Name);
});
var firstCall = time0;
var secondCall = firstCall + callDuration + TimeSpan.FromTicks(1000); // Delay for 1000 ticks
var thirdCall = secondCall + callDuration + TimeSpan.FromTicks(2000); // Delay for 2000 ticks
var fourthCall = thirdCall + callDuration + TimeSpan.FromTicks(4000); // Delay for 4000 ticks
var fifthCall = fourthCall + callDuration + TimeSpan.FromTicks(5000); // Delay for 5000 ticks, as that's the max
server.AssertCallTimes(firstCall, secondCall, thirdCall, fourthCall, fifthCall);
// Time of last action was when the call returned
Assert.Equal(fifthCall + callDuration, scheduler.Clock.GetCurrentDateTimeUtc());
}
private async Task RunBulkMessagingImpl(
TopicName topicName, SubscriptionName subscriptionName,
int messageCount, int minMessageSize, int maxMessageSize, int maxMessagesInFlight, int initialNackCount,
TimeSpan?timeouts = null, int?cancelAfterRecvCount = null, TimeSpan?interPublishDelay = null,
TimeSpan?debugOutputPeriod = null, int?publisherChannelCount = null, int?clientCount = null)
{
// Force messages to be at least 4 bytes long, so an int ID can be used.
minMessageSize = Math.Max(4, minMessageSize);
// Create PublisherClient and SubscriberClient
var publisher = await PublisherClient.CreateAsync(topicName,
clientCreationSettings : new PublisherClient.ClientCreationSettings(
clientCount: publisherChannelCount,
publisherServiceApiSettings: timeouts == null ? null : new PublisherServiceApiSettings
{
PublishSettings = CallSettings
.FromRetry(RetrySettings.FromExponentialBackoff(
maxAttempts: int.MaxValue,
initialBackoff: TimeSpan.FromMilliseconds(100),
maxBackoff: TimeSpan.FromSeconds(6),
backoffMultiplier: 1.3,
retryFilter: RetrySettings.FilterForStatusCodes(StatusCode.Unavailable)))
.WithTimeout(timeouts.Value)
}
)).ConfigureAwait(false);
var subscriber = await SubscriberClient.CreateAsync(subscriptionName,
clientCreationSettings : new SubscriberClient.ClientCreationSettings(clientCount: clientCount),
settings : new SubscriberClient.Settings
{
AckDeadline = timeouts,
FlowControlSettings = new FlowControlSettings(maxMessagesInFlight, null)
}).ConfigureAwait(false);
Console.WriteLine("Topic, Subscription, Publisher and Subscriber all created");
// Subscribe
int recvCount = 0; // Count of received messages
int dupCount = 0; // Count of duplicate messages
long recvSum = 0L; // Sum of bytes of received messages
var recvedIds = new ConcurrentDictionary <int, bool>();
var nackedIds = new HashSet <int>();
Task subTask = subscriber.StartAsync((msg, ct) =>
{
int id = BitConverter.ToInt32(msg.Data.ToArray(), 0);
lock (nackedIds)
{
if (nackedIds.Count < initialNackCount)
{
if (nackedIds.Add(id))
{
// This ID not already nacked
Interlocked.Increment(ref recvCount);
return(Task.FromResult(SubscriberClient.Reply.Nack));
}
}
}
bool wasAdded = recvedIds.TryAdd(id, false);
if (wasAdded)
{
var localRecvCount = Interlocked.Increment(ref recvCount);
Interlocked.Add(ref recvSum, msg.Data.Sum(x => (long)x));
if (localRecvCount == cancelAfterRecvCount || localRecvCount >= messageCount + initialNackCount)
{
// Test finished, so stop subscriber
Console.WriteLine("All msgs received, stopping subscriber.");
Task unused = subscriber.StopAsync(TimeSpan.FromSeconds(15));
}
}
else
{
Interlocked.Add(ref dupCount, 1);
}
// ACK all messages
return(Task.FromResult(SubscriberClient.Reply.Ack));
});
// Publish
var rnd = new Random(1234);
var activePubs = new HashSet <Task>();
int sentCount = 0;
long sentSum = 0L; // Sum of bytes of sent messages
// Watchdog to report progress and fail test on deadlock
CancellationTokenSource watchdogCts = new CancellationTokenSource();
Task.Run(async() =>
{
var debugOutputPeriod1 = debugOutputPeriod ?? TimeSpan.FromSeconds(1);
int prevSentCount = -1;
int prevRecvCount = -1;
int noProgressCount = 0;
while (true)
{
await Task.Delay(debugOutputPeriod1, watchdogCts.Token).ConfigureAwait(false);
var localSentCount = Interlocked.Add(ref sentCount, 0);
var localRecvCount = Interlocked.Add(ref recvCount, 0);
var localDupCount = Interlocked.Add(ref dupCount, 0);
if (prevSentCount == localSentCount && prevRecvCount == localRecvCount)
{
if (noProgressCount > 60)
{
// Deadlock, shutdown subscriber, and cancel
Console.WriteLine("Deadlock detected. Cancelling test");
subscriber.StopAsync(new CancellationToken(true));
watchdogCts.Cancel();
break;
}
noProgressCount += 1;
}
else
{
noProgressCount = 0;
}
prevSentCount = localSentCount;
prevRecvCount = localRecvCount;
Console.WriteLine($"[{DateTime.Now}] Sent: {localSentCount} (in-flight: {activePubs.Locked(() => activePubs.Count)}); Recv: {localRecvCount} (dups: {localDupCount})");
}
});
for (int i = 0; i < messageCount; i++)
{
if (watchdogCts.IsCancellationRequested)
{
Assert.True(false, "Test cancelled by watchdog");
}
if (subTask.IsCompleted)
{
break;
}
if (i > 0 && interPublishDelay is TimeSpan delay)
{
await Task.Delay(delay, watchdogCts.Token).ConfigureAwait(false);
}
var msgSize = rnd.Next(minMessageSize, maxMessageSize + 1);
var msg = new byte[msgSize];
rnd.NextBytes(msg);
// Insert an int ID into message
Array.Copy(BitConverter.GetBytes(i), msg, 4);
sentSum += msg.Sum(x => (long)x);
// Send message, and record Task
var pubTask = publisher.PublishAsync(msg);
Interlocked.Increment(ref sentCount);
activePubs.Locked(() => activePubs.Add(pubTask));
// Remove Task from active when the message has been sent to server
pubTask.ContinueWith(t => activePubs.Locked(() => activePubs.Remove(pubTask)));
// If too many messages are currently in flight, wait a bit
while (activePubs.Locked(() => activePubs.Count) >= maxMessagesInFlight)
{
await Task.Delay(TimeSpan.FromMilliseconds(1)).ConfigureAwait(false);
}
}
Console.WriteLine("Publishing complete");
// Wait for all messages to be sent to server
await Task.WhenAll(activePubs.Locked(() => activePubs.ToArray())).ConfigureAwait(false);
Console.WriteLine("Publishing completed sending to server");
// Wait for subscriber to finish shutdown
await subTask.ConfigureAwait(false);
watchdogCts.Cancel();
Console.WriteLine("Subscriber finished shutdown");
Console.WriteLine($"Sent: {sentCount}; Recv: {recvCount}");
if (cancelAfterRecvCount is int cancelAfter)
{
Assert.True(recvCount >= cancelAfter && recvCount <= cancelAfter + maxMessagesInFlight, $"Incorrect recvCount: {recvCount}");
}
else
{
// Check that all messages are correctly received.
Assert.Equal(messageCount + initialNackCount, recvCount);
// This isn't foolproof (we can get to the right sum with wrong values) but it's a pretty strong indicator.
Assert.Equal(sentSum, recvSum);
}
Console.WriteLine("Test complete.");
}
