public async Task ErrorWithBackoffAfterDeadline_FailsRetry()
{
// Create a clock that starts at zero ticks.
var clock = new FakeClock(0);
var scheduler = new AdvanceFakeClockScheduler(clock);
var callSettings = CallSettings.FromExpiration(Expiration.FromDeadline(new DateTime(TimeSpan.FromSeconds(10).Ticks, DateTimeKind.Utc)));
var state = new RetryState(clock, scheduler, s_retrySettings, callSettings);
// The retry info contains a wait time that is past the deadline of the call.
// The retry state will throw a DeadlineExceeded error without waiting.
var retryInfo = new Rpc.RetryInfo {
RetryDelay = Duration.FromTimeSpan(TimeSpan.FromSeconds(20))
};
Metadata trailers = new Metadata
{
{ RetryState.RetryInfoKey, retryInfo.ToByteArray() }
};
var exception = new RpcException(new Status(StatusCode.Unavailable, "Bang"), trailers);
Assert.True(state.CanRetry(exception));
await Assert.ThrowsAsync <RpcException>(() => state.WaitAsync(exception, default));
// Check that the clock has not been advanced to verify that the retry state did not wait 20 seconds
// before throwing an exception.
Assert.Equal(0, clock.GetCurrentDateTimeUtc().Ticks);
}
public async Task MultipleRetryableErrors_ForDifferentResumeTokens_ResetsTimeout(System.Type type)
{
var results = CreateResultSets("token1", "token2", "token3");
var filter1 = CreateExceptionFilter("token2", RetriableStatusCode);
var filter2 = CreateExceptionFilter("token3", RetriableStatusCode);
var client = new FakeSpannerClient(results, prs => filter2(filter1(prs)));
var clock = new FakeClock();
client.Settings.Clock = clock;
client.Settings.Scheduler = new AdvanceFakeClockScheduler(clock);
// The call will have a timeout of 15 seconds and a backoff of 10 seconds.
// One retry will therefore not cause it to fail. Two retries for different
// calls will also not cause it to fail, as the timeout is reset after each
// successful RPC.
var callSettings = CallSettings.FromExpiration(Expiration.FromTimeout(TimeSpan.FromSeconds(15)));
var retrySettings = RetrySettings.FromExponentialBackoff(
maxAttempts: int.MaxValue,
initialBackoff: TimeSpan.FromSeconds(10),
maxBackoff: TimeSpan.FromSeconds(10),
backoffMultiplier: 1.0,
retryFilter: ignored => false,
RetrySettings.NoJitter);
var stream = CreateResultStream(type, client, maxBufferSize: 10, callSettings, retrySettings);
await AssertResultsAsync(stream, results);
Assert.Equal(3, client.Calls);
}
public async Task MultipleRetryableErrors_ForSameResumeToken_CausesTimeout(System.Type type)
{
var results = CreateResultSets("token1", "token2", "token3");
var filter = CreateExceptionFilter("token3", RetriableStatusCode, 2);
var client = new FakeSpannerClient(results, filter);
var clock = new FakeClock();
client.Settings.Clock = clock;
client.Settings.Scheduler = new AdvanceFakeClockScheduler(clock);
// The call will have a timeout of 15 seconds and a backoff of 10 seconds.
// token3 will return two consecutive retryable errors. After the second backoff
// the timeout of the call has been exceeded and a DeadlineExceeded error is thrown.
var callSettings = CallSettings.FromExpiration(Expiration.FromTimeout(TimeSpan.FromSeconds(15)));
var retrySettings = RetrySettings.FromExponentialBackoff(
maxAttempts: int.MaxValue,
initialBackoff: TimeSpan.FromSeconds(10),
maxBackoff: TimeSpan.FromSeconds(10),
backoffMultiplier: 1.0,
retryFilter: ignored => false,
RetrySettings.NoJitter);
var stream = CreateResultStream(type, client, maxBufferSize: 10, callSettings, retrySettings);
await AssertResultsThenExceptionAsync(stream, results, 2);
// 2 calls == 1 initial call + 1 retry of token3.
// A second retry of token3 is never executed as the deadline has already been exceeded.
Assert.Equal(2, client.Calls);
}
public async Task ConsecutiveErrors_FailsRetryWhenDeadlineExceeded()
{
var clock = new FakeClock(0);
var scheduler = new AdvanceFakeClockScheduler(clock);
var callSettings = CallSettings.FromExpiration(Expiration.FromDeadline(new DateTime(TimeSpan.FromSeconds(7).Ticks, DateTimeKind.Utc)));
var state = new RetryState(clock, scheduler, s_retrySettings, callSettings);
var retryInfo = new Rpc.RetryInfo {
RetryDelay = Duration.FromTimeSpan(TimeSpan.FromSeconds(3))
};
Metadata trailers = new Metadata
{
{ RetryState.RetryInfoKey, retryInfo.ToByteArray() }
};
var exception = new RpcException(new Status(StatusCode.Unavailable, "Bang"), trailers);
Assert.True(state.CanRetry(exception));
await state.WaitAsync(exception, default);
Assert.True(state.CanRetry(exception));
await state.WaitAsync(exception, default);
Assert.True(state.CanRetry(exception));
await Assert.ThrowsAsync <RpcException>(() => state.WaitAsync(exception, default));
// Verify that the clock has been advanced 6 seconds.
Assert.Equal(TimeSpan.FromSeconds(6).Ticks, clock.GetCurrentDateTimeUtc().Ticks);
}
public async Task RetryableError_ExceedsTimeout(System.Type type)
{
var results = CreateResultSets("token1", "token2", "token3");
var filter = CreateExceptionFilter("token2", RetriableStatusCode);
var client = new FakeSpannerClient(results, filter);
var clock = new FakeClock();
client.Settings.Clock = clock;
client.Settings.Scheduler = new AdvanceFakeClockScheduler(clock);
// Create a call that will timeout if it has to backoff and retry once.
var callSettings = CallSettings.FromExpiration(Expiration.FromTimeout(TimeSpan.FromSeconds(10)));
var retrySettings = RetrySettings.FromExponentialBackoff(
maxAttempts: int.MaxValue,
initialBackoff: TimeSpan.FromSeconds(20),
maxBackoff: TimeSpan.FromSeconds(30),
backoffMultiplier: 2.0,
retryFilter: ignored => false,
RetrySettings.NoJitter);
var stream = CreateResultStream(type, client, maxBufferSize: 10, callSettings, retrySettings);
await AssertResultsThenExceptionAsync(stream, results, 1);
Assert.Equal(1, client.Calls);
}
public async Task ResetDeadline()
{
var clock = new FakeClock(0);
var scheduler = new AdvanceFakeClockScheduler(clock);
var callSettings = CallSettings.FromExpiration(Expiration.FromDeadline(new DateTime(TimeSpan.FromSeconds(7).Ticks, DateTimeKind.Utc)));
var state = new RetryState(clock, scheduler, s_retrySettings, callSettings);
var retryInfo = new Rpc.RetryInfo {
RetryDelay = Duration.FromTimeSpan(TimeSpan.FromSeconds(3))
};
Metadata trailers = new Metadata
{
{ RetryState.RetryInfoKey, retryInfo.ToByteArray() }
};
var exception = new RpcException(new Status(StatusCode.Unavailable, "Bang"), trailers);
Assert.True(state.CanRetry(exception));
await state.WaitAsync(exception, default);
Assert.True(state.CanRetry(exception));
await state.WaitAsync(exception, default);
// Reset does not change the absolute deadline of the call.
// The next retry attempt will therefore fail.
state.Reset();
Assert.True(state.CanRetry(exception));
await Assert.ThrowsAsync <RpcException>(() => state.WaitAsync(exception, default));
Assert.Equal(TimeSpan.FromSeconds(6).Ticks, clock.GetCurrentDateTimeUtc().Ticks);
}
// [START monitoring_uptime_check_create]
public static object CreateUptimeCheck(string projectId, string hostName,
string displayName)
{
// Define a new config.
var config = new UptimeCheckConfig()
{
DisplayName = displayName,
MonitoredResource = new MonitoredResource()
{
Type = "uptime_url",
Labels = { { "host", hostName } }
},
HttpCheck = new UptimeCheckConfig.Types.HttpCheck()
{
Path = "/",
Port = 80,
},
Timeout = TimeSpan.FromSeconds(10).ToDuration(),
Period = TimeSpan.FromMinutes(5).ToDuration()
};
// Create a client.
var client = UptimeCheckServiceClient.Create();
ProjectName projectName = new ProjectName(projectId);
// Create the config.
var newConfig = client.CreateUptimeCheckConfig(
projectName,
config,
CallSettings.FromExpiration(
Expiration.FromTimeout(
TimeSpan.FromMinutes(2))));
Console.WriteLine(newConfig.Name);
return(0);
}
public async Task RetryAfterTotalExpiration()
{
var settings = new BigtableServiceApiSettings();
// Don't allow for any time to retry.
settings.MutateRowsSettings = CallSettings.FromExpiration(Expiration.FromTimeout(TimeSpan.Zero));
var request = new MutateRowsRequest
{
Entries =
{
Mutations.CreateEntry("a", Mutations.DeleteFromRow()),
Mutations.CreateEntry("b", Mutations.DeleteFromRow()),
Mutations.CreateEntry("c", Mutations.DeleteFromRow())
}
};
var client = Utilities.CreateMutateRowsMockClient(
request,
entriesForInitialStream: new[]
{
Utilities.CreateMutateRowsResponseEntry(0, Code.Ok),
Utilities.CreateMutateRowsResponseEntry(1, Code.DeadlineExceeded),
Utilities.CreateMutateRowsResponseEntry(2, Code.Ok)
},
entriesForRetryStreams: new[]
{
// 1st retry response entries
new[] { Utilities.CreateMutateRowsResponseEntry(0, Code.Ok) }
},
settings: settings);
var exception = await Assert.ThrowsAsync <RpcException>(() => client.MutateRowsAsync(request));
Assert.Equal(StatusCode.DeadlineExceeded, exception.StatusCode);
}
public void FromExpiration()
{
Assert.Null(CallSettings.FromExpiration(null));
var expiration = Expiration.FromTimeout(TimeSpan.FromSeconds(1));
var settings = CallSettings.FromExpiration(expiration);
Assert.Same(expiration, settings.Expiration);
}
public void ToCallOptions_ExpirationNone()
{
var deadline = new DateTime(2015, 6, 19, 5, 2, 3, DateTimeKind.Utc);
var mockClock = new Mock <IClock>();
CallSettings callSettings = CallSettings.FromExpiration(Expiration.None);
var options = callSettings.ToCallOptions(mockClock.Object);
Assert.Null(options.Deadline);
mockClock.Verify(c => c.GetCurrentDateTimeUtc(), Times.Never);
}
public void ToCallOptions_ExpirationTimeout()
{
var clock = new FakeClock();
var timeout = TimeSpan.FromSeconds(1);
CallSettings callSettings = CallSettings.FromExpiration(Expiration.FromTimeout(timeout));
var options = callSettings.ToCallOptions(clock);
// Value should be exact, as we control time precisely.
Assert.Equal(options.Deadline.Value, clock.GetCurrentDateTimeUtc() + timeout);
}
public async Task SucceedWithExpiration()
{
var apiCall = ApiServerStreamingCall.Create <int, int>(
(request, callOptions) => null,
CallSettings.FromExpiration(Expiration.FromTimeout(TimeSpan.FromSeconds(100))),
new FakeClock());
Assert.Null(await apiCall.CallAsync(0, null));
Assert.Null(apiCall.Call(0, null));
}
public void SucceedWithExpiration()
{
var apiCall = ApiBidirectionalStreamingCall.Create <int, int>(
callOptions => null,
CallSettings.FromExpiration(Expiration.FromTimeout(TimeSpan.FromSeconds(100))),
new BidirectionalStreamingSettings(100),
new FakeClock());
Assert.Null(apiCall.Call(null));
}
public void ToCallOptions_ExpirationDeadline()
{
var deadline = new DateTime(2015, 6, 19, 5, 2, 3, DateTimeKind.Utc);
var mockClock = new Mock <IClock>();
CallSettings callSettings = CallSettings.FromExpiration(Expiration.FromDeadline(deadline));
var options = callSettings.ToCallOptions(mockClock.Object);
// Value should be exact, as we control time precisely.
Assert.Equal(options.Deadline.Value, deadline);
mockClock.Verify(c => c.GetCurrentDateTimeUtc(), Times.Never);
}
/// <summary>
/// Makes a server streaming call using a custom client timeout.
/// </summary>
/// <param name="client">The Google Ads client.</param>
/// <param name="customerId">The customer ID for which the call is made.</param>
// [START set_custom_client_timeouts]
private void MakeServerStreamingCall(GoogleAdsClient client, long customerId)
{
StringWriter writer = new StringWriter();
// Set a custom timeout.
CallSettings callSettings = CallSettings.FromExpiration(
Expiration.FromTimeout(TimeSpan.FromMilliseconds(CLIENT_TIMEOUT_MILLIS)));
try
{
// Get the GoogleAdsService.
GoogleAdsServiceClient googleAdsService = client.GetService(
Services.V10.GoogleAdsService);
string query = "SELECT campaign.id FROM campaign";
SearchGoogleAdsStreamRequest request = new SearchGoogleAdsStreamRequest()
{
CustomerId = customerId.ToString(),
Query = query
};
// Issue a search request.
googleAdsService.SearchStream(request,
delegate(SearchGoogleAdsStreamResponse resp)
{
// Iterates over all rows in all messages and collects the campaign IDs.
foreach (GoogleAdsRow googleAdsRow in resp.Results)
{
writer.WriteLine(googleAdsRow.Campaign.Id);
}
},
callSettings
);
Console.WriteLine("The server streaming call completed before the timeout");
}
catch (GoogleAdsException ex)
{
if (ex.StatusCode == StatusCode.DeadlineExceeded)
{
Console.WriteLine("The server streaming call did not complete before the " +
"timeout.");
}
else
{
throw;
}
}
finally
{
Console.WriteLine($"All campaign IDs retrieved: {writer}.");
}
}
public async Task RetryingAfterTotalExpiration()
{
var settings = new BigtableServiceApiSettings();
// Don't allow for any time to retry.
settings.ReadRowsSettings = CallSettings.FromExpiration(Expiration.FromTimeout(TimeSpan.Zero));
var request = new ReadRowsRequest
{
Rows = RowSet.FromRowKeys("a", "b", "c")
};
var client = Utilities.CreateReadRowsMockClient(
request,
initialStreamResponse: new[]
{
new ReadRowsResponse
{
Chunks =
{
CreateChunk("a", "cf1", "column1", "value1", commitRow: true)
}
}
},
responsesForRetryStreams: new[]
{
null, // A null entry will throw an Unavailable RpcException
new []
{
new ReadRowsResponse
{
Chunks =
{
CreateChunk("b", "cf1", "column2", "value2", commitRow: true)
}
}
}
},
settings: settings);
var exception = await Assert.ThrowsAsync <RpcException>(() => client.ReadRows(request).ToListAsync().AsTask());
Assert.Equal(StatusCode.Unavailable, exception.StatusCode);
}
public void Overrides()
{
string projectId = _fixture.ProjectId;
string topicId = _fixture.CreateTopicId();
DateTime deadline = DateTime.MaxValue;
CancellationToken cancellationToken = new CancellationTokenSource().Token;
// Sample: Overrides
// Create a default PublisherServiceApiSettings, with customizations for CreateTopic RPCs:
// * A custom "ClientVersion" header.
// * A custom 5-second timeout Timing.
// * No cancellation token.
PublisherServiceApiSettings publisherSettings = new PublisherServiceApiSettings();
publisherSettings.CreateTopicSettings = publisherSettings.CreateTopicSettings
.WithCancellationToken(CancellationToken.None)
.WithTimeout(TimeSpan.FromSeconds(5))
.WithHeader("ClientVersion", "1.0.0");
// Override the above Timing and CancellationToken in the client-wide CallSettings;
// the Headers are not overridden.
publisherSettings.CallSettings = CallSettings
.FromExpiration(Expiration.FromDeadline(deadline))
.WithCancellationToken(CancellationToken.None);
// Create the client with the configured publisherSettings
PublisherServiceApiClient client = new PublisherServiceApiClientBuilder {
Settings = publisherSettings
}.Build();
// Create a topic name from the projectId and topicId.
TopicName topicName = new TopicName(projectId, topicId);
// Call CreateTopic(). Override only the CancellationToken, using a per-RPC-method CallSettings.
// The CallSettings used during this RPC invocation is:
// * A custom "ClientVersion" header.
// * A Timing deadline of 'deadline' (*not* the overridden 5-second timeout).
// * The CancellationToken 'cancellationToken' (*not* CancellationToken.None).
Topic topic = client.CreateTopic(topicName, CallSettings.FromCancellationToken(cancellationToken));
// End sample
}
public async Task <long> CustomTimeoutsAndRetriesAsync(string projectId, string instanceId, string databaseId)
{
// Create a SessionPool.
SpannerClient client = SpannerClient.Create();
SessionPool sessionPool = new SessionPool(client, new SessionPoolOptions());
// Acquire a session with a read-write transaction to run a query.
DatabaseName databaseName =
DatabaseName.FromProjectInstanceDatabase(projectId, instanceId, databaseId);
TransactionOptions transactionOptions = new TransactionOptions
{
ReadWrite = new ReadWrite()
};
using PooledSession session = await sessionPool.AcquireSessionAsync(
databaseName, transactionOptions, CancellationToken.None);
ExecuteSqlRequest request = new ExecuteSqlRequest
{
Sql = "INSERT Singers (SingerId, FirstName, LastName) VALUES (20, 'George', 'Washington')"
};
// Prepare the call settings with custom timeout and retry settings.
CallSettings settings = CallSettings
.FromExpiration(Expiration.FromTimeout(TimeSpan.FromSeconds(60)))
.WithRetry(RetrySettings.FromExponentialBackoff(
maxAttempts: 12,
initialBackoff: TimeSpan.FromMilliseconds(500),
maxBackoff: TimeSpan.FromMilliseconds(6400),
backoffMultiplier: 1.5,
retryFilter: RetrySettings.FilterForStatusCodes(
new StatusCode[] { StatusCode.Unavailable, StatusCode.DeadlineExceeded })));
ResultSet result = await session.ExecuteSqlAsync(request, settings);
await session.CommitAsync(new CommitRequest(), null);
return(result.Stats.RowCountExact);
}
public async Task ResetTimeout()
{
var clock = new FakeClock(0);
var scheduler = new AdvanceFakeClockScheduler(clock);
var callSettings = CallSettings.FromExpiration(Expiration.FromTimeout(TimeSpan.FromSeconds(7)));
var state = new RetryState(clock, scheduler, s_retrySettings, callSettings);
var retryInfo = new Rpc.RetryInfo {
RetryDelay = Duration.FromTimeSpan(TimeSpan.FromSeconds(3))
};
Metadata trailers = new Metadata
{
{ RetryState.RetryInfoKey, retryInfo.ToByteArray() }
};
var exception = new RpcException(new Status(StatusCode.Unavailable, "Bang"), trailers);
Assert.True(state.CanRetry(exception));
await state.WaitAsync(exception, default);
Assert.True(state.CanRetry(exception));
await state.WaitAsync(exception, default);
// Reset should set the deadline of the call to CurrentTime + Timeout.
// That means that we can do two new retries without a timeout exception.
state.Reset();
Assert.True(state.CanRetry(exception));
await state.WaitAsync(exception, default);
Assert.True(state.CanRetry(exception));
await state.WaitAsync(exception, default);
Assert.True(state.CanRetry(exception));
await Assert.ThrowsAsync <RpcException>(() => state.WaitAsync(exception, default));
// Verify that the clock has been advanced 12 seconds.
Assert.Equal(TimeSpan.FromSeconds(12).Ticks, clock.GetCurrentDateTimeUtc().Ticks);
}
private static async Task CreateSubsriptionIfDoesntExist(SubscriptionName subscribtionName, TopicName topicName)
{
var subscriberService = await SubscriberServiceApiClient.CreateAsync();
try
{
subscriberService.GetSubscription(subscribtionName);
}
catch (RpcException rex)
{
if (rex.Status.StatusCode == StatusCode.NotFound)
{
// expire subscription if it's not in use
var callSettings = Expiration.FromTimeout(TimeSpan.FromHours(1));
subscriberService.CreateSubscription(subscribtionName, topicName, pushConfig: null, ackDeadlineSeconds: 60, callSettings: CallSettings.FromExpiration(callSettings));
}
else
{
throw;
}
}
}
