void TestCreateDatabase()
{
try
{
// Attempt to create another database with same name. Should fail.
ConsoleOutput created_again = _spannerCmd.Run("createSampleDatabase",
_fixture.ProjectId, _fixture.InstanceId, _fixture.DatabaseId);
}
catch (AggregateException e)
{
bool rethrow = true;
foreach (var innerException in e.InnerExceptions)
{
SpannerException spannerException = innerException as SpannerException;
if (spannerException != null && spannerException.Message.ToLower().Contains("duplicate"))
{
Console.WriteLine($"Database {_fixture.DatabaseId} already exists.");
rethrow = false;
break;
}
}
if (rethrow)
{
throw;
}
}
// List tables to confirm database tables exist.
ConsoleOutput output = _spannerCmd.Run("listDatabaseTables",
_fixture.ProjectId, _fixture.InstanceId, _fixture.DatabaseId);
Assert.Equal(0, output.ExitCode);
Assert.Contains("Albums", output.Stdout);
Assert.Contains("Singers", output.Stdout);
}
internal static bool SpannerExceptionsEqualForRetry(SpannerException e1, SpannerException e2)
{
// Quick return for the most common case.
if (e1 == null && e2 == null)
{
return(true);
}
if (!Equals(e1?.ErrorCode, e2?.ErrorCode))
{
return(false);
}
if (!Equals(e1?.Message, e2?.Message))
{
return(false);
}
if (!Equals(e1?.InnerException?.GetType(), e2?.InnerException?.GetType()))
{
return(false);
}
if (e1?.InnerException is RpcException)
{
Status status1 = ((RpcException)e1.InnerException).Status;
Status status2 = ((RpcException)e2.InnerException).Status;
if (!(Equals(status1.StatusCode, status2.StatusCode) && Equals(status1.Detail, status2.Detail)))
{
return(false);
}
}
return(true);
}
public override async Task <bool> ReadAsync(CancellationToken cancellationToken)
{
while (true)
{
try
{
bool res = await _spannerDataReader.ReadAsync(cancellationToken);
_currentChecksum = CalculateNextChecksum(_spannerDataReader, _currentChecksum, res);
_numberOfReadCalls++;
return(res);
}
catch (SpannerException e) when(e.ErrorCode == ErrorCode.Aborted)
{
// Retry the transaction and then retry the ReadAsync call.
await Transaction.RetryAsync(e, cancellationToken);
}
catch (SpannerException e)
{
if (_firstException == null)
{
_firstException = e;
}
_numberOfReadCalls++;
throw;
}
}
}
/// <summary>
/// Asserts that the given exception was due to a timeout. Sometimes gRPC
/// will return an exception with a status code of Unavailable when it's really
/// a timeout. This appears to depend on which combination of tests is being run,
/// so may well be around how the underlying gRPC channels are being used. We test
/// in a slightly looser way than might be expected to work around this.
/// </summary>
public static void IsTimeout(SpannerException exception)
{
// TODO: Investigate the cause of this further.
var code = exception.ErrorCode;
Assert.True(code == ErrorCode.DeadlineExceeded || code == ErrorCode.Unavailable,
$"Expected code of DeadlineExceeded or Unavailable; was {code}");
Assert.False(exception.IsTransientSpannerFault());
}
internal async Task RetryAsync(SpannerException abortedException, CancellationToken cancellationToken, int timeoutSeconds = MAX_TIMEOUT_SECONDS)
{
if (!EnableInternalRetries)
{
throw abortedException;
}
DateTime?overallDeadline = _options.CalculateDeadline(_clock);
TimeSpan retryDelay = _options.InitialDelay;
// If there's a recommended retry delay specified on the exception
// we should respect it.
retryDelay = abortedException.RecommendedRetryDelay ?? _options.Jitter(retryDelay);
while (true)
{
if (RetryCount >= MaxInternalRetryCount)
{
throw new SpannerException(ErrorCode.Aborted, "Transaction was aborted because it aborted and retried too many times");
}
DateTime expectedRetryTime = _clock.GetCurrentDateTimeUtc() + retryDelay;
if (expectedRetryTime > overallDeadline)
{
throw new SpannerException(ErrorCode.Aborted, "Transaction was aborted because it timed out while retrying");
}
await _scheduler.Delay(retryDelay, cancellationToken).ConfigureAwait(false);
// TODO: Preferably the Spanner client library should have some 'reset' option on an existing
// transaction instead of having to begin a new transaction. This will potentially use a transaction
// on a different session, while the recommended behavior for a retry is to use the same session as
// the original attempt.
SpannerTransaction.Dispose();
SpannerTransaction = await Connection.SpannerConnection.BeginTransactionAsync(cancellationToken);
RetryCount++;
try
{
foreach (IRetriableStatement statement in _retriableStatements)
{
await statement.RetryAsync(this, cancellationToken, timeoutSeconds);
}
break;
}
catch (SpannerAbortedDueToConcurrentModificationException)
{
// Retry failed because of a concurrent modification.
throw;
}
catch (SpannerException e) when(e.ErrorCode == ErrorCode.Aborted)
{
// Ignore and retry.
retryDelay = e.RecommendedRetryDelay ?? _options.Jitter(_options.NextDelay(retryDelay));
}
}
}
/// <summary>
/// Returns true if an AggregateException contains a SpannerException
/// with the given error code.
/// </summary>
/// <param name="e">The exception to examine.</param>
/// <param name="errorCode">The error code to look for.</param>
/// <returns></returns>
static bool ContainsError(AggregateException e, ErrorCode errorCode)
{
foreach (var innerException in e.InnerExceptions)
{
SpannerException spannerException = innerException as SpannerException;
if (spannerException != null && spannerException.ErrorCode == errorCode)
{
return(true);
}
}
return(false);
}
async Task IRetriableStatement.RetryAsync(SpannerRetriableTransaction transaction, CancellationToken cancellationToken, int timeoutSeconds)
{
_spannerCommand.Transaction = transaction.SpannerTransaction;
var reader = await _spannerCommand.ExecuteReaderAsync(cancellationToken);
int counter = 0;
bool read = true;
byte[] newChecksum = new byte[0];
SpannerException newException = null;
while (read && counter < _numberOfReadCalls)
{
try
{
read = await reader.ReadAsync(cancellationToken);
newChecksum = CalculateNextChecksum(reader, newChecksum, read);
counter++;
}
catch (SpannerException e) when(e.ErrorCode == ErrorCode.Aborted)
{
// Propagate Aborted errors to trigger a new retry.
throw;
}
catch (SpannerException e)
{
newException = e;
counter++;
break;
}
}
if (counter == _numberOfReadCalls &&
newChecksum.SequenceEqual(_currentChecksum) &&
SpannerRetriableTransaction.SpannerExceptionsEqualForRetry(newException, _firstException))
{
// Checksum is ok, we only need to replace the delegate result set if it's still open.
if (IsClosed)
{
reader.Close();
}
else
{
_spannerDataReader = reader;
}
}
else
{
// The results are not equal, there is an actual concurrent modification, so we cannot
// continue the transaction.
throw new SpannerAbortedDueToConcurrentModificationException();
}
}
/// <summary>
/// Returns true if an AggregateException contains a SpannerException
/// with the given error code.
/// </summary>
/// <param name="e">The exception to examine.</param>
/// <param name="errorCode">The error code to look for.</param>
/// <returns></returns>
public static bool ContainsError(AggregateException e, params ErrorCode[] errorCode)
{
foreach (var innerException in e.InnerExceptions)
{
SpannerException spannerException = innerException as SpannerException;
if (spannerException != null && errorCode.Contains(spannerException.ErrorCode))
{
return(true);
}
}
return(false);
}
public async Task AbortedThrownCorrectly()
{
await WriteSampleRowsAsync();
// connection 1 starts a transaction and reads
// connection 2 starts a transaction and reads the same row
// connection 1 writes and commits
// connection 2 reads again -- abort should be thrown.
var connection1 = new SpannerConnection(_testFixture.ConnectionString);
var connection2 = new SpannerConnection(_testFixture.ConnectionString);
await Task.WhenAll(connection1.OpenAsync(), connection2.OpenAsync());
var tx1 = await connection1.BeginTransactionAsync();
// TX1 READ
using (var cmd = connection1.CreateSelectCommand(
"SELECT * FROM TX WHERE K=@k",
new SpannerParameterCollection {
{ "k", _key, SpannerDbType.String }
}))
{
cmd.Transaction = tx1;
using (var reader = await cmd.ExecuteReaderAsync())
{
Assert.True(await reader.ReadAsync());
}
}
// TX2 START
var tx2 = await connection2.BeginTransactionAsync();
// TX2 READ
using (var cmd = connection2.CreateSelectCommand(
"SELECT * FROM TX WHERE K=@k",
new SpannerParameterCollection {
{ "k", _key, SpannerDbType.String }
}))
{
cmd.Transaction = tx2;
using (var reader = await cmd.ExecuteReaderAsync())
{
Assert.True(await reader.ReadAsync());
}
}
// TX1 WRITE/COMMIT
using (var cmd = connection1.CreateUpdateCommand(
"TX",
new SpannerParameterCollection
{
{ "k", _key, SpannerDbType.String },
{ "Int64Value", 0, SpannerDbType.Int64 }
}))
{
cmd.Transaction = tx1;
await cmd.ExecuteNonQueryAsync().ConfigureAwait(false);
await tx1.CommitAsync().ConfigureAwait(false);
tx1.Dispose();
}
connection1.Dispose();
// TX2 READ AGAIN/THROWS
SpannerException thrownException = null;
try
{
using (var cmd = connection2.CreateSelectCommand(
"SELECT * FROM TX WHERE K=@k",
new SpannerParameterCollection {
{ "k", _key, SpannerDbType.String }
}))
{
cmd.Transaction = tx2;
using (var reader = await cmd.ExecuteReaderAsync())
{
Assert.True(await reader.ReadAsync());
}
}
}
catch (SpannerException ex)
{
thrownException = ex;
}
finally
{
tx2.Dispose();
connection2.Dispose();
}
Assert.True(thrownException?.IsRetryable ?? false);
}
internal void Retry(SpannerException abortedException, int timeoutSeconds = MAX_TIMEOUT_SECONDS) => RetryAsync(abortedException, CancellationToken.None, timeoutSeconds).WaitWithUnwrappedExceptions();
internal FailedBatchDmlStatement(SpannerRetriableBatchCommand command, SpannerException exception)
{
_command = command;
_exception = exception;
}
internal FailedDmlStatement(SpannerCommand command, SpannerException exception)
{
_command = (SpannerCommand)command.Clone();
_exception = exception;
}
