void TestCreateDatabase() { try { // Attempt to create another database with same name. Should fail. ConsoleOutput created_again = _spannerCmd.Run("createSampleDatabase", _fixture.ProjectId, _fixture.InstanceId, _fixture.DatabaseId); } catch (AggregateException e) { bool rethrow = true; foreach (var innerException in e.InnerExceptions) { SpannerException spannerException = innerException as SpannerException; if (spannerException != null && spannerException.Message.ToLower().Contains("duplicate")) { Console.WriteLine($"Database {_fixture.DatabaseId} already exists."); rethrow = false; break; } } if (rethrow) { throw; } } // List tables to confirm database tables exist. ConsoleOutput output = _spannerCmd.Run("listDatabaseTables", _fixture.ProjectId, _fixture.InstanceId, _fixture.DatabaseId); Assert.Equal(0, output.ExitCode); Assert.Contains("Albums", output.Stdout); Assert.Contains("Singers", output.Stdout); }
internal static bool SpannerExceptionsEqualForRetry(SpannerException e1, SpannerException e2) { // Quick return for the most common case. if (e1 == null && e2 == null) { return(true); } if (!Equals(e1?.ErrorCode, e2?.ErrorCode)) { return(false); } if (!Equals(e1?.Message, e2?.Message)) { return(false); } if (!Equals(e1?.InnerException?.GetType(), e2?.InnerException?.GetType())) { return(false); } if (e1?.InnerException is RpcException) { Status status1 = ((RpcException)e1.InnerException).Status; Status status2 = ((RpcException)e2.InnerException).Status; if (!(Equals(status1.StatusCode, status2.StatusCode) && Equals(status1.Detail, status2.Detail))) { return(false); } } return(true); }
public override async Task <bool> ReadAsync(CancellationToken cancellationToken) { while (true) { try { bool res = await _spannerDataReader.ReadAsync(cancellationToken); _currentChecksum = CalculateNextChecksum(_spannerDataReader, _currentChecksum, res); _numberOfReadCalls++; return(res); } catch (SpannerException e) when(e.ErrorCode == ErrorCode.Aborted) { // Retry the transaction and then retry the ReadAsync call. await Transaction.RetryAsync(e, cancellationToken); } catch (SpannerException e) { if (_firstException == null) { _firstException = e; } _numberOfReadCalls++; throw; } } }
/// <summary> /// Asserts that the given exception was due to a timeout. Sometimes gRPC /// will return an exception with a status code of Unavailable when it's really /// a timeout. This appears to depend on which combination of tests is being run, /// so may well be around how the underlying gRPC channels are being used. We test /// in a slightly looser way than might be expected to work around this. /// </summary> public static void IsTimeout(SpannerException exception) { // TODO: Investigate the cause of this further. var code = exception.ErrorCode; Assert.True(code == ErrorCode.DeadlineExceeded || code == ErrorCode.Unavailable, $"Expected code of DeadlineExceeded or Unavailable; was {code}"); Assert.False(exception.IsTransientSpannerFault()); }
internal async Task RetryAsync(SpannerException abortedException, CancellationToken cancellationToken, int timeoutSeconds = MAX_TIMEOUT_SECONDS) { if (!EnableInternalRetries) { throw abortedException; } DateTime?overallDeadline = _options.CalculateDeadline(_clock); TimeSpan retryDelay = _options.InitialDelay; // If there's a recommended retry delay specified on the exception // we should respect it. retryDelay = abortedException.RecommendedRetryDelay ?? _options.Jitter(retryDelay); while (true) { if (RetryCount >= MaxInternalRetryCount) { throw new SpannerException(ErrorCode.Aborted, "Transaction was aborted because it aborted and retried too many times"); } DateTime expectedRetryTime = _clock.GetCurrentDateTimeUtc() + retryDelay; if (expectedRetryTime > overallDeadline) { throw new SpannerException(ErrorCode.Aborted, "Transaction was aborted because it timed out while retrying"); } await _scheduler.Delay(retryDelay, cancellationToken).ConfigureAwait(false); // TODO: Preferably the Spanner client library should have some 'reset' option on an existing // transaction instead of having to begin a new transaction. This will potentially use a transaction // on a different session, while the recommended behavior for a retry is to use the same session as // the original attempt. SpannerTransaction.Dispose(); SpannerTransaction = await Connection.SpannerConnection.BeginTransactionAsync(cancellationToken); RetryCount++; try { foreach (IRetriableStatement statement in _retriableStatements) { await statement.RetryAsync(this, cancellationToken, timeoutSeconds); } break; } catch (SpannerAbortedDueToConcurrentModificationException) { // Retry failed because of a concurrent modification. throw; } catch (SpannerException e) when(e.ErrorCode == ErrorCode.Aborted) { // Ignore and retry. retryDelay = e.RecommendedRetryDelay ?? _options.Jitter(_options.NextDelay(retryDelay)); } } }
/// <summary> /// Returns true if an AggregateException contains a SpannerException /// with the given error code. /// </summary> /// <param name="e">The exception to examine.</param> /// <param name="errorCode">The error code to look for.</param> /// <returns></returns> static bool ContainsError(AggregateException e, ErrorCode errorCode) { foreach (var innerException in e.InnerExceptions) { SpannerException spannerException = innerException as SpannerException; if (spannerException != null && spannerException.ErrorCode == errorCode) { return(true); } } return(false); }
async Task IRetriableStatement.RetryAsync(SpannerRetriableTransaction transaction, CancellationToken cancellationToken, int timeoutSeconds) { _spannerCommand.Transaction = transaction.SpannerTransaction; var reader = await _spannerCommand.ExecuteReaderAsync(cancellationToken); int counter = 0; bool read = true; byte[] newChecksum = new byte[0]; SpannerException newException = null; while (read && counter < _numberOfReadCalls) { try { read = await reader.ReadAsync(cancellationToken); newChecksum = CalculateNextChecksum(reader, newChecksum, read); counter++; } catch (SpannerException e) when(e.ErrorCode == ErrorCode.Aborted) { // Propagate Aborted errors to trigger a new retry. throw; } catch (SpannerException e) { newException = e; counter++; break; } } if (counter == _numberOfReadCalls && newChecksum.SequenceEqual(_currentChecksum) && SpannerRetriableTransaction.SpannerExceptionsEqualForRetry(newException, _firstException)) { // Checksum is ok, we only need to replace the delegate result set if it's still open. if (IsClosed) { reader.Close(); } else { _spannerDataReader = reader; } } else { // The results are not equal, there is an actual concurrent modification, so we cannot // continue the transaction. throw new SpannerAbortedDueToConcurrentModificationException(); } }
/// <summary> /// Returns true if an AggregateException contains a SpannerException /// with the given error code. /// </summary> /// <param name="e">The exception to examine.</param> /// <param name="errorCode">The error code to look for.</param> /// <returns></returns> public static bool ContainsError(AggregateException e, params ErrorCode[] errorCode) { foreach (var innerException in e.InnerExceptions) { SpannerException spannerException = innerException as SpannerException; if (spannerException != null && errorCode.Contains(spannerException.ErrorCode)) { return(true); } } return(false); }
public async Task AbortedThrownCorrectly() { await WriteSampleRowsAsync(); // connection 1 starts a transaction and reads // connection 2 starts a transaction and reads the same row // connection 1 writes and commits // connection 2 reads again -- abort should be thrown. var connection1 = new SpannerConnection(_testFixture.ConnectionString); var connection2 = new SpannerConnection(_testFixture.ConnectionString); await Task.WhenAll(connection1.OpenAsync(), connection2.OpenAsync()); var tx1 = await connection1.BeginTransactionAsync(); // TX1 READ using (var cmd = connection1.CreateSelectCommand( "SELECT * FROM TX WHERE K=@k", new SpannerParameterCollection { { "k", _key, SpannerDbType.String } })) { cmd.Transaction = tx1; using (var reader = await cmd.ExecuteReaderAsync()) { Assert.True(await reader.ReadAsync()); } } // TX2 START var tx2 = await connection2.BeginTransactionAsync(); // TX2 READ using (var cmd = connection2.CreateSelectCommand( "SELECT * FROM TX WHERE K=@k", new SpannerParameterCollection { { "k", _key, SpannerDbType.String } })) { cmd.Transaction = tx2; using (var reader = await cmd.ExecuteReaderAsync()) { Assert.True(await reader.ReadAsync()); } } // TX1 WRITE/COMMIT using (var cmd = connection1.CreateUpdateCommand( "TX", new SpannerParameterCollection { { "k", _key, SpannerDbType.String }, { "Int64Value", 0, SpannerDbType.Int64 } })) { cmd.Transaction = tx1; await cmd.ExecuteNonQueryAsync().ConfigureAwait(false); await tx1.CommitAsync().ConfigureAwait(false); tx1.Dispose(); } connection1.Dispose(); // TX2 READ AGAIN/THROWS SpannerException thrownException = null; try { using (var cmd = connection2.CreateSelectCommand( "SELECT * FROM TX WHERE K=@k", new SpannerParameterCollection { { "k", _key, SpannerDbType.String } })) { cmd.Transaction = tx2; using (var reader = await cmd.ExecuteReaderAsync()) { Assert.True(await reader.ReadAsync()); } } } catch (SpannerException ex) { thrownException = ex; } finally { tx2.Dispose(); connection2.Dispose(); } Assert.True(thrownException?.IsRetryable ?? false); }
internal void Retry(SpannerException abortedException, int timeoutSeconds = MAX_TIMEOUT_SECONDS) => RetryAsync(abortedException, CancellationToken.None, timeoutSeconds).WaitWithUnwrappedExceptions();
internal FailedBatchDmlStatement(SpannerRetriableBatchCommand command, SpannerException exception) { _command = command; _exception = exception; }
internal FailedDmlStatement(SpannerCommand command, SpannerException exception) { _command = (SpannerCommand)command.Clone(); _exception = exception; }