public async Task TestTimeoutEvenWhenServerHangs() { await using var harness = await new MiniKuduClusterBuilder() .AddTabletServerFlag("--scanner_inject_latency_on_each_batch_ms=200000") .BuildHarnessAsync(); await using var client = harness.CreateClient(); var tableBuilder = ClientTestUtil.GetBasicSchema() .SetTableName(nameof(TestTimeoutEvenWhenServerHangs)); var table = await client.CreateTableAsync(tableBuilder); var row = ClientTestUtil.CreateBasicSchemaInsert(table, 1); await client.WriteAsync(new[] { row }); var scanner = client.NewScanBuilder(table).Build(); // Scan with a short timeout. var timeout = TimeSpan.FromSeconds(1); using var cts = new CancellationTokenSource(timeout); // The server will not respond for the lifetime of the test, so we // expect the operation to time out. await Assert.ThrowsAsync <OperationCanceledException>( async() => await scanner.CountAsync(cts.Token)); }
public async Task TestSubmitWriteOpAfterCommit() { await using var harness = await new MiniKuduClusterBuilder() .AddMasterServerFlag("--txn_manager_enabled") .AddTabletServerFlag("--enable_txn_system_client_init=true") .BuildHarnessAsync(); await using var client = harness.CreateClient(); var builder = ClientTestUtil.GetBasicSchema() .SetTableName(nameof(TestSubmitWriteOpAfterCommit)) .AddHashPartitions(2, "key"); var table = await client.CreateTableAsync(builder); using var transaction = await client.NewTransactionAsync(); int key = 0; var insert1 = ClientTestUtil.CreateBasicSchemaInsert(table, key++); await transaction.WriteAsync(new[] { insert1 }); await transaction.CommitAsync(); await transaction.WaitForCommitAsync(); var insert2 = ClientTestUtil.CreateBasicSchemaInsert(table, key); var exception = await Assert.ThrowsAsync <NonRecoverableException>( async() => await transaction.WriteAsync(new[] { insert2 })); Assert.Matches(".* transaction ID .* not open: COMMITTED", exception.Message); }
private async Task <long> SetupTableForDiffScansAsync(KuduTable table, int numRows) { for (int i = 0; i < numRows / 2; i++) { var row = ClientTestUtil.CreateBasicSchemaInsert(table, i); await _session.EnqueueAsync(row); } await _session.FlushAsync(); // Grab the timestamp, then add more data so there's a diff. long timestamp = _client.LastPropagatedTimestamp; for (int i = numRows / 2; i < numRows; i++) { var row = ClientTestUtil.CreateBasicSchemaInsert(table, i); await _session.EnqueueAsync(row); } await _session.FlushAsync(); // Delete some data so the is_deleted column can be tested. for (int i = 0; i < numRows / 4; i++) { var row = table.NewDelete(); row.SetInt32(0, i); await _session.EnqueueAsync(row); } await _session.FlushAsync(); return(timestamp); }
public async Task TestInsertAfterInsertIgnoreHasRowError() { var builder = ClientTestUtil.GetBasicSchema() .SetTableName(nameof(TestInsertAfterInsertIgnoreHasRowError)); var table = await _client.CreateTableAsync(builder); var rows = new[] { ClientTestUtil.CreateBasicSchemaInsertIgnore(table, 1), ClientTestUtil.CreateBasicSchemaInsert(table, 1) }; var exception = await Assert.ThrowsAsync <KuduWriteException>( () => _client.WriteAsync(rows)); var rowError = Assert.Single(exception.PerRowErrors); Assert.True(rowError.IsAlreadyPresent); var rowStrings = await ClientTestUtil.ScanTableToStringsAsync(_client, table); var rowString = Assert.Single(rowStrings); Assert.Equal( "INT32 key=1, INT32 column1_i=2, INT32 column2_i=3, " + "STRING column3_s=a string, BOOL column4_b=True", rowString); }
public async Task TestCount() { await using var miniCluster = await new MiniKuduClusterBuilder().BuildAsync(); await using var client = miniCluster.CreateClient(); var builder = ClientTestUtil.GetBasicSchema() .SetTableName(nameof(TestCount)) .AddHashPartitions(4, "key"); var table = await client.CreateTableAsync(builder); int numRows = 123; var rows = Enumerable.Range(0, numRows) .Select(i => ClientTestUtil.CreateBasicSchemaInsert(table, i)); await client.WriteAsync(rows); var scanner = client.NewScanBuilder(table) .SetEmptyProjection() .SetReadMode(ReadMode.ReadYourWrites) .Build(); long numScannedRows = await scanner.CountAsync(); Assert.Empty(scanner.ProjectionSchema.Columns); Assert.Equal(numRows, numScannedRows); }
public async Task TestUpdateIgnore() { var builder = ClientTestUtil.GetBasicSchema() .SetTableName(nameof(TestUpdateIgnore)); var table = await _client.CreateTableAsync(builder); // Test update ignore does not return a row error. var update1 = ClientTestUtil.CreateBasicSchemaUpdateIgnore(table, 1, 1, false); await _client.WriteAsync(new[] { update1 }); Assert.Empty(await ClientTestUtil.ScanTableToStringsAsync(_client, table)); var insert = ClientTestUtil.CreateBasicSchemaInsert(table, 1); await _client.WriteAsync(new[] { insert }); Assert.Single(await ClientTestUtil.ScanTableToStringsAsync(_client, table)); // Test update ignore implements normal update. var update2 = ClientTestUtil.CreateBasicSchemaUpdateIgnore(table, 1, 2, false); await _client.WriteAsync(new[] { update2 }); var rowStrings = await ClientTestUtil.ScanTableToStringsAsync(_client, table); var rowString = Assert.Single(rowStrings); Assert.Equal( "INT32 key=1, INT32 column1_i=2, INT32 column2_i=3, " + "STRING column3_s=a string, BOOL column4_b=True", rowString); }
private async Task WriteAsync(CancellationToken cancellationToken) { Exception sessionException = null; Exception exception; ValueTask HandleSessionExceptionAsync(SessionExceptionContext context) { Volatile.Write(ref sessionException, context.Exception); return(new ValueTask()); } var options = new KuduSessionOptions { ExceptionHandler = HandleSessionExceptionAsync }; await using var session = _client.NewSession(options); int currentRowKey = 0; bool flush = false; while (true) { if (cancellationToken.IsCancellationRequested) { break; } exception = Volatile.Read(ref sessionException); if (exception != null) { throw exception; } var row = ClientTestUtil.CreateBasicSchemaInsert(_table, currentRowKey); await session.EnqueueAsync(row, CancellationToken.None); if (flush) { await session.FlushAsync(CancellationToken.None); } // Every 10 rows we flush and change the flush mode randomly. if (currentRowKey % 10 == 0) { flush = ThreadSafeRandom.Instance.NextBool(); } currentRowKey++; } await session.FlushAsync(CancellationToken.None); exception = Volatile.Read(ref sessionException); if (exception != null) { throw exception; } }
public async Task TestTxnKeepaliveSwitchesToOtherTxnManager() { await using var harness = await new MiniKuduClusterBuilder() .AddMasterServerFlag("--txn_manager_enabled") // Set Raft heartbeat interval short for faster test runtime: speed up // leader failure detection and new leader election. .AddMasterServerFlag("--raft_heartbeat_interval_ms=100") // The txn keepalive interval should be long enough to accommodate Raft // leader failure detection and election. .AddTabletServerFlag("--txn_keepalive_interval_ms=1000") .AddTabletServerFlag("--txn_staleness_tracker_interval_ms=250") .AddTabletServerFlag("--enable_txn_system_client_init=true") .BuildHarnessAsync(); await using var client = harness.CreateClient(); var builder = ClientTestUtil.GetBasicSchema() .SetTableName(nameof(TestTxnKeepaliveSwitchesToOtherTxnManager)) .AddHashPartitions(2, "key"); var table = await client.CreateTableAsync(builder); using var transaction = await client.NewTransactionAsync(); var insert = ClientTestUtil.CreateBasicSchemaInsert(table, 0); await transaction.WriteAsync(new[] { insert }); await harness.KillLeaderMasterServerAsync(); // Wait for two keepalive intervals to make sure the backend got a chance // to automatically abort the transaction if not receiving txn keepalive // messages. await Task.Delay(2 * 1000); // It should be possible to commit the transaction. This is to verify that // // * the client eventually starts sending txn keepalive messages to other // TxnManager instance (the original was hosted by former leader master // which is no longer available), so the backend doesn't abort the // transaction automatically due to not receiving keepalive messages // // * the client switches to the new TxnManager for other txn-related // operations as well await transaction.CommitAsync(); await transaction.WaitForCommitAsync(); // An extra sanity check: read back the rows written into the table in the // context of the transaction. var scanner = client.NewScanBuilder(table) .SetReadMode(ReadMode.ReadYourWrites) .SetReplicaSelection(ReplicaSelection.LeaderOnly) .Build(); Assert.Equal(1, await scanner.CountAsync()); }
public async Task TestPropagateTxnCommitTimestamp() { await using var harness = await new MiniKuduClusterBuilder() .AddMasterServerFlag("--txn_manager_enabled") // Inject latency to have a chance spotting the transaction in the // FINALIZE_IN_PROGRESS state and make KuduTransaction.WaitForCommitAsync() // have to poll multiple times. .AddTabletServerFlag("--txn_status_manager_inject_latency_finalize_commit_ms=250") .AddTabletServerFlag("--enable_txn_system_client_init=true") .BuildHarnessAsync(); await using var client = harness.CreateClient(); var builder = ClientTestUtil.GetBasicSchema() .SetTableName(nameof(TestPropagateTxnCommitTimestamp)) .AddHashPartitions(8, "key"); var table = await client.CreateTableAsync(builder); // Make sure the commit timestamp for a transaction is propagated to the // client upon committing a transaction. using (var transaction = await client.NewTransactionAsync()) { // Insert many rows: the goal is to get at least one row inserted into // every tablet of the hash-partitioned test table, so every tablet would // be a participant in the transaction, and most likely every tablet // server would be involved. var inserts = Enumerable .Range(0, 128) .Select(key => ClientTestUtil.CreateBasicSchemaInsert(table, key)); await transaction.WriteAsync(inserts); await CommitAndVerifyTransactionAsync(client, transaction); } // Make sure the commit timestamp for a transaction is propagated to the // client upon committing a transaction (using a session). using (var transaction = await client.NewTransactionAsync()) { await using var session = transaction.NewSession(); // Insert many rows: the goal is to get at least one row inserted into // every tablet of the hash-partitioned test table, so every tablet would // be a participant in the transaction, and most likely every tablet // server would be involved. for (int key = 128; key < 256; key++) { var insert = ClientTestUtil.CreateBasicSchemaInsert(table, key); await session.EnqueueAsync(insert); } await session.FlushAsync(); await CommitAndVerifyTransactionAsync(client, transaction); } }
private static async ValueTask InsertRowsAsync( IKuduSession session, KuduTable table, int startRow, int numRows) { var end = startRow + numRows; for (var i = startRow; i < end; i++) { var insert = ClientTestUtil.CreateBasicSchemaInsert(table, i); await session.EnqueueAsync(insert); } }
public async Task TestGetTableStatistics() { await using var harness = await new MiniKuduClusterBuilder() .AddTabletServerFlag("--update_tablet_stats_interval_ms=200") .AddTabletServerFlag("--heartbeat_interval_ms=100") .BuildHarnessAsync(); await using var client = harness.CreateClient(); // Create a table. var builder = ClientTestUtil.GetBasicSchema().SetTableName(_tableName); var table = await client.CreateTableAsync(builder); // Insert some rows and test the statistics. var prevStatistics = new KuduTableStatistics(-1, -1); var currentStatistics = new KuduTableStatistics(-1, -1); var session = client.NewSession(); int num = 100; for (int i = 0; i < num; ++i) { // Get current table statistics. currentStatistics = await client.GetTableStatisticsAsync(_tableName); Assert.True(currentStatistics.OnDiskSize >= prevStatistics.OnDiskSize); Assert.True(currentStatistics.LiveRowCount >= prevStatistics.LiveRowCount); Assert.True(currentStatistics.LiveRowCount <= i + 1); prevStatistics = currentStatistics; // Insert row. var insert = ClientTestUtil.CreateBasicSchemaInsert(table, i); await session.EnqueueAsync(insert); await session.FlushAsync(); long numRows = await ClientTestUtil.CountRowsAsync(client, table); Assert.Equal(i + 1, numRows); } // Final accuracy test. // Wait for master to aggregate table statistics. await Task.Delay(200 * 6); currentStatistics = await client.GetTableStatisticsAsync(_tableName); Assert.True(currentStatistics.OnDiskSize >= prevStatistics.OnDiskSize); Assert.True(currentStatistics.LiveRowCount >= prevStatistics.LiveRowCount); Assert.Equal(num, currentStatistics.LiveRowCount); }
public async Task TestSwitchToOtherTxnManagerInFlightCalls() { await using var harness = await new MiniKuduClusterBuilder() .AddMasterServerFlag("--txn_manager_enabled") // Set Raft heartbeat interval short for faster test runtime: speed up // leader failure detection and new leader election. .AddMasterServerFlag("--raft_heartbeat_interval_ms=100") .AddTabletServerFlag("--enable_txn_system_client_init=true") .BuildHarnessAsync(); await using var client = harness.CreateClient(); var builder = ClientTestUtil.GetBasicSchema() .SetTableName(nameof(TestSwitchToOtherTxnManagerInFlightCalls)) .AddHashPartitions(2, "key"); var table = await client.CreateTableAsync(builder); using var transaction = await client.NewTransactionAsync(); var insert = ClientTestUtil.CreateBasicSchemaInsert(table, 0); await transaction.WriteAsync(new[] { insert }); await harness.KillAllMasterServersAsync(); var startMastersTask = Task.Run(async() => { // Sleep for some time to allow the commit call // below issue RPCs to non-running TxnManangers. await Task.Delay(1000); await harness.StartAllMasterServersAsync(); }); // It should be possible to commit the transaction. await transaction.CommitAsync(); await transaction.WaitForCommitAsync(); await startMastersTask; // An extra sanity check: read back the rows written into the table in the // context of the transaction. var scanner = client.NewScanBuilder(table) .SetReadMode(ReadMode.ReadYourWrites) .SetReplicaSelection(ReplicaSelection.LeaderOnly) .Build(); Assert.Equal(1, await scanner.CountAsync()); }
public async Task TestExceptionCallback() { int numCallbacks = 0; SessionExceptionContext sessionContext = null; var builder = ClientTestUtil.GetBasicSchema() .SetTableName(nameof(TestExceptionCallback)); var table = await _client.CreateTableAsync(builder); var row1 = ClientTestUtil.CreateBasicSchemaInsert(table, 1); var row2 = ClientTestUtil.CreateBasicSchemaInsert(table, 1); var sessionOptions = new KuduSessionOptions { ExceptionHandler = HandleSessionExceptionAsync }; await using var session = _client.NewSession(sessionOptions); await session.EnqueueAsync(row1); await session.FlushAsync(); await session.EnqueueAsync(row2); await session.FlushAsync(); ValueTask HandleSessionExceptionAsync(SessionExceptionContext context) { numCallbacks++; sessionContext = context; return(new ValueTask()); } Assert.Equal(1, numCallbacks); var errorRow = Assert.Single(sessionContext.Rows); Assert.Same(row2, errorRow); var exception = Assert.IsType <KuduWriteException>(sessionContext.Exception); var exceptionRow = Assert.Single(exception.PerRowErrors); Assert.True(exceptionRow.IsAlreadyPresent); }
public async Task TestScannerExpiration() { await using var miniCluster = await new MiniKuduClusterBuilder() .AddTabletServerFlag($"--scanner_ttl_ms={ShortScannerTtlMs}") .AddTabletServerFlag($"--scanner_gc_check_interval_us={ShortScannerGcUs}") .BuildAsync(); await using var client = miniCluster.CreateClient(); var builder = ClientTestUtil.GetBasicSchema() .SetTableName("TestScannerExpiration") .AddHashPartitions(2, "key"); var table = await client.CreateTableAsync(builder); int numRows = 1000; var rows = Enumerable.Range(0, numRows) .Select(i => ClientTestUtil.CreateBasicSchemaInsert(table, i)); await client.WriteAsync(rows); var scanner = client.NewScanBuilder(table) .SetReadMode(ReadMode.ReadYourWrites) .SetReplicaSelection(ReplicaSelection.ClosestReplica) .SetBatchSizeBytes(100) // Use a small batch size so we get many batches. .Build(); var scanEnumerator = scanner.GetAsyncEnumerator(); // Initialize the scanner and verify we can read rows. Assert.True(await scanEnumerator.MoveNextAsync()); Assert.True(scanEnumerator.Current.Count > 0); // Wait for the scanner to time out. await Task.Delay(ShortScannerTtlMs * 2); var exception = await Assert.ThrowsAsync <NonRecoverableException>(async() => await scanEnumerator.MoveNextAsync()); Assert.Matches(".*Scanner .* not found.*", exception.Message); // Closing an expired scanner shouldn't throw an exception. await scanEnumerator.DisposeAsync(); }
public async Task TestFailover(bool restart) { await using var harness = await new MiniKuduClusterBuilder() .NumMasters(3) .NumTservers(3) .BuildHarnessAsync(); await using var client = harness.CreateClient(); var builder = ClientTestUtil.GetBasicSchema() .SetTableName("LeaderFailoverTest") .CreateBasicRangePartition(); var table = await client.CreateTableAsync(builder); var rows = Enumerable.Range(0, 3) .Select(i => ClientTestUtil.CreateBasicSchemaInsert(table, i)); await client.WriteAsync(rows); // Make sure the rows are in there before messing things up. long numRows = await ClientTestUtil.CountRowsAsync(client, table); Assert.Equal(3, numRows); if (restart) { await harness.RestartLeaderMasterAsync(); } else { await harness.KillLeaderMasterServerAsync(); } var rows2 = Enumerable.Range(3, 3) .Select(i => ClientTestUtil.CreateBasicSchemaInsert(table, i)); await client.WriteAsync(rows2); long numRows2 = await ClientTestUtil.CountRowsAsync(client, table); Assert.Equal(6, numRows2); }
public async Task TestTxnSessionClose() { await using var harness = await new MiniKuduClusterBuilder() .AddMasterServerFlag("--txn_manager_enabled") .AddTabletServerFlag("--enable_txn_system_client_init=true") .BuildHarnessAsync(); await using var client = harness.CreateClient(); var builder = ClientTestUtil.GetBasicSchema() .SetTableName(nameof(TestTxnSessionClose)) .AddHashPartitions(2, "key"); var table = await client.CreateTableAsync(builder); // Open and close an empty transaction session. using (var transaction = await client.NewTransactionAsync()) { await using var session = transaction.NewSession(); } // Open new transaction, insert one row for a session, close the session // and then rollback the transaction. No rows should be persisted. using (var transaction = await client.NewTransactionAsync()) { await using var session = transaction.NewSession(); var insert = ClientTestUtil.CreateBasicSchemaInsert(table, 1); await session.EnqueueAsync(insert); await session.FlushAsync(); await transaction.RollbackAsync(); var scanner = client.NewScanBuilder(table) .SetReadMode(ReadMode.ReadYourWrites) .SetReplicaSelection(ReplicaSelection.LeaderOnly) .Build(); Assert.Equal(0, await scanner.CountAsync()); } }
public async Task TestDeleteIgnore() { var builder = ClientTestUtil.GetBasicSchema() .SetTableName(nameof(TestDeleteIgnore)); var table = await _client.CreateTableAsync(builder); // Test delete ignore does not return a row error. var delete = ClientTestUtil.CreateBasicSchemaDeleteIgnore(table, 1); await _client.WriteAsync(new[] { delete }); var insert = ClientTestUtil.CreateBasicSchemaInsert(table, 1); await _client.WriteAsync(new[] { insert }); Assert.Single(await ClientTestUtil.ScanTableToStringsAsync(_client, table)); // Test delete ignore implements normal delete. await _client.WriteAsync(new[] { delete }); Assert.Empty(await ClientTestUtil.ScanTableToStringsAsync(_client, table)); }
public async Task TestInsertIgnoreAfterInsertHasNoRowError() { var builder = ClientTestUtil.GetBasicSchema() .SetTableName(nameof(TestInsertIgnoreAfterInsertHasNoRowError)); var table = await _client.CreateTableAsync(builder); var rows = new[] { ClientTestUtil.CreateBasicSchemaInsert(table, 1), ClientTestUtil.CreateBasicSchemaUpsert(table, 1, 1, false), ClientTestUtil.CreateBasicSchemaInsertIgnore(table, 1) }; await _client.WriteAsync(rows); var rowStrings = await ClientTestUtil.ScanTableToStringsAsync(_client, table); var rowString = Assert.Single(rowStrings); Assert.Equal( "INT32 key=1, INT32 column1_i=1, INT32 column2_i=3, " + "STRING column3_s=a string, BOOL column4_b=True", rowString); }
public async Task TestKeepAlive() { await using var miniCluster = await new MiniKuduClusterBuilder() .AddTabletServerFlag($"--scanner_ttl_ms={ShortScannerTtlMs}") .AddTabletServerFlag($"--scanner_gc_check_interval_us={ShortScannerGcUs}") .BuildAsync(); await using var client = miniCluster.CreateClient(); var builder = ClientTestUtil.GetBasicSchema() .SetTableName("TestKeepAlive") .AddHashPartitions(2, "key"); var table = await client.CreateTableAsync(builder); int numRows = 1000; var rows = Enumerable.Range(0, numRows) .Select(i => ClientTestUtil.CreateBasicSchemaInsert(table, i)); await client.WriteAsync(rows); var scanner = client.NewScanBuilder(table) .SetReadMode(ReadMode.ReadYourWrites) .SetReplicaSelection(ReplicaSelection.LeaderOnly) .SetBatchSizeBytes(100) // Use a small batch size so we get many batches. .Build(); await using var scanEnumerator = scanner.GetAsyncEnumerator(); // KeepAlive on uninitialized scanner should be ok. await scanEnumerator.KeepAliveAsync(); // Get the first batch and initialize the scanner. Assert.True(await scanEnumerator.MoveNextAsync()); long accum = scanEnumerator.Current.Count; while (await scanEnumerator.MoveNextAsync()) { accum += scanEnumerator.Current.Count; // Break when we are between tablets. if (scanEnumerator.Tablet is null) { break; } // Ensure we actually end up between tablets. Assert.NotEqual(numRows, accum); } // In between scanners now and should be ok. await scanEnumerator.KeepAliveAsync(); // Initialize the next scanner or keepAlive will have no effect. Assert.True(await scanEnumerator.MoveNextAsync()); accum += scanEnumerator.Current.Count; // Wait for longer than the scanner ttl calling keepAlive throughout. // Each loop sleeps 20% of the scanner ttl and we loop 12 times to ensure // we extend over 2x the scanner ttl. for (int i = 0; i < 12; i++) { await Task.Delay(ShortScannerTtlMs / 5); await scanEnumerator.KeepAliveAsync(); } // Finish out the rows. while (await scanEnumerator.MoveNextAsync()) { accum += scanEnumerator.Current.Count; } Assert.Equal(numRows, accum); // At this point the scanner is closed and there is nothing to keep alive. var exception = await Assert.ThrowsAsync <Exception>(async() => await scanEnumerator.KeepAliveAsync()); Assert.Contains("Scanner has already been closed", exception.Message); }
public async Task TestScanTokensInterleavedRangePartitionDrops() { int numRows = 30; var builder = ClientTestUtil.GetBasicSchema() .SetTableName(_tableName) .AddHashPartitions(2, "key") .CreateBasicRangePartition() .AddRangePartition((lower, upper) => { lower.SetInt32("key", 0); upper.SetInt32("key", numRows / 3); }) .AddRangePartition((lower, upper) => { lower.SetInt32("key", numRows / 3); upper.SetInt32("key", 2 * numRows / 3); }) .AddRangePartition((lower, upper) => { lower.SetInt32("key", 2 * numRows / 3); upper.SetInt32("key", numRows); }); var table = await _client.CreateTableAsync(builder); for (int i = 0; i < numRows; i++) { var row = ClientTestUtil.CreateBasicSchemaInsert(table, i); await _session.EnqueueAsync(row); } await _session.FlushAsync(); // Build the scan tokens. List <KuduScanToken> tokens = await _client.NewScanTokenBuilder(table) .BuildAsync(); Assert.Equal(6, tokens.Count); // Drop the range partition [10, 20). await _client.AlterTableAsync(new AlterTableBuilder(table) .DropRangePartition((lower, upper) => { lower.SetInt32("key", numRows / 3); upper.SetInt32("key", 2 * numRows / 3); })); // Rehydrate the tokens. var scanners = new List <KuduScanner>(); foreach (var token in tokens) { var scanBuilder = await _client.NewScanBuilderFromTokenAsync(token); var scanner = scanBuilder.Build(); scanners.Add(scanner); } // Drop the range partition [20, 30). await _client.AlterTableAsync(new AlterTableBuilder(table) .DropRangePartition((lower, upper) => { lower.SetInt32("key", 2 * numRows / 3); upper.SetInt32("key", numRows); })); // Check the scanners work. The scanners for the tablets in the range // [10, 20) definitely won't see any rows. The scanners for the tablets // in the range [20, 30) might see rows. long scannedRows = 0; foreach (var scanner in scanners) { scannedRows += await scanner.CountAsync(); } Assert.True(scannedRows >= numRows / 3); Assert.True(scannedRows <= 2 * numRows / 3); }
public async Task TestMultipleFailover(bool restart) { int rowsPerIteration = 3; int numIterations = 10; int totalRowsToInsert = rowsPerIteration + numIterations * rowsPerIteration; await using var harness = await new MiniKuduClusterBuilder() .NumMasters(3) .NumTservers(3) .BuildHarnessAsync(); await using var client = harness.CreateClient(); var builder = ClientTestUtil.GetBasicSchema() .SetTableName("MultipleLeaderFailoverTest"); var table = await client.CreateTableAsync(builder); await using var session = client.NewSession(); for (int i = 0; i < rowsPerIteration; i++) { var row = ClientTestUtil.CreateBasicSchemaInsert(table, i); await session.EnqueueAsync(row); } await session.FlushAsync(); var rowCount = await ClientTestUtil.CountRowsAsync(client, table); Assert.Equal(rowsPerIteration, rowCount); int currentRows = rowsPerIteration; for (int i = 0; i < numIterations; i++) { var tablets = await client.GetTableLocationsAsync( table.TableId, Array.Empty <byte>(), 1); Assert.Single(tablets); if (restart) { await harness.RestartTabletServerAsync(tablets[0]); } else { await harness.KillTabletLeaderAsync(tablets[0]); } for (int j = 0; j < rowsPerIteration; j++) { var row = ClientTestUtil.CreateBasicSchemaInsert(table, currentRows); await session.EnqueueAsync(row); currentRows++; } await session.FlushAsync(); if (!restart) { await harness.StartAllTabletServersAsync(); } rowCount = await ClientTestUtil.CountRowsAsync(client, table); Assert.Equal(currentRows, rowCount); } rowCount = await ClientTestUtil.CountRowsAsync(client, table); Assert.Equal(totalRowsToInsert, rowCount); }
public async Task TestTxnKeepaliveRollingSwitchToOtherTxnManager() { await using var harness = await new MiniKuduClusterBuilder() .AddMasterServerFlag("--txn_manager_enabled") // Set Raft heartbeat interval short for faster test runtime: speed up // leader failure detection and new leader election. .AddMasterServerFlag("--raft_heartbeat_interval_ms=100") // The txn keepalive interval should be long enough to accommodate Raft // leader failure detection and election. .AddTabletServerFlag("--txn_keepalive_interval_ms=1000") .AddTabletServerFlag("--txn_staleness_tracker_interval_ms=250") .AddTabletServerFlag("--enable_txn_system_client_init=true") .BuildHarnessAsync(); await using var client = harness.CreateClient(); var builder = ClientTestUtil.GetBasicSchema() .SetTableName(nameof(TestTxnKeepaliveRollingSwitchToOtherTxnManager)) .AddHashPartitions(2, "key"); var table = await client.CreateTableAsync(builder); using var transaction = await client.NewTransactionAsync(); // Cycle the leadership among masters, making sure the client successfully // switches to every newly elected leader master to send keepalive messages. int numMasters = harness.GetMasterServers().Count; for (int i = 0; i < numMasters; i++) { // Shutdown the leader master. var hostPort = await harness.KillLeaderMasterServerAsync(); // Wait for two keepalive intervals to give the backend a chance // to automatically abort the transaction if not receiving txn keepalive // messages. await Task.Delay(2 * 1000); // The transaction should be still alive. var exception = await Assert.ThrowsAsync <NonRecoverableException>( async() => await transaction.WaitForCommitAsync()); Assert.True(exception.Status.IsIllegalState); Assert.Equal("transaction is still open", exception.Message); // In addition, it should be possible to insert rows in the context // of the transaction. var insert = ClientTestUtil.CreateBasicSchemaInsert(table, i); await transaction.WriteAsync(new[] { insert }); // Start the master back. await harness.StartMasterAsync(hostPort); } // Make sure the client properly processes error responses sent back by // TxnManager when the TxnStatusManager isn't available. So, shutdown all // tablet servers: this is to make sure TxnStatusManager isn't there. await harness.KillAllTabletServersAsync(); var startTabletServersTask = Task.Run(async() => { // Sleep for some time to allow the commit call below issue RPCs when // TxnStatusManager is not yet around. await Task.Delay(2 * 1000); // Start all the tablet servers back so the TxnStatusManager is back. await harness.StartAllTabletServersAsync(); }); // The transaction should be still alive, and it should be possible to // commit it. await transaction.CommitAsync(); await transaction.WaitForCommitAsync(); await startTabletServersTask; // An extra sanity check: read back the rows written into the table in the // context of the transaction. var scanner = client.NewScanBuilder(table) .SetReadMode(ReadMode.ReadYourWrites) .SetReplicaSelection(ReplicaSelection.LeaderOnly) .Build(); Assert.Equal(numMasters, await scanner.CountAsync()); }
public async Task TestSwitchToOtherTxnManager() { await using var harness = await new MiniKuduClusterBuilder() .AddMasterServerFlag("--txn_manager_enabled") // Set Raft heartbeat interval short for faster test runtime: speed up // leader failure detection and new leader election. .AddMasterServerFlag("--raft_heartbeat_interval_ms=100") .AddTabletServerFlag("--enable_txn_system_client_init=true") .BuildHarnessAsync(); await using var client = harness.CreateClient(); var builder = ClientTestUtil.GetBasicSchema() .SetTableName(nameof(TestSwitchToOtherTxnManager)) .AddHashPartitions(2, "key"); var table = await client.CreateTableAsync(builder); // Start a transaction, then restart every available TxnManager instance // before attempting any txn-related operation. using (var transaction = await client.NewTransactionAsync()) { var insert = ClientTestUtil.CreateBasicSchemaInsert(table, 0); await transaction.WriteAsync(new[] { insert }); await harness.KillAllMasterServersAsync(); await harness.StartAllMasterServersAsync(); // Querying the status of a transaction should be possible, as usual. // Since the transaction is still open, KuduTransaction.WaitForCommitAsync() // should throw corresponding exception. var exception = await Assert.ThrowsAsync <NonRecoverableException>( async() => await transaction.WaitForCommitAsync()); Assert.True(exception.Status.IsIllegalState); Assert.Equal("transaction is still open", exception.Message); await harness.KillAllMasterServersAsync(); await harness.StartAllMasterServersAsync(); // It should be possible to commit the transaction. await transaction.CommitAsync(); await transaction.WaitForCommitAsync(); // An extra sanity check: read back the rows written into the table in the // context of the transaction. var scanner = client.NewScanBuilder(table) .SetReadMode(ReadMode.ReadYourWrites) .SetReplicaSelection(ReplicaSelection.LeaderOnly) .Build(); Assert.Equal(1, await scanner.CountAsync()); } // Similar to the above, but run KuduTransaction.commit() when only 2 out // of 3 masters are running while the TxnManager which used to start the // transaction is no longer around. using (var transaction = await client.NewTransactionAsync()) { var insert = ClientTestUtil.CreateBasicSchemaInsert(table, 1); await transaction.WriteAsync(new[] { insert }); await harness.KillLeaderMasterServerAsync(); // It should be possible to commit the transaction: 2 out of 3 masters are // running and Raft should be able to establish a leader master. So, // txn-related operations routed through TxnManager should succeed. await transaction.CommitAsync(); await transaction.WaitForCommitAsync(); // An extra sanity check: read back the rows written into the table in the // context of the transaction. var scanner = client.NewScanBuilder(table) .SetReadMode(ReadMode.ReadYourWrites) .SetReplicaSelection(ReplicaSelection.LeaderOnly) .Build(); // 1 row should be there from earlier, plus the one we just committed. Assert.Equal(2, await scanner.CountAsync()); } }