public async Task TestInsertIgnore() { var builder = ClientTestUtil.GetBasicSchema() .SetTableName(nameof(TestInsertIgnore)); var table = await _client.CreateTableAsync(builder); // Test insert ignore implements normal insert. var insert = ClientTestUtil.CreateBasicSchemaInsertIgnore(table, 1); await _client.WriteAsync(new[] { insert }); var rowStrings = await ClientTestUtil.ScanTableToStringsAsync(_client, table); var rowString = Assert.Single(rowStrings); Assert.Equal( "INT32 key=1, INT32 column1_i=2, INT32 column2_i=3, " + "STRING column3_s=a string, BOOL column4_b=True", rowString); // Test insert ignore does not return a row error. await _client.WriteAsync(new[] { insert }); rowStrings = await ClientTestUtil.ScanTableToStringsAsync(_client, table); rowString = Assert.Single(rowStrings); Assert.Equal( "INT32 key=1, INT32 column1_i=2, INT32 column2_i=3, " + "STRING column3_s=a string, BOOL column4_b=True", rowString); }
public async Task TestSubmitWriteOpAfterCommit() { await using var harness = await new MiniKuduClusterBuilder() .AddMasterServerFlag("--txn_manager_enabled") .AddTabletServerFlag("--enable_txn_system_client_init=true") .BuildHarnessAsync(); await using var client = harness.CreateClient(); var builder = ClientTestUtil.GetBasicSchema() .SetTableName(nameof(TestSubmitWriteOpAfterCommit)) .AddHashPartitions(2, "key"); var table = await client.CreateTableAsync(builder); using var transaction = await client.NewTransactionAsync(); int key = 0; var insert1 = ClientTestUtil.CreateBasicSchemaInsert(table, key++); await transaction.WriteAsync(new[] { insert1 }); await transaction.CommitAsync(); await transaction.WaitForCommitAsync(); var insert2 = ClientTestUtil.CreateBasicSchemaInsert(table, key); var exception = await Assert.ThrowsAsync <NonRecoverableException>( async() => await transaction.WriteAsync(new[] { insert2 })); Assert.Matches(".* transaction ID .* not open: COMMITTED", exception.Message); }
public async Task TestOpenScanWithDroppedPartition() { await using var miniCluster = await new MiniKuduClusterBuilder().BuildAsync(); await using var client = miniCluster.CreateClient(); var builder = ClientTestUtil.GetBasicSchema() .SetTableName("TestOpenScanWithDroppedPartition") .CreateBasicRangePartition() .AddRangePartition((lower, upper) => { lower.SetInt32("key", 0); upper.SetInt32("key", 1000); }) .AddRangePartition((lower, upper) => { lower.SetInt32("key", 1000); upper.SetInt32("key", 2000); }); var table = await client.CreateTableAsync(builder); // Load rows into both partitions. int numRows = 1999; await ClientTestUtil.LoadDefaultTableAsync(client, table, numRows); // Scan the rows while dropping a partition. var scanner = client.NewScanBuilder(table) // Set a small batch size so the first scan doesn't read all the rows. .SetBatchSizeBytes(100) .SetReadMode(ReadMode.ReadYourWrites) .Build(); long rowsScanned = 0; int batchNum = 0; await foreach (var resultSet in scanner) { if (batchNum == 1) { // Drop the partition. await client.AlterTableAsync(new AlterTableBuilder(table) .DropRangePartition((lower, upper) => { lower.SetInt32("key", 0); upper.SetInt32("key", 1000); })); // Give time for the background drop operations. await Task.Delay(1000); // TODO: Verify the partition was dropped. } rowsScanned += resultSet.Count; batchNum++; } Assert.True(batchNum > 1); Assert.Equal(numRows, rowsScanned); }
public async Task TestTimeoutEvenWhenServerHangs() { await using var harness = await new MiniKuduClusterBuilder() .AddTabletServerFlag("--scanner_inject_latency_on_each_batch_ms=200000") .BuildHarnessAsync(); await using var client = harness.CreateClient(); var tableBuilder = ClientTestUtil.GetBasicSchema() .SetTableName(nameof(TestTimeoutEvenWhenServerHangs)); var table = await client.CreateTableAsync(tableBuilder); var row = ClientTestUtil.CreateBasicSchemaInsert(table, 1); await client.WriteAsync(new[] { row }); var scanner = client.NewScanBuilder(table).Build(); // Scan with a short timeout. var timeout = TimeSpan.FromSeconds(1); using var cts = new CancellationTokenSource(timeout); // The server will not respond for the lifetime of the test, so we // expect the operation to time out. await Assert.ThrowsAsync <OperationCanceledException>( async() => await scanner.CountAsync(cts.Token)); }
public async Task TestDiffScanTokensConcurrentColumnRename() { var builder = ClientTestUtil.GetBasicSchema() .SetTableName(_tableName) .SetNumReplicas(1); var table = await _client.CreateTableAsync(builder); // Set up the table for a diff scan. int numRows = 20; long timestamp = await SetupTableForDiffScansAsync(table, numRows); // Since the diff scan interval is [start, end), increment the start timestamp // to exclude the last row inserted in the first group of ops, and increment the // end timestamp to include the last row deleted in the second group of ops. List <KuduScanToken> tokens = await _client.NewScanTokenBuilder(table) // TODO(KUDU-3146): Disable including the table metadata so the new column // name is retrieved when deserializing the scanner. .IncludeTableMetadata(false) .DiffScan(timestamp + 1, _client.LastPropagatedTimestamp + 1) .BuildAsync(); var token = Assert.Single(tokens); // Rename a column between when the token is created and when it is // rehydrated into a scanner await _client.AlterTableAsync(new AlterTableBuilder(table) .RenameColumn("column1_i", "column1_i_new")); var scanBuilder = await _client.NewScanBuilderFromTokenAsync(token); var scanner = scanBuilder.Build(); await CheckDiffScanResultsAsync(scanner, 3 *numRows / 4, numRows / 4); }
public async Task TestInsertAfterInsertIgnoreHasRowError() { var builder = ClientTestUtil.GetBasicSchema() .SetTableName(nameof(TestInsertAfterInsertIgnoreHasRowError)); var table = await _client.CreateTableAsync(builder); var rows = new[] { ClientTestUtil.CreateBasicSchemaInsertIgnore(table, 1), ClientTestUtil.CreateBasicSchemaInsert(table, 1) }; var exception = await Assert.ThrowsAsync <KuduWriteException>( () => _client.WriteAsync(rows)); var rowError = Assert.Single(exception.PerRowErrors); Assert.True(rowError.IsAlreadyPresent); var rowStrings = await ClientTestUtil.ScanTableToStringsAsync(_client, table); var rowString = Assert.Single(rowStrings); Assert.Equal( "INT32 key=1, INT32 column1_i=2, INT32 column2_i=3, " + "STRING column3_s=a string, BOOL column4_b=True", rowString); }
private async Task <long> SetupTableForDiffScansAsync(KuduTable table, int numRows) { for (int i = 0; i < numRows / 2; i++) { var row = ClientTestUtil.CreateBasicSchemaInsert(table, i); await _session.EnqueueAsync(row); } await _session.FlushAsync(); // Grab the timestamp, then add more data so there's a diff. long timestamp = _client.LastPropagatedTimestamp; for (int i = numRows / 2; i < numRows; i++) { var row = ClientTestUtil.CreateBasicSchemaInsert(table, i); await _session.EnqueueAsync(row); } await _session.FlushAsync(); // Delete some data so the is_deleted column can be tested. for (int i = 0; i < numRows / 4; i++) { var row = table.NewDelete(); row.SetInt32(0, i); await _session.EnqueueAsync(row); } await _session.FlushAsync(); return(timestamp); }
public async Task TestCount() { await using var miniCluster = await new MiniKuduClusterBuilder().BuildAsync(); await using var client = miniCluster.CreateClient(); var builder = ClientTestUtil.GetBasicSchema() .SetTableName(nameof(TestCount)) .AddHashPartitions(4, "key"); var table = await client.CreateTableAsync(builder); int numRows = 123; var rows = Enumerable.Range(0, numRows) .Select(i => ClientTestUtil.CreateBasicSchemaInsert(table, i)); await client.WriteAsync(rows); var scanner = client.NewScanBuilder(table) .SetEmptyProjection() .SetReadMode(ReadMode.ReadYourWrites) .Build(); long numScannedRows = await scanner.CountAsync(); Assert.Empty(scanner.ProjectionSchema.Columns); Assert.Equal(numRows, numScannedRows); }
public async Task InitializeAsync() { _harness = await new MiniKuduClusterBuilder().BuildHarnessAsync(); var options = _harness.CreateClientBuilder().BuildOptions(); var securityContext = new SecurityContext(); var systemClock = new SystemClock(); var connectionFactory = new KuduConnectionFactory( options, securityContext, NullLoggerFactory.Instance); _testConnectionFactory = new TestConnectionFactory(connectionFactory); _client = new KuduClient( options, securityContext, _testConnectionFactory, systemClock, NullLoggerFactory.Instance); var builder = ClientTestUtil.GetBasicSchema() .SetTableName("chaos_test_table") .SetNumReplicas(3) .CreateBasicRangePartition(); _table = await _client.CreateTableAsync(builder); }
public async Task TestScanTokenRequestsWithMetadata( bool includeTableMetadata, bool includeTabletMetadata) { var builder = ClientTestUtil.GetBasicSchema() .SetTableName(_tableName) .SetNumReplicas(3); var table = await _client.CreateTableAsync(builder); await ClientTestUtil.LoadDefaultTableAsync(_client, table, 1); var tokens = await _client.NewScanTokenBuilder(table) .IncludeTableMetadata(includeTableMetadata) .IncludeTabletMetadata(includeTabletMetadata) .BuildAsync(); var token = Assert.Single(tokens); // Use a new client to simulate hydrating in a new process. var newClient = _harness.CreateClient(); var scanBuilder = await newClient.NewScanBuilderFromTokenAsync(token.Serialize()); var scanner = scanBuilder.Build(); Assert.Equal(1, await scanner.CountAsync()); }
public async Task TestAlterModifyColumns() { KuduTable table = await CreateTableAsync(); await InsertRowsAsync(table, 0, 100); Assert.Equal(100, await ClientTestUtil.CountRowsAsync(_client, table)); // Check for expected defaults. ColumnSchema col = table.Schema.GetColumn(1); Assert.Equal(CompressionType.DefaultCompression, col.Compression); Assert.Equal(EncodingType.AutoEncoding, col.Encoding); Assert.Null(col.DefaultValue); // Alter the table. await _client.AlterTableAsync(new AlterTableBuilder(table) .ChangeCompressionAlgorithm(col.Name, CompressionType.Snappy) .ChangeEncoding(col.Name, EncodingType.Rle) .ChangeDefault(col.Name, 0)); // Check for new values. table = await _client.OpenTableAsync(_tableName); col = table.Schema.GetColumn(1); Assert.Equal(CompressionType.Snappy, col.Compression); Assert.Equal(EncodingType.Rle, col.Encoding); Assert.Equal(0, col.DefaultValue); }
private async Task WriteAsync(CancellationToken cancellationToken) { Exception sessionException = null; Exception exception; ValueTask HandleSessionExceptionAsync(SessionExceptionContext context) { Volatile.Write(ref sessionException, context.Exception); return(new ValueTask()); } var options = new KuduSessionOptions { ExceptionHandler = HandleSessionExceptionAsync }; await using var session = _client.NewSession(options); int currentRowKey = 0; bool flush = false; while (true) { if (cancellationToken.IsCancellationRequested) { break; } exception = Volatile.Read(ref sessionException); if (exception != null) { throw exception; } var row = ClientTestUtil.CreateBasicSchemaInsert(_table, currentRowKey); await session.EnqueueAsync(row, CancellationToken.None); if (flush) { await session.FlushAsync(CancellationToken.None); } // Every 10 rows we flush and change the flush mode randomly. if (currentRowKey % 10 == 0) { flush = ThreadSafeRandom.Instance.NextBool(); } currentRowKey++; } await session.FlushAsync(CancellationToken.None); exception = Volatile.Read(ref sessionException); if (exception != null) { throw exception; } }
public async Task TestIterable() { await using var miniCluster = await new MiniKuduClusterBuilder().BuildAsync(); await using var client = miniCluster.CreateClient(); await using var session = client.NewSession(); var builder = ClientTestUtil.GetBasicSchema() .SetTableName("TestIterable") .CreateBasicRangePartition(); var table = await client.CreateTableAsync(builder); IDictionary <int, PartialRow> inserts = new Dictionary <int, PartialRow>(); int numRows = 10; for (int i = 0; i < numRows; i++) { var insert = table.NewInsert(); _generator.RandomizeRow(insert); inserts.TryAdd(insert.GetInt32(0), insert); await session.EnqueueAsync(insert); } await session.FlushAsync(); var scanner = client.NewScanBuilder(table) .SetReadMode(ReadMode.ReadYourWrites) .SetReplicaSelection(ReplicaSelection.LeaderOnly) .Build(); await foreach (var resultSet in scanner) { foreach (var row in resultSet) { var key = row.GetInt32(0); var insert = Assert.Contains(key, inserts); Assert.Equal(insert.GetInt32(1), row.GetInt32(1)); Assert.Equal(insert.GetInt32(2), row.GetInt32(2)); if (insert.IsNull(3)) { Assert.True(row.IsNull(3)); } else { Assert.Equal(insert.GetString(3), row.GetString(3)); } Assert.Equal(insert.GetBool(4), row.GetBool(4)); inserts.Remove(key); } } Assert.Empty(inserts); }
public async Task TestTxnKeepaliveSwitchesToOtherTxnManager() { await using var harness = await new MiniKuduClusterBuilder() .AddMasterServerFlag("--txn_manager_enabled") // Set Raft heartbeat interval short for faster test runtime: speed up // leader failure detection and new leader election. .AddMasterServerFlag("--raft_heartbeat_interval_ms=100") // The txn keepalive interval should be long enough to accommodate Raft // leader failure detection and election. .AddTabletServerFlag("--txn_keepalive_interval_ms=1000") .AddTabletServerFlag("--txn_staleness_tracker_interval_ms=250") .AddTabletServerFlag("--enable_txn_system_client_init=true") .BuildHarnessAsync(); await using var client = harness.CreateClient(); var builder = ClientTestUtil.GetBasicSchema() .SetTableName(nameof(TestTxnKeepaliveSwitchesToOtherTxnManager)) .AddHashPartitions(2, "key"); var table = await client.CreateTableAsync(builder); using var transaction = await client.NewTransactionAsync(); var insert = ClientTestUtil.CreateBasicSchemaInsert(table, 0); await transaction.WriteAsync(new[] { insert }); await harness.KillLeaderMasterServerAsync(); // Wait for two keepalive intervals to make sure the backend got a chance // to automatically abort the transaction if not receiving txn keepalive // messages. await Task.Delay(2 * 1000); // It should be possible to commit the transaction. This is to verify that // // * the client eventually starts sending txn keepalive messages to other // TxnManager instance (the original was hosted by former leader master // which is no longer available), so the backend doesn't abort the // transaction automatically due to not receiving keepalive messages // // * the client switches to the new TxnManager for other txn-related // operations as well await transaction.CommitAsync(); await transaction.WaitForCommitAsync(); // An extra sanity check: read back the rows written into the table in the // context of the transaction. var scanner = client.NewScanBuilder(table) .SetReadMode(ReadMode.ReadYourWrites) .SetReplicaSelection(ReplicaSelection.LeaderOnly) .Build(); Assert.Equal(1, await scanner.CountAsync()); }
public async Task TestPropagateTxnCommitTimestamp() { await using var harness = await new MiniKuduClusterBuilder() .AddMasterServerFlag("--txn_manager_enabled") // Inject latency to have a chance spotting the transaction in the // FINALIZE_IN_PROGRESS state and make KuduTransaction.WaitForCommitAsync() // have to poll multiple times. .AddTabletServerFlag("--txn_status_manager_inject_latency_finalize_commit_ms=250") .AddTabletServerFlag("--enable_txn_system_client_init=true") .BuildHarnessAsync(); await using var client = harness.CreateClient(); var builder = ClientTestUtil.GetBasicSchema() .SetTableName(nameof(TestPropagateTxnCommitTimestamp)) .AddHashPartitions(8, "key"); var table = await client.CreateTableAsync(builder); // Make sure the commit timestamp for a transaction is propagated to the // client upon committing a transaction. using (var transaction = await client.NewTransactionAsync()) { // Insert many rows: the goal is to get at least one row inserted into // every tablet of the hash-partitioned test table, so every tablet would // be a participant in the transaction, and most likely every tablet // server would be involved. var inserts = Enumerable .Range(0, 128) .Select(key => ClientTestUtil.CreateBasicSchemaInsert(table, key)); await transaction.WriteAsync(inserts); await CommitAndVerifyTransactionAsync(client, transaction); } // Make sure the commit timestamp for a transaction is propagated to the // client upon committing a transaction (using a session). using (var transaction = await client.NewTransactionAsync()) { await using var session = transaction.NewSession(); // Insert many rows: the goal is to get at least one row inserted into // every tablet of the hash-partitioned test table, so every tablet would // be a participant in the transaction, and most likely every tablet // server would be involved. for (int key = 128; key < 256; key++) { var insert = ClientTestUtil.CreateBasicSchemaInsert(table, key); await session.EnqueueAsync(insert); } await session.FlushAsync(); await CommitAndVerifyTransactionAsync(client, transaction); } }
private static async ValueTask InsertRowsAsync( IKuduSession session, KuduTable table, int startRow, int numRows) { var end = startRow + numRows; for (var i = startRow; i < end; i++) { var insert = ClientTestUtil.CreateBasicSchemaInsert(table, i); await session.EnqueueAsync(insert); } }
public async Task TestScanTokensNonCoveringRangePartitions() { var builder = ClientTestUtil.CreateManyStringsSchema() .SetTableName(_tableName) .AddHashPartitions(2, "key") .CreateBasicRangePartition() .AddRangePartition((lower, upper) => { lower.SetString("key", "a"); upper.SetString("key", "f"); }) .AddRangePartition((lower, upper) => { lower.SetString("key", "h"); upper.SetString("key", "z"); }) .AddSplitRow(row => row.SetString("key", "k")); var table = await _client.CreateTableAsync(builder); for (char c = 'a'; c < 'f'; c++) { var row = table.NewInsert(); row.SetString("key", $"{c}"); row.SetString("c1", $"c1_{c}"); row.SetString("c2", $"c2_{c}"); await _session.EnqueueAsync(row); } for (char c = 'h'; c < 'z'; c++) { var row = table.NewInsert(); row.SetString("key", $"{c}"); row.SetString("c1", $"c1_{c}"); row.SetString("c2", $"c2_{c}"); await _session.EnqueueAsync(row); } await _session.FlushAsync(); var tokenBuilder = _client.NewScanTokenBuilder(table) .SetEmptyProjection(); List <KuduScanToken> tokens = await tokenBuilder.BuildAsync(); Assert.Equal(6, tokens.Count); await using var newClient = _harness.CreateClient(); var rowCount = await CountScanTokenRowsAsync(newClient, tokens); Assert.Equal('f' - 'a' + 'z' - 'h', rowCount); }
public async Task TestRenameKeyColumn() { KuduTable table = await CreateTableAsync(); await InsertRowsAsync(table, 0, 100); Assert.Equal(100, await ClientTestUtil.CountRowsAsync(_client, table)); await _client.AlterTableAsync(new AlterTableBuilder(table) .RenameColumn("c0", "c0Key")); var exception = await Assert.ThrowsAsync <NonRecoverableException>(async() => { // Scanning with the old schema. var scanner = _client.NewScanBuilder(table) .SetProjectedColumns("c0", "c1") .Build(); await foreach (var resultSet in scanner) { } }); Assert.True(exception.Status.IsInvalidArgument); Assert.Contains( "Some columns are not present in the current schema: c0", exception.Status.Message); // Reopen table for the new schema. table = await _client.OpenTableAsync(_tableName); Assert.Equal("c0Key", table.Schema.GetColumn(0).Name); Assert.Equal(2, table.Schema.Columns.Count); // Add a row var insert = table.NewInsert(); insert.SetInt32("c0Key", 101); insert.SetInt32("c1", 101); await _session.EnqueueAsync(insert); await _session.FlushAsync(); var scanner2 = _client.NewScanBuilder(table) .SetProjectedColumns("c0Key", "c1") .Build(); var rows = await scanner2.ScanToListAsync <(int c0Key, int c1)>(); Assert.Equal(101, rows.Count); Assert.All(rows, row => Assert.Equal(row.c0Key, row.c1)); }
public async Task TestAlterAddColumns() { KuduTable table = await CreateTableAsync(); await InsertRowsAsync(table, 0, 100); Assert.Equal(100, await ClientTestUtil.CountRowsAsync(_client, table)); await _client.AlterTableAsync(new AlterTableBuilder(table) .AddColumn("addNonNull", KuduType.Int32, opt => opt .Nullable(false) .DefaultValue(100)) .AddColumn("addNullable", KuduType.Int32) .AddColumn("addNullableDef", KuduType.Int32, opt => opt .DefaultValue(200))); // Reopen table for the new schema. table = await _client.OpenTableAsync(_tableName); Assert.Equal(5, table.Schema.Columns.Count); // Add a row with addNullableDef=null var insert = table.NewInsert(); insert.SetInt32("c0", 101); insert.SetInt32("c1", 101); insert.SetInt32("addNonNull", 101); insert.SetInt32("addNullable", 101); insert.SetNull("addNullableDef"); await _session.EnqueueAsync(insert); await _session.FlushAsync(); // Check defaults applied, and that row key=101 var results = await ClientTestUtil.ScanTableToStringsAsync(_client, table); var expected = new List <string>(101); for (int i = 0; i < 100; i++) { expected.Add($"INT32 c0={i}, INT32 c1={i}, INT32 addNonNull=100, " + "INT32 addNullable=NULL, INT32 addNullableDef=200"); } expected.Add("INT32 c0=101, INT32 c1=101, INT32 addNonNull=101, " + "INT32 addNullable=101, INT32 addNullableDef=NULL"); Assert.Equal( expected.OrderBy(r => r), results.OrderBy(r => r)); }
public async Task TestKuduRequireEncryption() { await using var harness = await new MiniKuduClusterBuilder().BuildHarnessAsync(); await using var client = harness.CreateClientBuilder() .SetEncryptionPolicy(EncryptionPolicy.Required) .Build(); var builder = ClientTestUtil.GetBasicSchema() .SetTableName(nameof(TestKuduRequireEncryption)); var table = await client.CreateTableAsync(builder); Assert.NotNull(table.TableId); }
public async Task TestGetTableStatistics() { await using var harness = await new MiniKuduClusterBuilder() .AddTabletServerFlag("--update_tablet_stats_interval_ms=200") .AddTabletServerFlag("--heartbeat_interval_ms=100") .BuildHarnessAsync(); await using var client = harness.CreateClient(); // Create a table. var builder = ClientTestUtil.GetBasicSchema().SetTableName(_tableName); var table = await client.CreateTableAsync(builder); // Insert some rows and test the statistics. var prevStatistics = new KuduTableStatistics(-1, -1); var currentStatistics = new KuduTableStatistics(-1, -1); var session = client.NewSession(); int num = 100; for (int i = 0; i < num; ++i) { // Get current table statistics. currentStatistics = await client.GetTableStatisticsAsync(_tableName); Assert.True(currentStatistics.OnDiskSize >= prevStatistics.OnDiskSize); Assert.True(currentStatistics.LiveRowCount >= prevStatistics.LiveRowCount); Assert.True(currentStatistics.LiveRowCount <= i + 1); prevStatistics = currentStatistics; // Insert row. var insert = ClientTestUtil.CreateBasicSchemaInsert(table, i); await session.EnqueueAsync(insert); await session.FlushAsync(); long numRows = await ClientTestUtil.CountRowsAsync(client, table); Assert.Equal(i + 1, numRows); } // Final accuracy test. // Wait for master to aggregate table statistics. await Task.Delay(200 * 6); currentStatistics = await client.GetTableStatisticsAsync(_tableName); Assert.True(currentStatistics.OnDiskSize >= prevStatistics.OnDiskSize); Assert.True(currentStatistics.LiveRowCount >= prevStatistics.LiveRowCount); Assert.Equal(num, currentStatistics.LiveRowCount); }
public async Task TestSwitchToOtherTxnManagerInFlightCalls() { await using var harness = await new MiniKuduClusterBuilder() .AddMasterServerFlag("--txn_manager_enabled") // Set Raft heartbeat interval short for faster test runtime: speed up // leader failure detection and new leader election. .AddMasterServerFlag("--raft_heartbeat_interval_ms=100") .AddTabletServerFlag("--enable_txn_system_client_init=true") .BuildHarnessAsync(); await using var client = harness.CreateClient(); var builder = ClientTestUtil.GetBasicSchema() .SetTableName(nameof(TestSwitchToOtherTxnManagerInFlightCalls)) .AddHashPartitions(2, "key"); var table = await client.CreateTableAsync(builder); using var transaction = await client.NewTransactionAsync(); var insert = ClientTestUtil.CreateBasicSchemaInsert(table, 0); await transaction.WriteAsync(new[] { insert }); await harness.KillAllMasterServersAsync(); var startMastersTask = Task.Run(async() => { // Sleep for some time to allow the commit call // below issue RPCs to non-running TxnManangers. await Task.Delay(1000); await harness.StartAllMasterServersAsync(); }); // It should be possible to commit the transaction. await transaction.CommitAsync(); await transaction.WaitForCommitAsync(); await startMastersTask; // An extra sanity check: read back the rows written into the table in the // context of the transaction. var scanner = client.NewScanBuilder(table) .SetReadMode(ReadMode.ReadYourWrites) .SetReplicaSelection(ReplicaSelection.LeaderOnly) .Build(); Assert.Equal(1, await scanner.CountAsync()); }
public async Task TestAlterExtraConfigs() { KuduTable table = await CreateTableAsync(); await InsertRowsAsync(table, 0, 100); Assert.Equal(100, await ClientTestUtil.CountRowsAsync(_client, table)); // 1. Check for expected defaults. table = await _client.OpenTableAsync(_tableName); Assert.DoesNotContain("kudu.table.history_max_age_sec", table.ExtraConfig); // 2. Alter history max age second to 3600 var alterExtraConfigs = new Dictionary <string, string> { { "kudu.table.history_max_age_sec", "3600" } }; await _client.AlterTableAsync(new AlterTableBuilder(table) .AlterExtraConfigs(alterExtraConfigs)); table = await _client.OpenTableAsync(_tableName); Assert.Equal("3600", table.ExtraConfig["kudu.table.history_max_age_sec"]); // 3. Alter history max age second to 7200 alterExtraConfigs = new Dictionary <string, string> { { "kudu.table.history_max_age_sec", "7200" } }; await _client.AlterTableAsync(new AlterTableBuilder(table) .AlterExtraConfigs(alterExtraConfigs)); table = await _client.OpenTableAsync(_tableName); Assert.Equal("7200", table.ExtraConfig["kudu.table.history_max_age_sec"]); // 4. Reset history max age second to default alterExtraConfigs = new Dictionary <string, string> { { "kudu.table.history_max_age_sec", "" } }; await _client.AlterTableAsync(new AlterTableBuilder(table) .AlterExtraConfigs(alterExtraConfigs)); table = await _client.OpenTableAsync(_tableName); Assert.Empty(table.ExtraConfig); }
public async Task TestResourceMetrics() { // Scan one tablet and the whole table. var oneTabletScanner = GetScanner("1", "1", "1", "4"); // Whole second tablet. await using var oneTabletScanEnumerator = oneTabletScanner.GetAsyncEnumerator(); Assert.Equal(3, await ClientTestUtil.CountRowsInScanAsync(oneTabletScanEnumerator)); var fullTableScanner = GetScanner(null, null, null, null); await using var fullTableScanEnumerator = fullTableScanner.GetAsyncEnumerator(); Assert.Equal(9, await ClientTestUtil.CountRowsInScanAsync(fullTableScanEnumerator)); // Both scans should take a positive amount of wait duration, total duration, // cpu user and cpu system time. ValidateResourceMetrics(oneTabletScanEnumerator.ResourceMetrics); ValidateResourceMetrics(fullTableScanEnumerator.ResourceMetrics);
public async Task TestKuduRequireAuthenticationInsecureCluster() { await using var harness = await new MiniKuduClusterBuilder().BuildHarnessAsync(); await using var client = harness.CreateClientBuilder() .RequireAuthentication(true) .Build(); var builder = ClientTestUtil.GetBasicSchema() .SetTableName(nameof(TestKuduRequireAuthenticationInsecureCluster)); var exception = await Assert.ThrowsAsync <NonRecoverableException>( () => client.CreateTableAsync(builder)); Assert.Contains( "Client requires authentication, but server does not have Kerberos enabled", exception.Message); }
public async Task TestExceptionCallback() { int numCallbacks = 0; SessionExceptionContext sessionContext = null; var builder = ClientTestUtil.GetBasicSchema() .SetTableName(nameof(TestExceptionCallback)); var table = await _client.CreateTableAsync(builder); var row1 = ClientTestUtil.CreateBasicSchemaInsert(table, 1); var row2 = ClientTestUtil.CreateBasicSchemaInsert(table, 1); var sessionOptions = new KuduSessionOptions { ExceptionHandler = HandleSessionExceptionAsync }; await using var session = _client.NewSession(sessionOptions); await session.EnqueueAsync(row1); await session.FlushAsync(); await session.EnqueueAsync(row2); await session.FlushAsync(); ValueTask HandleSessionExceptionAsync(SessionExceptionContext context) { numCallbacks++; sessionContext = context; return(new ValueTask()); } Assert.Equal(1, numCallbacks); var errorRow = Assert.Single(sessionContext.Rows); Assert.Same(row2, errorRow); var exception = Assert.IsType <KuduWriteException>(sessionContext.Exception); var exceptionRow = Assert.Single(exception.PerRowErrors); Assert.True(exceptionRow.IsAlreadyPresent); }
public async Task TestAuthzTokensDuringElection() { await using var client = _harness.CreateClient(); await using var session = client.NewSession(); // Test sending various requests that require authorization. var builder = ClientTestUtil.GetBasicSchema() .SetTableName(_tableName) .CreateBasicRangePartition() .SetNumReplicas(1); var table = await client.CreateTableAsync(builder); // Restart the masters to trigger an election. await _harness.KillAllMasterServersAsync(); await _harness.StartAllMasterServersAsync(); int numReqs = 10; await InsertRowsAsync(session, table, 0, numReqs); await session.FlushAsync(); // Do the same for batches of inserts. await _harness.KillAllMasterServersAsync(); await _harness.StartAllMasterServersAsync(); await InsertRowsAsync(session, table, numReqs, numReqs); await session.FlushAsync(); // And for scans. await _harness.KillAllMasterServersAsync(); await _harness.StartAllMasterServersAsync(); for (int i = 0; i < numReqs; i++) { var numRows = await ClientTestUtil.CountRowsAsync(client, table); Assert.Equal(2 * numReqs, numRows); } }
public async Task Test() { var testRuntime = TimeSpan.FromMinutes(1); using var cts = new CancellationTokenSource(testRuntime); var token = cts.Token; var chaosTask = RunTaskAsync(() => DoChaosAsync(token), cts); var writeTask = RunTaskAsync(() => WriteAsync(token), cts); var scanTask = RunTaskAsync(() => ScanAsync(token), cts); await Task.WhenAll(chaosTask, writeTask, scanTask); // If the test passed, do some extra validation at the end. var rowCount = await ClientTestUtil.CountRowsAsync(_client, _table); Assert.True(rowCount > 0); }
public async Task TestAlterRangePartitioningExclusiveInclusive() { // Create initial table with single range partition covering (-1, 99]. var builder = new TableBuilder(_tableName) .SetNumReplicas(1) .AddColumn("c0", KuduType.Int32, opt => opt.Key(true)) .AddColumn("c1", KuduType.Int32, opt => opt.Nullable(false)) .SetRangePartitionColumns("c0") .AddRangePartition((lower, upper) => { lower.SetInt32("c0", -1); upper.SetInt32("c0", 99); }, RangePartitionBound.Exclusive, RangePartitionBound.Inclusive); KuduTable table = await _client.CreateTableAsync(builder); await _client.AlterTableAsync(new AlterTableBuilder(table) .AddRangePartition((lower, upper) => { lower.SetInt32("c0", 199); upper.SetInt32("c0", 299); }, RangePartitionBound.Exclusive, RangePartitionBound.Inclusive)); // Insert some rows, and then drop the partition and ensure that the table is empty. await InsertRowsAsync(table, 0, 100); await InsertRowsAsync(table, 200, 300); Assert.Equal(200, await ClientTestUtil.CountRowsAsync(_client, table)); await _client.AlterTableAsync(new AlterTableBuilder(table) .DropRangePartition((lower, upper) => { lower.SetInt32("c0", 0); upper.SetInt32("c0", 100); }, RangePartitionBound.Inclusive, RangePartitionBound.Exclusive) .DropRangePartition((lower, upper) => { lower.SetInt32("c0", 199); upper.SetInt32("c0", 299); }, RangePartitionBound.Exclusive, RangePartitionBound.Inclusive)); Assert.Equal(0, await ClientTestUtil.CountRowsAsync(_client, table)); }
public async Task TestScannerExpiration() { await using var miniCluster = await new MiniKuduClusterBuilder() .AddTabletServerFlag($"--scanner_ttl_ms={ShortScannerTtlMs}") .AddTabletServerFlag($"--scanner_gc_check_interval_us={ShortScannerGcUs}") .BuildAsync(); await using var client = miniCluster.CreateClient(); var builder = ClientTestUtil.GetBasicSchema() .SetTableName("TestScannerExpiration") .AddHashPartitions(2, "key"); var table = await client.CreateTableAsync(builder); int numRows = 1000; var rows = Enumerable.Range(0, numRows) .Select(i => ClientTestUtil.CreateBasicSchemaInsert(table, i)); await client.WriteAsync(rows); var scanner = client.NewScanBuilder(table) .SetReadMode(ReadMode.ReadYourWrites) .SetReplicaSelection(ReplicaSelection.ClosestReplica) .SetBatchSizeBytes(100) // Use a small batch size so we get many batches. .Build(); var scanEnumerator = scanner.GetAsyncEnumerator(); // Initialize the scanner and verify we can read rows. Assert.True(await scanEnumerator.MoveNextAsync()); Assert.True(scanEnumerator.Current.Count > 0); // Wait for the scanner to time out. await Task.Delay(ShortScannerTtlMs * 2); var exception = await Assert.ThrowsAsync <NonRecoverableException>(async() => await scanEnumerator.MoveNextAsync()); Assert.Matches(".*Scanner .* not found.*", exception.Message); // Closing an expired scanner shouldn't throw an exception. await scanEnumerator.DisposeAsync(); }