public async Task TestAlterModifyColumns() { KuduTable table = await CreateTableAsync(); await InsertRowsAsync(table, 0, 100); Assert.Equal(100, await ClientTestUtil.CountRowsAsync(_client, table)); // Check for expected defaults. ColumnSchema col = table.Schema.GetColumn(1); Assert.Equal(CompressionType.DefaultCompression, col.Compression); Assert.Equal(EncodingType.AutoEncoding, col.Encoding); Assert.Null(col.DefaultValue); // Alter the table. await _client.AlterTableAsync(new AlterTableBuilder(table) .ChangeCompressionAlgorithm(col.Name, CompressionType.Snappy) .ChangeEncoding(col.Name, EncodingType.Rle) .ChangeDefault(col.Name, 0)); // Check for new values. table = await _client.OpenTableAsync(_tableName); col = table.Schema.GetColumn(1); Assert.Equal(CompressionType.Snappy, col.Compression); Assert.Equal(EncodingType.Rle, col.Encoding); Assert.Equal(0, col.DefaultValue); }
public async Task TestRenameKeyColumn() { KuduTable table = await CreateTableAsync(); await InsertRowsAsync(table, 0, 100); Assert.Equal(100, await ClientTestUtil.CountRowsAsync(_client, table)); await _client.AlterTableAsync(new AlterTableBuilder(table) .RenameColumn("c0", "c0Key")); var exception = await Assert.ThrowsAsync <NonRecoverableException>(async() => { // Scanning with the old schema. var scanner = _client.NewScanBuilder(table) .SetProjectedColumns("c0", "c1") .Build(); await foreach (var resultSet in scanner) { } }); Assert.True(exception.Status.IsInvalidArgument); Assert.Contains( "Some columns are not present in the current schema: c0", exception.Status.Message); // Reopen table for the new schema. table = await _client.OpenTableAsync(_tableName); Assert.Equal("c0Key", table.Schema.GetColumn(0).Name); Assert.Equal(2, table.Schema.Columns.Count); // Add a row var insert = table.NewInsert(); insert.SetInt32("c0Key", 101); insert.SetInt32("c1", 101); await _session.EnqueueAsync(insert); await _session.FlushAsync(); var scanner2 = _client.NewScanBuilder(table) .SetProjectedColumns("c0Key", "c1") .Build(); var rows = await scanner2.ScanToListAsync <(int c0Key, int c1)>(); Assert.Equal(101, rows.Count); Assert.All(rows, row => Assert.Equal(row.c0Key, row.c1)); }
public async Task TestAlterAddColumns() { KuduTable table = await CreateTableAsync(); await InsertRowsAsync(table, 0, 100); Assert.Equal(100, await ClientTestUtil.CountRowsAsync(_client, table)); await _client.AlterTableAsync(new AlterTableBuilder(table) .AddColumn("addNonNull", KuduType.Int32, opt => opt .Nullable(false) .DefaultValue(100)) .AddColumn("addNullable", KuduType.Int32) .AddColumn("addNullableDef", KuduType.Int32, opt => opt .DefaultValue(200))); // Reopen table for the new schema. table = await _client.OpenTableAsync(_tableName); Assert.Equal(5, table.Schema.Columns.Count); // Add a row with addNullableDef=null var insert = table.NewInsert(); insert.SetInt32("c0", 101); insert.SetInt32("c1", 101); insert.SetInt32("addNonNull", 101); insert.SetInt32("addNullable", 101); insert.SetNull("addNullableDef"); await _session.EnqueueAsync(insert); await _session.FlushAsync(); // Check defaults applied, and that row key=101 var results = await ClientTestUtil.ScanTableToStringsAsync(_client, table); var expected = new List <string>(101); for (int i = 0; i < 100; i++) { expected.Add($"INT32 c0={i}, INT32 c1={i}, INT32 addNonNull=100, " + "INT32 addNullable=NULL, INT32 addNullableDef=200"); } expected.Add("INT32 c0=101, INT32 c1=101, INT32 addNonNull=101, " + "INT32 addNullable=101, INT32 addNullableDef=NULL"); Assert.Equal( expected.OrderBy(r => r), results.OrderBy(r => r)); }
public async Task TestAlterExtraConfigs() { KuduTable table = await CreateTableAsync(); await InsertRowsAsync(table, 0, 100); Assert.Equal(100, await ClientTestUtil.CountRowsAsync(_client, table)); // 1. Check for expected defaults. table = await _client.OpenTableAsync(_tableName); Assert.DoesNotContain("kudu.table.history_max_age_sec", table.ExtraConfig); // 2. Alter history max age second to 3600 var alterExtraConfigs = new Dictionary <string, string> { { "kudu.table.history_max_age_sec", "3600" } }; await _client.AlterTableAsync(new AlterTableBuilder(table) .AlterExtraConfigs(alterExtraConfigs)); table = await _client.OpenTableAsync(_tableName); Assert.Equal("3600", table.ExtraConfig["kudu.table.history_max_age_sec"]); // 3. Alter history max age second to 7200 alterExtraConfigs = new Dictionary <string, string> { { "kudu.table.history_max_age_sec", "7200" } }; await _client.AlterTableAsync(new AlterTableBuilder(table) .AlterExtraConfigs(alterExtraConfigs)); table = await _client.OpenTableAsync(_tableName); Assert.Equal("7200", table.ExtraConfig["kudu.table.history_max_age_sec"]); // 4. Reset history max age second to default alterExtraConfigs = new Dictionary <string, string> { { "kudu.table.history_max_age_sec", "" } }; await _client.AlterTableAsync(new AlterTableBuilder(table) .AlterExtraConfigs(alterExtraConfigs)); table = await _client.OpenTableAsync(_tableName); Assert.Empty(table.ExtraConfig); }
public async Task TestGetTableStatistics() { await using var harness = await new MiniKuduClusterBuilder() .AddTabletServerFlag("--update_tablet_stats_interval_ms=200") .AddTabletServerFlag("--heartbeat_interval_ms=100") .BuildHarnessAsync(); await using var client = harness.CreateClient(); // Create a table. var builder = ClientTestUtil.GetBasicSchema().SetTableName(_tableName); var table = await client.CreateTableAsync(builder); // Insert some rows and test the statistics. var prevStatistics = new KuduTableStatistics(-1, -1); var currentStatistics = new KuduTableStatistics(-1, -1); var session = client.NewSession(); int num = 100; for (int i = 0; i < num; ++i) { // Get current table statistics. currentStatistics = await client.GetTableStatisticsAsync(_tableName); Assert.True(currentStatistics.OnDiskSize >= prevStatistics.OnDiskSize); Assert.True(currentStatistics.LiveRowCount >= prevStatistics.LiveRowCount); Assert.True(currentStatistics.LiveRowCount <= i + 1); prevStatistics = currentStatistics; // Insert row. var insert = ClientTestUtil.CreateBasicSchemaInsert(table, i); await session.EnqueueAsync(insert); await session.FlushAsync(); long numRows = await ClientTestUtil.CountRowsAsync(client, table); Assert.Equal(i + 1, numRows); } // Final accuracy test. // Wait for master to aggregate table statistics. await Task.Delay(200 * 6); currentStatistics = await client.GetTableStatisticsAsync(_tableName); Assert.True(currentStatistics.OnDiskSize >= prevStatistics.OnDiskSize); Assert.True(currentStatistics.LiveRowCount >= prevStatistics.LiveRowCount); Assert.Equal(num, currentStatistics.LiveRowCount); }
public async Task TestAuthzTokensDuringElection() { await using var client = _harness.CreateClient(); await using var session = client.NewSession(); // Test sending various requests that require authorization. var builder = ClientTestUtil.GetBasicSchema() .SetTableName(_tableName) .CreateBasicRangePartition() .SetNumReplicas(1); var table = await client.CreateTableAsync(builder); // Restart the masters to trigger an election. await _harness.KillAllMasterServersAsync(); await _harness.StartAllMasterServersAsync(); int numReqs = 10; await InsertRowsAsync(session, table, 0, numReqs); await session.FlushAsync(); // Do the same for batches of inserts. await _harness.KillAllMasterServersAsync(); await _harness.StartAllMasterServersAsync(); await InsertRowsAsync(session, table, numReqs, numReqs); await session.FlushAsync(); // And for scans. await _harness.KillAllMasterServersAsync(); await _harness.StartAllMasterServersAsync(); for (int i = 0; i < numReqs; i++) { var numRows = await ClientTestUtil.CountRowsAsync(client, table); Assert.Equal(2 * numReqs, numRows); } }
public async Task Test() { var testRuntime = TimeSpan.FromMinutes(1); using var cts = new CancellationTokenSource(testRuntime); var token = cts.Token; var chaosTask = RunTaskAsync(() => DoChaosAsync(token), cts); var writeTask = RunTaskAsync(() => WriteAsync(token), cts); var scanTask = RunTaskAsync(() => ScanAsync(token), cts); await Task.WhenAll(chaosTask, writeTask, scanTask); // If the test passed, do some extra validation at the end. var rowCount = await ClientTestUtil.CountRowsAsync(_client, _table); Assert.True(rowCount > 0); }
public async Task TestAlterRangePartitioningExclusiveInclusive() { // Create initial table with single range partition covering (-1, 99]. var builder = new TableBuilder(_tableName) .SetNumReplicas(1) .AddColumn("c0", KuduType.Int32, opt => opt.Key(true)) .AddColumn("c1", KuduType.Int32, opt => opt.Nullable(false)) .SetRangePartitionColumns("c0") .AddRangePartition((lower, upper) => { lower.SetInt32("c0", -1); upper.SetInt32("c0", 99); }, RangePartitionBound.Exclusive, RangePartitionBound.Inclusive); KuduTable table = await _client.CreateTableAsync(builder); await _client.AlterTableAsync(new AlterTableBuilder(table) .AddRangePartition((lower, upper) => { lower.SetInt32("c0", 199); upper.SetInt32("c0", 299); }, RangePartitionBound.Exclusive, RangePartitionBound.Inclusive)); // Insert some rows, and then drop the partition and ensure that the table is empty. await InsertRowsAsync(table, 0, 100); await InsertRowsAsync(table, 200, 300); Assert.Equal(200, await ClientTestUtil.CountRowsAsync(_client, table)); await _client.AlterTableAsync(new AlterTableBuilder(table) .DropRangePartition((lower, upper) => { lower.SetInt32("c0", 0); upper.SetInt32("c0", 100); }, RangePartitionBound.Inclusive, RangePartitionBound.Exclusive) .DropRangePartition((lower, upper) => { lower.SetInt32("c0", 199); upper.SetInt32("c0", 299); }, RangePartitionBound.Exclusive, RangePartitionBound.Inclusive)); Assert.Equal(0, await ClientTestUtil.CountRowsAsync(_client, table)); }
public async Task TestFailover(bool restart) { await using var harness = await new MiniKuduClusterBuilder() .NumMasters(3) .NumTservers(3) .BuildHarnessAsync(); await using var client = harness.CreateClient(); var builder = ClientTestUtil.GetBasicSchema() .SetTableName("LeaderFailoverTest") .CreateBasicRangePartition(); var table = await client.CreateTableAsync(builder); var rows = Enumerable.Range(0, 3) .Select(i => ClientTestUtil.CreateBasicSchemaInsert(table, i)); await client.WriteAsync(rows); // Make sure the rows are in there before messing things up. long numRows = await ClientTestUtil.CountRowsAsync(client, table); Assert.Equal(3, numRows); if (restart) { await harness.RestartLeaderMasterAsync(); } else { await harness.KillLeaderMasterServerAsync(); } var rows2 = Enumerable.Range(3, 3) .Select(i => ClientTestUtil.CreateBasicSchemaInsert(table, i)); await client.WriteAsync(rows2); long numRows2 = await ClientTestUtil.CountRowsAsync(client, table); Assert.Equal(6, numRows2); }
public async Task TestAuthzTokenExpiration() { // Test a long-running concurrent workload with different types of requests // being sent, all the while injecting invalid tokens, with a short authz // token expiration time. The threads should reacquire tokens as needed // without surfacing token errors to the client. await using var client = _harness.CreateClientBuilder() .SetDefaultOperationTimeout(TimeSpan.FromMinutes(1)) .Build(); using var cts = new CancellationTokenSource(); var stopToken = cts.Token; var builder = ClientTestUtil.GetBasicSchema() .SetTableName(_tableName) .CreateBasicRangePartition() .SetNumReplicas(1); var table = await client.CreateTableAsync(builder); cts.CancelAfter(TimeSpan.FromSeconds(10)); int rowStart = 0; var loop1 = WriteRowsAsync(flush: true); var loop2 = WriteRowsAsync(flush: false); var scanTask = ScanTableAsync(); await loop1; await loop2; await scanTask; var expected = Interlocked.Add(ref rowStart, 0) * 10; var rows = await ClientTestUtil.CountRowsAsync(client, table); Assert.Equal(expected, rows); async Task WriteRowsAsync(bool flush) { await using var session = client.NewSession(); while (!stopToken.IsCancellationRequested) { var start = Interlocked.Increment(ref rowStart) * 10; await InsertRowsAsync(session, table, start, 10); if (flush) { await session.FlushAsync(); } } } async Task ScanTableAsync() { while (!stopToken.IsCancellationRequested) { // We can't validate the row count until the end, but this // still ensures the scanner doesn't throw any exceptions. await ClientTestUtil.CountRowsAsync(client, table); } } }
public async Task TestAlterRangeParitioningInvalid() { // Create initial table with single range partition covering [0, 100). KuduTable table = await CreateTableAsync((0, 100)); await InsertRowsAsync(table, 0, 100); Assert.Equal(100, await ClientTestUtil.CountRowsAsync(_client, table)); // ADD [0, 100) <- already present (duplicate) var ex = await Assert.ThrowsAsync <NonRecoverableException>(async() => { await _client.AlterTableAsync(new AlterTableBuilder(table) .AddRangePartition((lower, upper) => { lower.SetInt32("c0", 0); upper.SetInt32("c0", 100); })); }); Assert.True(ex.Status.IsAlreadyPresent); Assert.Contains("range partition already exists", ex.Status.Message); Assert.Equal(100, await ClientTestUtil.CountRowsAsync(_client, table)); // ADD [50, 150) <- illegal (overlap) ex = await Assert.ThrowsAsync <NonRecoverableException>(async() => { await _client.AlterTableAsync(new AlterTableBuilder(table) .AddRangePartition((lower, upper) => { lower.SetInt32("c0", 50); upper.SetInt32("c0", 150); })); }); Assert.True(ex.Status.IsInvalidArgument); Assert.Contains("new range partition conflicts with existing one", ex.Status.Message); Assert.Equal(100, await ClientTestUtil.CountRowsAsync(_client, table)); // ADD [-50, 50) <- illegal (overlap) ex = await Assert.ThrowsAsync <NonRecoverableException>(async() => { await _client.AlterTableAsync(new AlterTableBuilder(table) .AddRangePartition((lower, upper) => { lower.SetInt32("c0", -50); upper.SetInt32("c0", 50); })); }); Assert.True(ex.Status.IsInvalidArgument); Assert.Contains("new range partition conflicts with existing one", ex.Status.Message); Assert.Equal(100, await ClientTestUtil.CountRowsAsync(_client, table)); // ADD [200, 300) // ADD [-50, 150) <- illegal (overlap) ex = await Assert.ThrowsAsync <NonRecoverableException>(async() => { await _client.AlterTableAsync(new AlterTableBuilder(table) .AddRangePartition((lower, upper) => { lower.SetInt32("c0", 200); upper.SetInt32("c0", 300); }) .AddRangePartition((lower, upper) => { lower.SetInt32("c0", -50); upper.SetInt32("c0", 150); })); }); Assert.True(ex.Status.IsInvalidArgument); Assert.Contains("new range partition conflicts with existing one", ex.Status.Message); Assert.Equal(100, await ClientTestUtil.CountRowsAsync(_client, table)); // DROP [<start>, <end>) ex = await Assert.ThrowsAsync <NonRecoverableException>(async() => { await _client.AlterTableAsync(new AlterTableBuilder(table) .DropRangePartition((lower, upper) => { })); }); Assert.True(ex.Status.IsInvalidArgument); Assert.Contains("no range partition to drop", ex.Status.Message); Assert.Equal(100, await ClientTestUtil.CountRowsAsync(_client, table)); // DROP [50, 150) // RENAME foo ex = await Assert.ThrowsAsync <NonRecoverableException>(async() => { await _client.AlterTableAsync(new AlterTableBuilder(table) .DropRangePartition((lower, upper) => { lower.SetInt32("c0", 50); upper.SetInt32("c0", 150); }) .RenameTable("foo")); }); Assert.True(ex.Status.IsInvalidArgument); Assert.Contains("no range partition to drop", ex.Status.Message); Assert.Equal(100, await ClientTestUtil.CountRowsAsync(_client, table)); Assert.Empty(await _client.GetTablesAsync("foo")); // DROP [0, 100) // ADD [100, 200) // DROP [100, 200) // ADD [150, 250) // DROP [0, 10) <- illegal ex = await Assert.ThrowsAsync <NonRecoverableException>(async() => { await _client.AlterTableAsync(new AlterTableBuilder(table) .DropRangePartition((lower, upper) => { lower.SetInt32("c0", 0); upper.SetInt32("c0", 100); }) .AddRangePartition((lower, upper) => { lower.SetInt32("c0", 100); upper.SetInt32("c0", 200); }) .DropRangePartition((lower, upper) => { lower.SetInt32("c0", 100); upper.SetInt32("c0", 200); }) .AddRangePartition((lower, upper) => { lower.SetInt32("c0", 150); upper.SetInt32("c0", 250); }) .DropRangePartition((lower, upper) => { lower.SetInt32("c0", 0); upper.SetInt32("c0", 10); })); }); Assert.True(ex.Status.IsInvalidArgument); Assert.Contains("no range partition to drop", ex.Status.Message); Assert.Equal(100, await ClientTestUtil.CountRowsAsync(_client, table)); }
public async Task TestAlterRangePartitioning() { KuduTable table = await CreateTableAsync(); KuduSchema schema = table.Schema; // Insert some rows, and then drop the partition and ensure that the table is empty. await InsertRowsAsync(table, 0, 100); Assert.Equal(100, await ClientTestUtil.CountRowsAsync(_client, table)); await _client.AlterTableAsync(new AlterTableBuilder(table) .DropRangePartition((lower, upper) => { })); Assert.Equal(0, await ClientTestUtil.CountRowsAsync(_client, table)); // Add new range partition and insert rows. await _client.AlterTableAsync(new AlterTableBuilder(table) .AddRangePartition((lower, upper) => { lower.SetInt32("c0", 0); upper.SetInt32("c0", 100); })); await InsertRowsAsync(table, 0, 100); Assert.Equal(100, await ClientTestUtil.CountRowsAsync(_client, table)); // Replace the range partition with a different one. await _client.AlterTableAsync(new AlterTableBuilder(table) .DropRangePartition((lower, upper) => { lower.SetInt32("c0", 0); upper.SetInt32("c0", 100); }) .AddRangePartition((lower, upper) => { lower.SetInt32("c0", 50); upper.SetInt32("c0", 150); })); Assert.Equal(0, await ClientTestUtil.CountRowsAsync(_client, table)); await InsertRowsAsync(table, 50, 125); Assert.Equal(75, await ClientTestUtil.CountRowsAsync(_client, table)); // Replace the range partition with the same one. await _client.AlterTableAsync(new AlterTableBuilder(table) .DropRangePartition((lower, upper) => { lower.SetInt32("c0", 50); upper.SetInt32("c0", 150); }) .AddRangePartition((lower, upper) => { lower.SetInt32("c0", 50); upper.SetInt32("c0", 150); })); Assert.Equal(0, await ClientTestUtil.CountRowsAsync(_client, table)); await InsertRowsAsync(table, 50, 125); Assert.Equal(75, await ClientTestUtil.CountRowsAsync(_client, table)); // Alter table partitioning + alter table schema var newTableName = $"{_tableName}-renamed"; await _client.AlterTableAsync(new AlterTableBuilder(table) .AddRangePartition((lower, upper) => { lower.SetInt32("c0", 200); upper.SetInt32("c0", 300); }) .RenameTable(newTableName) .AddColumn("c2", KuduType.Int32)); await InsertRowsAsync(table, 200, 300); Assert.Equal(175, await ClientTestUtil.CountRowsAsync(_client, table)); Assert.Equal(3, (await _client.OpenTableAsync(newTableName)).Schema.Columns.Count); // Drop all range partitions + alter table schema. This also serves to test // specifying range bounds with a subset schema (since a column was // previously added). await _client.AlterTableAsync(new AlterTableBuilder(table) .DropRangePartition((lower, upper) => { lower.SetInt32("c0", 200); upper.SetInt32("c0", 300); }) .DropRangePartition((lower, upper) => { lower.SetInt32("c0", 50); upper.SetInt32("c0", 150); }) .DropColumn("c2")); Assert.Equal(0, await ClientTestUtil.CountRowsAsync(_client, table)); Assert.Equal(2, (await _client.OpenTableAsync(newTableName)).Schema.Columns.Count); }
public async Task TestMultipleFailover(bool restart) { int rowsPerIteration = 3; int numIterations = 10; int totalRowsToInsert = rowsPerIteration + numIterations * rowsPerIteration; await using var harness = await new MiniKuduClusterBuilder() .NumMasters(3) .NumTservers(3) .BuildHarnessAsync(); await using var client = harness.CreateClient(); var builder = ClientTestUtil.GetBasicSchema() .SetTableName("MultipleLeaderFailoverTest"); var table = await client.CreateTableAsync(builder); await using var session = client.NewSession(); for (int i = 0; i < rowsPerIteration; i++) { var row = ClientTestUtil.CreateBasicSchemaInsert(table, i); await session.EnqueueAsync(row); } await session.FlushAsync(); var rowCount = await ClientTestUtil.CountRowsAsync(client, table); Assert.Equal(rowsPerIteration, rowCount); int currentRows = rowsPerIteration; for (int i = 0; i < numIterations; i++) { var tablets = await client.GetTableLocationsAsync( table.TableId, Array.Empty <byte>(), 1); Assert.Single(tablets); if (restart) { await harness.RestartTabletServerAsync(tablets[0]); } else { await harness.KillTabletLeaderAsync(tablets[0]); } for (int j = 0; j < rowsPerIteration; j++) { var row = ClientTestUtil.CreateBasicSchemaInsert(table, currentRows); await session.EnqueueAsync(row); currentRows++; } await session.FlushAsync(); if (!restart) { await harness.StartAllTabletServersAsync(); } rowCount = await ClientTestUtil.CountRowsAsync(client, table); Assert.Equal(currentRows, rowCount); } rowCount = await ClientTestUtil.CountRowsAsync(client, table); Assert.Equal(totalRowsToInsert, rowCount); }