public async Task TestAlterModifyColumns()
{
KuduTable table = await CreateTableAsync();
await InsertRowsAsync(table, 0, 100);
Assert.Equal(100, await ClientTestUtil.CountRowsAsync(_client, table));
// Check for expected defaults.
ColumnSchema col = table.Schema.GetColumn(1);
Assert.Equal(CompressionType.DefaultCompression, col.Compression);
Assert.Equal(EncodingType.AutoEncoding, col.Encoding);
Assert.Null(col.DefaultValue);
// Alter the table.
await _client.AlterTableAsync(new AlterTableBuilder(table)
.ChangeCompressionAlgorithm(col.Name, CompressionType.Snappy)
.ChangeEncoding(col.Name, EncodingType.Rle)
.ChangeDefault(col.Name, 0));
// Check for new values.
table = await _client.OpenTableAsync(_tableName);
col = table.Schema.GetColumn(1);
Assert.Equal(CompressionType.Snappy, col.Compression);
Assert.Equal(EncodingType.Rle, col.Encoding);
Assert.Equal(0, col.DefaultValue);
}
public async Task TestRenameKeyColumn()
{
KuduTable table = await CreateTableAsync();
await InsertRowsAsync(table, 0, 100);
Assert.Equal(100, await ClientTestUtil.CountRowsAsync(_client, table));
await _client.AlterTableAsync(new AlterTableBuilder(table)
.RenameColumn("c0", "c0Key"));
var exception = await Assert.ThrowsAsync <NonRecoverableException>(async() =>
{
// Scanning with the old schema.
var scanner = _client.NewScanBuilder(table)
.SetProjectedColumns("c0", "c1")
.Build();
await foreach (var resultSet in scanner)
{
}
});
Assert.True(exception.Status.IsInvalidArgument);
Assert.Contains(
"Some columns are not present in the current schema: c0",
exception.Status.Message);
// Reopen table for the new schema.
table = await _client.OpenTableAsync(_tableName);
Assert.Equal("c0Key", table.Schema.GetColumn(0).Name);
Assert.Equal(2, table.Schema.Columns.Count);
// Add a row
var insert = table.NewInsert();
insert.SetInt32("c0Key", 101);
insert.SetInt32("c1", 101);
await _session.EnqueueAsync(insert);
await _session.FlushAsync();
var scanner2 = _client.NewScanBuilder(table)
.SetProjectedColumns("c0Key", "c1")
.Build();
var rows = await scanner2.ScanToListAsync <(int c0Key, int c1)>();
Assert.Equal(101, rows.Count);
Assert.All(rows, row => Assert.Equal(row.c0Key, row.c1));
}
public async Task TestAlterAddColumns()
{
KuduTable table = await CreateTableAsync();
await InsertRowsAsync(table, 0, 100);
Assert.Equal(100, await ClientTestUtil.CountRowsAsync(_client, table));
await _client.AlterTableAsync(new AlterTableBuilder(table)
.AddColumn("addNonNull", KuduType.Int32, opt => opt
.Nullable(false)
.DefaultValue(100))
.AddColumn("addNullable", KuduType.Int32)
.AddColumn("addNullableDef", KuduType.Int32, opt => opt
.DefaultValue(200)));
// Reopen table for the new schema.
table = await _client.OpenTableAsync(_tableName);
Assert.Equal(5, table.Schema.Columns.Count);
// Add a row with addNullableDef=null
var insert = table.NewInsert();
insert.SetInt32("c0", 101);
insert.SetInt32("c1", 101);
insert.SetInt32("addNonNull", 101);
insert.SetInt32("addNullable", 101);
insert.SetNull("addNullableDef");
await _session.EnqueueAsync(insert);
await _session.FlushAsync();
// Check defaults applied, and that row key=101
var results = await ClientTestUtil.ScanTableToStringsAsync(_client, table);
var expected = new List <string>(101);
for (int i = 0; i < 100; i++)
{
expected.Add($"INT32 c0={i}, INT32 c1={i}, INT32 addNonNull=100, " +
"INT32 addNullable=NULL, INT32 addNullableDef=200");
}
expected.Add("INT32 c0=101, INT32 c1=101, INT32 addNonNull=101, " +
"INT32 addNullable=101, INT32 addNullableDef=NULL");
Assert.Equal(
expected.OrderBy(r => r),
results.OrderBy(r => r));
}
public async Task TestAlterExtraConfigs()
{
KuduTable table = await CreateTableAsync();
await InsertRowsAsync(table, 0, 100);
Assert.Equal(100, await ClientTestUtil.CountRowsAsync(_client, table));
// 1. Check for expected defaults.
table = await _client.OpenTableAsync(_tableName);
Assert.DoesNotContain("kudu.table.history_max_age_sec", table.ExtraConfig);
// 2. Alter history max age second to 3600
var alterExtraConfigs = new Dictionary <string, string>
{
{ "kudu.table.history_max_age_sec", "3600" }
};
await _client.AlterTableAsync(new AlterTableBuilder(table)
.AlterExtraConfigs(alterExtraConfigs));
table = await _client.OpenTableAsync(_tableName);
Assert.Equal("3600", table.ExtraConfig["kudu.table.history_max_age_sec"]);
// 3. Alter history max age second to 7200
alterExtraConfigs = new Dictionary <string, string>
{
{ "kudu.table.history_max_age_sec", "7200" }
};
await _client.AlterTableAsync(new AlterTableBuilder(table)
.AlterExtraConfigs(alterExtraConfigs));
table = await _client.OpenTableAsync(_tableName);
Assert.Equal("7200", table.ExtraConfig["kudu.table.history_max_age_sec"]);
// 4. Reset history max age second to default
alterExtraConfigs = new Dictionary <string, string>
{
{ "kudu.table.history_max_age_sec", "" }
};
await _client.AlterTableAsync(new AlterTableBuilder(table)
.AlterExtraConfigs(alterExtraConfigs));
table = await _client.OpenTableAsync(_tableName);
Assert.Empty(table.ExtraConfig);
}
public async Task TestGetTableStatistics()
{
await using var harness = await new MiniKuduClusterBuilder()
.AddTabletServerFlag("--update_tablet_stats_interval_ms=200")
.AddTabletServerFlag("--heartbeat_interval_ms=100")
.BuildHarnessAsync();
await using var client = harness.CreateClient();
// Create a table.
var builder = ClientTestUtil.GetBasicSchema().SetTableName(_tableName);
var table = await client.CreateTableAsync(builder);
// Insert some rows and test the statistics.
var prevStatistics = new KuduTableStatistics(-1, -1);
var currentStatistics = new KuduTableStatistics(-1, -1);
var session = client.NewSession();
int num = 100;
for (int i = 0; i < num; ++i)
{
// Get current table statistics.
currentStatistics = await client.GetTableStatisticsAsync(_tableName);
Assert.True(currentStatistics.OnDiskSize >= prevStatistics.OnDiskSize);
Assert.True(currentStatistics.LiveRowCount >= prevStatistics.LiveRowCount);
Assert.True(currentStatistics.LiveRowCount <= i + 1);
prevStatistics = currentStatistics;
// Insert row.
var insert = ClientTestUtil.CreateBasicSchemaInsert(table, i);
await session.EnqueueAsync(insert);
await session.FlushAsync();
long numRows = await ClientTestUtil.CountRowsAsync(client, table);
Assert.Equal(i + 1, numRows);
}
// Final accuracy test.
// Wait for master to aggregate table statistics.
await Task.Delay(200 * 6);
currentStatistics = await client.GetTableStatisticsAsync(_tableName);
Assert.True(currentStatistics.OnDiskSize >= prevStatistics.OnDiskSize);
Assert.True(currentStatistics.LiveRowCount >= prevStatistics.LiveRowCount);
Assert.Equal(num, currentStatistics.LiveRowCount);
}
public async Task TestAuthzTokensDuringElection()
{
await using var client = _harness.CreateClient();
await using var session = client.NewSession();
// Test sending various requests that require authorization.
var builder = ClientTestUtil.GetBasicSchema()
.SetTableName(_tableName)
.CreateBasicRangePartition()
.SetNumReplicas(1);
var table = await client.CreateTableAsync(builder);
// Restart the masters to trigger an election.
await _harness.KillAllMasterServersAsync();
await _harness.StartAllMasterServersAsync();
int numReqs = 10;
await InsertRowsAsync(session, table, 0, numReqs);
await session.FlushAsync();
// Do the same for batches of inserts.
await _harness.KillAllMasterServersAsync();
await _harness.StartAllMasterServersAsync();
await InsertRowsAsync(session, table, numReqs, numReqs);
await session.FlushAsync();
// And for scans.
await _harness.KillAllMasterServersAsync();
await _harness.StartAllMasterServersAsync();
for (int i = 0; i < numReqs; i++)
{
var numRows = await ClientTestUtil.CountRowsAsync(client, table);
Assert.Equal(2 * numReqs, numRows);
}
}
public async Task Test()
{
var testRuntime = TimeSpan.FromMinutes(1);
using var cts = new CancellationTokenSource(testRuntime);
var token = cts.Token;
var chaosTask = RunTaskAsync(() => DoChaosAsync(token), cts);
var writeTask = RunTaskAsync(() => WriteAsync(token), cts);
var scanTask = RunTaskAsync(() => ScanAsync(token), cts);
await Task.WhenAll(chaosTask, writeTask, scanTask);
// If the test passed, do some extra validation at the end.
var rowCount = await ClientTestUtil.CountRowsAsync(_client, _table);
Assert.True(rowCount > 0);
}
public async Task TestAlterRangePartitioningExclusiveInclusive()
{
// Create initial table with single range partition covering (-1, 99].
var builder = new TableBuilder(_tableName)
.SetNumReplicas(1)
.AddColumn("c0", KuduType.Int32, opt => opt.Key(true))
.AddColumn("c1", KuduType.Int32, opt => opt.Nullable(false))
.SetRangePartitionColumns("c0")
.AddRangePartition((lower, upper) =>
{
lower.SetInt32("c0", -1);
upper.SetInt32("c0", 99);
}, RangePartitionBound.Exclusive, RangePartitionBound.Inclusive);
KuduTable table = await _client.CreateTableAsync(builder);
await _client.AlterTableAsync(new AlterTableBuilder(table)
.AddRangePartition((lower, upper) =>
{
lower.SetInt32("c0", 199);
upper.SetInt32("c0", 299);
}, RangePartitionBound.Exclusive, RangePartitionBound.Inclusive));
// Insert some rows, and then drop the partition and ensure that the table is empty.
await InsertRowsAsync(table, 0, 100);
await InsertRowsAsync(table, 200, 300);
Assert.Equal(200, await ClientTestUtil.CountRowsAsync(_client, table));
await _client.AlterTableAsync(new AlterTableBuilder(table)
.DropRangePartition((lower, upper) =>
{
lower.SetInt32("c0", 0);
upper.SetInt32("c0", 100);
}, RangePartitionBound.Inclusive, RangePartitionBound.Exclusive)
.DropRangePartition((lower, upper) =>
{
lower.SetInt32("c0", 199);
upper.SetInt32("c0", 299);
}, RangePartitionBound.Exclusive, RangePartitionBound.Inclusive));
Assert.Equal(0, await ClientTestUtil.CountRowsAsync(_client, table));
}
public async Task TestFailover(bool restart)
{
await using var harness = await new MiniKuduClusterBuilder()
.NumMasters(3)
.NumTservers(3)
.BuildHarnessAsync();
await using var client = harness.CreateClient();
var builder = ClientTestUtil.GetBasicSchema()
.SetTableName("LeaderFailoverTest")
.CreateBasicRangePartition();
var table = await client.CreateTableAsync(builder);
var rows = Enumerable.Range(0, 3)
.Select(i => ClientTestUtil.CreateBasicSchemaInsert(table, i));
await client.WriteAsync(rows);
// Make sure the rows are in there before messing things up.
long numRows = await ClientTestUtil.CountRowsAsync(client, table);
Assert.Equal(3, numRows);
if (restart)
{
await harness.RestartLeaderMasterAsync();
}
else
{
await harness.KillLeaderMasterServerAsync();
}
var rows2 = Enumerable.Range(3, 3)
.Select(i => ClientTestUtil.CreateBasicSchemaInsert(table, i));
await client.WriteAsync(rows2);
long numRows2 = await ClientTestUtil.CountRowsAsync(client, table);
Assert.Equal(6, numRows2);
}
public async Task TestAuthzTokenExpiration()
{
// Test a long-running concurrent workload with different types of requests
// being sent, all the while injecting invalid tokens, with a short authz
// token expiration time. The threads should reacquire tokens as needed
// without surfacing token errors to the client.
await using var client = _harness.CreateClientBuilder()
.SetDefaultOperationTimeout(TimeSpan.FromMinutes(1))
.Build();
using var cts = new CancellationTokenSource();
var stopToken = cts.Token;
var builder = ClientTestUtil.GetBasicSchema()
.SetTableName(_tableName)
.CreateBasicRangePartition()
.SetNumReplicas(1);
var table = await client.CreateTableAsync(builder);
cts.CancelAfter(TimeSpan.FromSeconds(10));
int rowStart = 0;
var loop1 = WriteRowsAsync(flush: true);
var loop2 = WriteRowsAsync(flush: false);
var scanTask = ScanTableAsync();
await loop1;
await loop2;
await scanTask;
var expected = Interlocked.Add(ref rowStart, 0) * 10;
var rows = await ClientTestUtil.CountRowsAsync(client, table);
Assert.Equal(expected, rows);
async Task WriteRowsAsync(bool flush)
{
await using var session = client.NewSession();
while (!stopToken.IsCancellationRequested)
{
var start = Interlocked.Increment(ref rowStart) * 10;
await InsertRowsAsync(session, table, start, 10);
if (flush)
{
await session.FlushAsync();
}
}
}
async Task ScanTableAsync()
{
while (!stopToken.IsCancellationRequested)
{
// We can't validate the row count until the end, but this
// still ensures the scanner doesn't throw any exceptions.
await ClientTestUtil.CountRowsAsync(client, table);
}
}
}
public async Task TestAlterRangeParitioningInvalid()
{
// Create initial table with single range partition covering [0, 100).
KuduTable table = await CreateTableAsync((0, 100));
await InsertRowsAsync(table, 0, 100);
Assert.Equal(100, await ClientTestUtil.CountRowsAsync(_client, table));
// ADD [0, 100) <- already present (duplicate)
var ex = await Assert.ThrowsAsync <NonRecoverableException>(async() =>
{
await _client.AlterTableAsync(new AlterTableBuilder(table)
.AddRangePartition((lower, upper) =>
{
lower.SetInt32("c0", 0);
upper.SetInt32("c0", 100);
}));
});
Assert.True(ex.Status.IsAlreadyPresent);
Assert.Contains("range partition already exists", ex.Status.Message);
Assert.Equal(100, await ClientTestUtil.CountRowsAsync(_client, table));
// ADD [50, 150) <- illegal (overlap)
ex = await Assert.ThrowsAsync <NonRecoverableException>(async() =>
{
await _client.AlterTableAsync(new AlterTableBuilder(table)
.AddRangePartition((lower, upper) =>
{
lower.SetInt32("c0", 50);
upper.SetInt32("c0", 150);
}));
});
Assert.True(ex.Status.IsInvalidArgument);
Assert.Contains("new range partition conflicts with existing one", ex.Status.Message);
Assert.Equal(100, await ClientTestUtil.CountRowsAsync(_client, table));
// ADD [-50, 50) <- illegal (overlap)
ex = await Assert.ThrowsAsync <NonRecoverableException>(async() =>
{
await _client.AlterTableAsync(new AlterTableBuilder(table)
.AddRangePartition((lower, upper) =>
{
lower.SetInt32("c0", -50);
upper.SetInt32("c0", 50);
}));
});
Assert.True(ex.Status.IsInvalidArgument);
Assert.Contains("new range partition conflicts with existing one", ex.Status.Message);
Assert.Equal(100, await ClientTestUtil.CountRowsAsync(_client, table));
// ADD [200, 300)
// ADD [-50, 150) <- illegal (overlap)
ex = await Assert.ThrowsAsync <NonRecoverableException>(async() =>
{
await _client.AlterTableAsync(new AlterTableBuilder(table)
.AddRangePartition((lower, upper) =>
{
lower.SetInt32("c0", 200);
upper.SetInt32("c0", 300);
})
.AddRangePartition((lower, upper) =>
{
lower.SetInt32("c0", -50);
upper.SetInt32("c0", 150);
}));
});
Assert.True(ex.Status.IsInvalidArgument);
Assert.Contains("new range partition conflicts with existing one", ex.Status.Message);
Assert.Equal(100, await ClientTestUtil.CountRowsAsync(_client, table));
// DROP [<start>, <end>)
ex = await Assert.ThrowsAsync <NonRecoverableException>(async() =>
{
await _client.AlterTableAsync(new AlterTableBuilder(table)
.DropRangePartition((lower, upper) => { }));
});
Assert.True(ex.Status.IsInvalidArgument);
Assert.Contains("no range partition to drop", ex.Status.Message);
Assert.Equal(100, await ClientTestUtil.CountRowsAsync(_client, table));
// DROP [50, 150)
// RENAME foo
ex = await Assert.ThrowsAsync <NonRecoverableException>(async() =>
{
await _client.AlterTableAsync(new AlterTableBuilder(table)
.DropRangePartition((lower, upper) =>
{
lower.SetInt32("c0", 50);
upper.SetInt32("c0", 150);
})
.RenameTable("foo"));
});
Assert.True(ex.Status.IsInvalidArgument);
Assert.Contains("no range partition to drop", ex.Status.Message);
Assert.Equal(100, await ClientTestUtil.CountRowsAsync(_client, table));
Assert.Empty(await _client.GetTablesAsync("foo"));
// DROP [0, 100)
// ADD [100, 200)
// DROP [100, 200)
// ADD [150, 250)
// DROP [0, 10) <- illegal
ex = await Assert.ThrowsAsync <NonRecoverableException>(async() =>
{
await _client.AlterTableAsync(new AlterTableBuilder(table)
.DropRangePartition((lower, upper) =>
{
lower.SetInt32("c0", 0);
upper.SetInt32("c0", 100);
})
.AddRangePartition((lower, upper) =>
{
lower.SetInt32("c0", 100);
upper.SetInt32("c0", 200);
})
.DropRangePartition((lower, upper) =>
{
lower.SetInt32("c0", 100);
upper.SetInt32("c0", 200);
})
.AddRangePartition((lower, upper) =>
{
lower.SetInt32("c0", 150);
upper.SetInt32("c0", 250);
})
.DropRangePartition((lower, upper) =>
{
lower.SetInt32("c0", 0);
upper.SetInt32("c0", 10);
}));
});
Assert.True(ex.Status.IsInvalidArgument);
Assert.Contains("no range partition to drop", ex.Status.Message);
Assert.Equal(100, await ClientTestUtil.CountRowsAsync(_client, table));
}
public async Task TestAlterRangePartitioning()
{
KuduTable table = await CreateTableAsync();
KuduSchema schema = table.Schema;
// Insert some rows, and then drop the partition and ensure that the table is empty.
await InsertRowsAsync(table, 0, 100);
Assert.Equal(100, await ClientTestUtil.CountRowsAsync(_client, table));
await _client.AlterTableAsync(new AlterTableBuilder(table)
.DropRangePartition((lower, upper) => { }));
Assert.Equal(0, await ClientTestUtil.CountRowsAsync(_client, table));
// Add new range partition and insert rows.
await _client.AlterTableAsync(new AlterTableBuilder(table)
.AddRangePartition((lower, upper) =>
{
lower.SetInt32("c0", 0);
upper.SetInt32("c0", 100);
}));
await InsertRowsAsync(table, 0, 100);
Assert.Equal(100, await ClientTestUtil.CountRowsAsync(_client, table));
// Replace the range partition with a different one.
await _client.AlterTableAsync(new AlterTableBuilder(table)
.DropRangePartition((lower, upper) =>
{
lower.SetInt32("c0", 0);
upper.SetInt32("c0", 100);
})
.AddRangePartition((lower, upper) =>
{
lower.SetInt32("c0", 50);
upper.SetInt32("c0", 150);
}));
Assert.Equal(0, await ClientTestUtil.CountRowsAsync(_client, table));
await InsertRowsAsync(table, 50, 125);
Assert.Equal(75, await ClientTestUtil.CountRowsAsync(_client, table));
// Replace the range partition with the same one.
await _client.AlterTableAsync(new AlterTableBuilder(table)
.DropRangePartition((lower, upper) =>
{
lower.SetInt32("c0", 50);
upper.SetInt32("c0", 150);
})
.AddRangePartition((lower, upper) =>
{
lower.SetInt32("c0", 50);
upper.SetInt32("c0", 150);
}));
Assert.Equal(0, await ClientTestUtil.CountRowsAsync(_client, table));
await InsertRowsAsync(table, 50, 125);
Assert.Equal(75, await ClientTestUtil.CountRowsAsync(_client, table));
// Alter table partitioning + alter table schema
var newTableName = $"{_tableName}-renamed";
await _client.AlterTableAsync(new AlterTableBuilder(table)
.AddRangePartition((lower, upper) =>
{
lower.SetInt32("c0", 200);
upper.SetInt32("c0", 300);
})
.RenameTable(newTableName)
.AddColumn("c2", KuduType.Int32));
await InsertRowsAsync(table, 200, 300);
Assert.Equal(175, await ClientTestUtil.CountRowsAsync(_client, table));
Assert.Equal(3, (await _client.OpenTableAsync(newTableName)).Schema.Columns.Count);
// Drop all range partitions + alter table schema. This also serves to test
// specifying range bounds with a subset schema (since a column was
// previously added).
await _client.AlterTableAsync(new AlterTableBuilder(table)
.DropRangePartition((lower, upper) =>
{
lower.SetInt32("c0", 200);
upper.SetInt32("c0", 300);
})
.DropRangePartition((lower, upper) =>
{
lower.SetInt32("c0", 50);
upper.SetInt32("c0", 150);
})
.DropColumn("c2"));
Assert.Equal(0, await ClientTestUtil.CountRowsAsync(_client, table));
Assert.Equal(2, (await _client.OpenTableAsync(newTableName)).Schema.Columns.Count);
}
public async Task TestMultipleFailover(bool restart)
{
int rowsPerIteration = 3;
int numIterations = 10;
int totalRowsToInsert = rowsPerIteration + numIterations * rowsPerIteration;
await using var harness = await new MiniKuduClusterBuilder()
.NumMasters(3)
.NumTservers(3)
.BuildHarnessAsync();
await using var client = harness.CreateClient();
var builder = ClientTestUtil.GetBasicSchema()
.SetTableName("MultipleLeaderFailoverTest");
var table = await client.CreateTableAsync(builder);
await using var session = client.NewSession();
for (int i = 0; i < rowsPerIteration; i++)
{
var row = ClientTestUtil.CreateBasicSchemaInsert(table, i);
await session.EnqueueAsync(row);
}
await session.FlushAsync();
var rowCount = await ClientTestUtil.CountRowsAsync(client, table);
Assert.Equal(rowsPerIteration, rowCount);
int currentRows = rowsPerIteration;
for (int i = 0; i < numIterations; i++)
{
var tablets = await client.GetTableLocationsAsync(
table.TableId, Array.Empty <byte>(), 1);
Assert.Single(tablets);
if (restart)
{
await harness.RestartTabletServerAsync(tablets[0]);
}
else
{
await harness.KillTabletLeaderAsync(tablets[0]);
}
for (int j = 0; j < rowsPerIteration; j++)
{
var row = ClientTestUtil.CreateBasicSchemaInsert(table, currentRows);
await session.EnqueueAsync(row);
currentRows++;
}
await session.FlushAsync();
if (!restart)
{
await harness.StartAllTabletServersAsync();
}
rowCount = await ClientTestUtil.CountRowsAsync(client, table);
Assert.Equal(currentRows, rowCount);
}
rowCount = await ClientTestUtil.CountRowsAsync(client, table);
Assert.Equal(totalRowsToInsert, rowCount);
}
