public async Task TestInsertIgnore()
{
var builder = ClientTestUtil.GetBasicSchema()
.SetTableName(nameof(TestInsertIgnore));
var table = await _client.CreateTableAsync(builder);
// Test insert ignore implements normal insert.
var insert = ClientTestUtil.CreateBasicSchemaInsertIgnore(table, 1);
await _client.WriteAsync(new[] { insert });
var rowStrings = await ClientTestUtil.ScanTableToStringsAsync(_client, table);
var rowString = Assert.Single(rowStrings);
Assert.Equal(
"INT32 key=1, INT32 column1_i=2, INT32 column2_i=3, " +
"STRING column3_s=a string, BOOL column4_b=True", rowString);
// Test insert ignore does not return a row error.
await _client.WriteAsync(new[] { insert });
rowStrings = await ClientTestUtil.ScanTableToStringsAsync(_client, table);
rowString = Assert.Single(rowStrings);
Assert.Equal(
"INT32 key=1, INT32 column1_i=2, INT32 column2_i=3, " +
"STRING column3_s=a string, BOOL column4_b=True", rowString);
}
public async Task TestSubmitWriteOpAfterCommit()
{
await using var harness = await new MiniKuduClusterBuilder()
.AddMasterServerFlag("--txn_manager_enabled")
.AddTabletServerFlag("--enable_txn_system_client_init=true")
.BuildHarnessAsync();
await using var client = harness.CreateClient();
var builder = ClientTestUtil.GetBasicSchema()
.SetTableName(nameof(TestSubmitWriteOpAfterCommit))
.AddHashPartitions(2, "key");
var table = await client.CreateTableAsync(builder);
using var transaction = await client.NewTransactionAsync();
int key = 0;
var insert1 = ClientTestUtil.CreateBasicSchemaInsert(table, key++);
await transaction.WriteAsync(new[] { insert1 });
await transaction.CommitAsync();
await transaction.WaitForCommitAsync();
var insert2 = ClientTestUtil.CreateBasicSchemaInsert(table, key);
var exception = await Assert.ThrowsAsync <NonRecoverableException>(
async() => await transaction.WriteAsync(new[] { insert2 }));
Assert.Matches(".* transaction ID .* not open: COMMITTED", exception.Message);
}
public async Task TestOpenScanWithDroppedPartition()
{
await using var miniCluster = await new MiniKuduClusterBuilder().BuildAsync();
await using var client = miniCluster.CreateClient();
var builder = ClientTestUtil.GetBasicSchema()
.SetTableName("TestOpenScanWithDroppedPartition")
.CreateBasicRangePartition()
.AddRangePartition((lower, upper) =>
{
lower.SetInt32("key", 0);
upper.SetInt32("key", 1000);
})
.AddRangePartition((lower, upper) =>
{
lower.SetInt32("key", 1000);
upper.SetInt32("key", 2000);
});
var table = await client.CreateTableAsync(builder);
// Load rows into both partitions.
int numRows = 1999;
await ClientTestUtil.LoadDefaultTableAsync(client, table, numRows);
// Scan the rows while dropping a partition.
var scanner = client.NewScanBuilder(table)
// Set a small batch size so the first scan doesn't read all the rows.
.SetBatchSizeBytes(100)
.SetReadMode(ReadMode.ReadYourWrites)
.Build();
long rowsScanned = 0;
int batchNum = 0;
await foreach (var resultSet in scanner)
{
if (batchNum == 1)
{
// Drop the partition.
await client.AlterTableAsync(new AlterTableBuilder(table)
.DropRangePartition((lower, upper) =>
{
lower.SetInt32("key", 0);
upper.SetInt32("key", 1000);
}));
// Give time for the background drop operations.
await Task.Delay(1000);
// TODO: Verify the partition was dropped.
}
rowsScanned += resultSet.Count;
batchNum++;
}
Assert.True(batchNum > 1);
Assert.Equal(numRows, rowsScanned);
}
public async Task TestTimeoutEvenWhenServerHangs()
{
await using var harness = await new MiniKuduClusterBuilder()
.AddTabletServerFlag("--scanner_inject_latency_on_each_batch_ms=200000")
.BuildHarnessAsync();
await using var client = harness.CreateClient();
var tableBuilder = ClientTestUtil.GetBasicSchema()
.SetTableName(nameof(TestTimeoutEvenWhenServerHangs));
var table = await client.CreateTableAsync(tableBuilder);
var row = ClientTestUtil.CreateBasicSchemaInsert(table, 1);
await client.WriteAsync(new[] { row });
var scanner = client.NewScanBuilder(table).Build();
// Scan with a short timeout.
var timeout = TimeSpan.FromSeconds(1);
using var cts = new CancellationTokenSource(timeout);
// The server will not respond for the lifetime of the test, so we
// expect the operation to time out.
await Assert.ThrowsAsync <OperationCanceledException>(
async() => await scanner.CountAsync(cts.Token));
}
public async Task TestDiffScanTokensConcurrentColumnRename()
{
var builder = ClientTestUtil.GetBasicSchema()
.SetTableName(_tableName)
.SetNumReplicas(1);
var table = await _client.CreateTableAsync(builder);
// Set up the table for a diff scan.
int numRows = 20;
long timestamp = await SetupTableForDiffScansAsync(table, numRows);
// Since the diff scan interval is [start, end), increment the start timestamp
// to exclude the last row inserted in the first group of ops, and increment the
// end timestamp to include the last row deleted in the second group of ops.
List <KuduScanToken> tokens = await _client.NewScanTokenBuilder(table)
// TODO(KUDU-3146): Disable including the table metadata so the new column
// name is retrieved when deserializing the scanner.
.IncludeTableMetadata(false)
.DiffScan(timestamp + 1, _client.LastPropagatedTimestamp + 1)
.BuildAsync();
var token = Assert.Single(tokens);
// Rename a column between when the token is created and when it is
// rehydrated into a scanner
await _client.AlterTableAsync(new AlterTableBuilder(table)
.RenameColumn("column1_i", "column1_i_new"));
var scanBuilder = await _client.NewScanBuilderFromTokenAsync(token);
var scanner = scanBuilder.Build();
await CheckDiffScanResultsAsync(scanner, 3 *numRows / 4, numRows / 4);
}
public async Task TestInsertAfterInsertIgnoreHasRowError()
{
var builder = ClientTestUtil.GetBasicSchema()
.SetTableName(nameof(TestInsertAfterInsertIgnoreHasRowError));
var table = await _client.CreateTableAsync(builder);
var rows = new[]
{
ClientTestUtil.CreateBasicSchemaInsertIgnore(table, 1),
ClientTestUtil.CreateBasicSchemaInsert(table, 1)
};
var exception = await Assert.ThrowsAsync <KuduWriteException>(
() => _client.WriteAsync(rows));
var rowError = Assert.Single(exception.PerRowErrors);
Assert.True(rowError.IsAlreadyPresent);
var rowStrings = await ClientTestUtil.ScanTableToStringsAsync(_client, table);
var rowString = Assert.Single(rowStrings);
Assert.Equal(
"INT32 key=1, INT32 column1_i=2, INT32 column2_i=3, " +
"STRING column3_s=a string, BOOL column4_b=True", rowString);
}
private async Task <long> SetupTableForDiffScansAsync(KuduTable table, int numRows)
{
for (int i = 0; i < numRows / 2; i++)
{
var row = ClientTestUtil.CreateBasicSchemaInsert(table, i);
await _session.EnqueueAsync(row);
}
await _session.FlushAsync();
// Grab the timestamp, then add more data so there's a diff.
long timestamp = _client.LastPropagatedTimestamp;
for (int i = numRows / 2; i < numRows; i++)
{
var row = ClientTestUtil.CreateBasicSchemaInsert(table, i);
await _session.EnqueueAsync(row);
}
await _session.FlushAsync();
// Delete some data so the is_deleted column can be tested.
for (int i = 0; i < numRows / 4; i++)
{
var row = table.NewDelete();
row.SetInt32(0, i);
await _session.EnqueueAsync(row);
}
await _session.FlushAsync();
return(timestamp);
}
public async Task TestCount()
{
await using var miniCluster = await new MiniKuduClusterBuilder().BuildAsync();
await using var client = miniCluster.CreateClient();
var builder = ClientTestUtil.GetBasicSchema()
.SetTableName(nameof(TestCount))
.AddHashPartitions(4, "key");
var table = await client.CreateTableAsync(builder);
int numRows = 123;
var rows = Enumerable.Range(0, numRows)
.Select(i => ClientTestUtil.CreateBasicSchemaInsert(table, i));
await client.WriteAsync(rows);
var scanner = client.NewScanBuilder(table)
.SetEmptyProjection()
.SetReadMode(ReadMode.ReadYourWrites)
.Build();
long numScannedRows = await scanner.CountAsync();
Assert.Empty(scanner.ProjectionSchema.Columns);
Assert.Equal(numRows, numScannedRows);
}
public async Task InitializeAsync()
{
_harness = await new MiniKuduClusterBuilder().BuildHarnessAsync();
var options = _harness.CreateClientBuilder().BuildOptions();
var securityContext = new SecurityContext();
var systemClock = new SystemClock();
var connectionFactory = new KuduConnectionFactory(
options, securityContext, NullLoggerFactory.Instance);
_testConnectionFactory = new TestConnectionFactory(connectionFactory);
_client = new KuduClient(
options,
securityContext,
_testConnectionFactory,
systemClock,
NullLoggerFactory.Instance);
var builder = ClientTestUtil.GetBasicSchema()
.SetTableName("chaos_test_table")
.SetNumReplicas(3)
.CreateBasicRangePartition();
_table = await _client.CreateTableAsync(builder);
}
public async Task TestScanTokenRequestsWithMetadata(
bool includeTableMetadata, bool includeTabletMetadata)
{
var builder = ClientTestUtil.GetBasicSchema()
.SetTableName(_tableName)
.SetNumReplicas(3);
var table = await _client.CreateTableAsync(builder);
await ClientTestUtil.LoadDefaultTableAsync(_client, table, 1);
var tokens = await _client.NewScanTokenBuilder(table)
.IncludeTableMetadata(includeTableMetadata)
.IncludeTabletMetadata(includeTabletMetadata)
.BuildAsync();
var token = Assert.Single(tokens);
// Use a new client to simulate hydrating in a new process.
var newClient = _harness.CreateClient();
var scanBuilder = await newClient.NewScanBuilderFromTokenAsync(token.Serialize());
var scanner = scanBuilder.Build();
Assert.Equal(1, await scanner.CountAsync());
}
public async Task TestAlterModifyColumns()
{
KuduTable table = await CreateTableAsync();
await InsertRowsAsync(table, 0, 100);
Assert.Equal(100, await ClientTestUtil.CountRowsAsync(_client, table));
// Check for expected defaults.
ColumnSchema col = table.Schema.GetColumn(1);
Assert.Equal(CompressionType.DefaultCompression, col.Compression);
Assert.Equal(EncodingType.AutoEncoding, col.Encoding);
Assert.Null(col.DefaultValue);
// Alter the table.
await _client.AlterTableAsync(new AlterTableBuilder(table)
.ChangeCompressionAlgorithm(col.Name, CompressionType.Snappy)
.ChangeEncoding(col.Name, EncodingType.Rle)
.ChangeDefault(col.Name, 0));
// Check for new values.
table = await _client.OpenTableAsync(_tableName);
col = table.Schema.GetColumn(1);
Assert.Equal(CompressionType.Snappy, col.Compression);
Assert.Equal(EncodingType.Rle, col.Encoding);
Assert.Equal(0, col.DefaultValue);
}
private async Task WriteAsync(CancellationToken cancellationToken)
{
Exception sessionException = null;
Exception exception;
ValueTask HandleSessionExceptionAsync(SessionExceptionContext context)
{
Volatile.Write(ref sessionException, context.Exception);
return(new ValueTask());
}
var options = new KuduSessionOptions
{
ExceptionHandler = HandleSessionExceptionAsync
};
await using var session = _client.NewSession(options);
int currentRowKey = 0;
bool flush = false;
while (true)
{
if (cancellationToken.IsCancellationRequested)
{
break;
}
exception = Volatile.Read(ref sessionException);
if (exception != null)
{
throw exception;
}
var row = ClientTestUtil.CreateBasicSchemaInsert(_table, currentRowKey);
await session.EnqueueAsync(row, CancellationToken.None);
if (flush)
{
await session.FlushAsync(CancellationToken.None);
}
// Every 10 rows we flush and change the flush mode randomly.
if (currentRowKey % 10 == 0)
{
flush = ThreadSafeRandom.Instance.NextBool();
}
currentRowKey++;
}
await session.FlushAsync(CancellationToken.None);
exception = Volatile.Read(ref sessionException);
if (exception != null)
{
throw exception;
}
}
public async Task TestIterable()
{
await using var miniCluster = await new MiniKuduClusterBuilder().BuildAsync();
await using var client = miniCluster.CreateClient();
await using var session = client.NewSession();
var builder = ClientTestUtil.GetBasicSchema()
.SetTableName("TestIterable")
.CreateBasicRangePartition();
var table = await client.CreateTableAsync(builder);
IDictionary <int, PartialRow> inserts = new Dictionary <int, PartialRow>();
int numRows = 10;
for (int i = 0; i < numRows; i++)
{
var insert = table.NewInsert();
_generator.RandomizeRow(insert);
inserts.TryAdd(insert.GetInt32(0), insert);
await session.EnqueueAsync(insert);
}
await session.FlushAsync();
var scanner = client.NewScanBuilder(table)
.SetReadMode(ReadMode.ReadYourWrites)
.SetReplicaSelection(ReplicaSelection.LeaderOnly)
.Build();
await foreach (var resultSet in scanner)
{
foreach (var row in resultSet)
{
var key = row.GetInt32(0);
var insert = Assert.Contains(key, inserts);
Assert.Equal(insert.GetInt32(1), row.GetInt32(1));
Assert.Equal(insert.GetInt32(2), row.GetInt32(2));
if (insert.IsNull(3))
{
Assert.True(row.IsNull(3));
}
else
{
Assert.Equal(insert.GetString(3), row.GetString(3));
}
Assert.Equal(insert.GetBool(4), row.GetBool(4));
inserts.Remove(key);
}
}
Assert.Empty(inserts);
}
public async Task TestTxnKeepaliveSwitchesToOtherTxnManager()
{
await using var harness = await new MiniKuduClusterBuilder()
.AddMasterServerFlag("--txn_manager_enabled")
// Set Raft heartbeat interval short for faster test runtime: speed up
// leader failure detection and new leader election.
.AddMasterServerFlag("--raft_heartbeat_interval_ms=100")
// The txn keepalive interval should be long enough to accommodate Raft
// leader failure detection and election.
.AddTabletServerFlag("--txn_keepalive_interval_ms=1000")
.AddTabletServerFlag("--txn_staleness_tracker_interval_ms=250")
.AddTabletServerFlag("--enable_txn_system_client_init=true")
.BuildHarnessAsync();
await using var client = harness.CreateClient();
var builder = ClientTestUtil.GetBasicSchema()
.SetTableName(nameof(TestTxnKeepaliveSwitchesToOtherTxnManager))
.AddHashPartitions(2, "key");
var table = await client.CreateTableAsync(builder);
using var transaction = await client.NewTransactionAsync();
var insert = ClientTestUtil.CreateBasicSchemaInsert(table, 0);
await transaction.WriteAsync(new[] { insert });
await harness.KillLeaderMasterServerAsync();
// Wait for two keepalive intervals to make sure the backend got a chance
// to automatically abort the transaction if not receiving txn keepalive
// messages.
await Task.Delay(2 * 1000);
// It should be possible to commit the transaction. This is to verify that
//
// * the client eventually starts sending txn keepalive messages to other
// TxnManager instance (the original was hosted by former leader master
// which is no longer available), so the backend doesn't abort the
// transaction automatically due to not receiving keepalive messages
//
// * the client switches to the new TxnManager for other txn-related
// operations as well
await transaction.CommitAsync();
await transaction.WaitForCommitAsync();
// An extra sanity check: read back the rows written into the table in the
// context of the transaction.
var scanner = client.NewScanBuilder(table)
.SetReadMode(ReadMode.ReadYourWrites)
.SetReplicaSelection(ReplicaSelection.LeaderOnly)
.Build();
Assert.Equal(1, await scanner.CountAsync());
}
public async Task TestPropagateTxnCommitTimestamp()
{
await using var harness = await new MiniKuduClusterBuilder()
.AddMasterServerFlag("--txn_manager_enabled")
// Inject latency to have a chance spotting the transaction in the
// FINALIZE_IN_PROGRESS state and make KuduTransaction.WaitForCommitAsync()
// have to poll multiple times.
.AddTabletServerFlag("--txn_status_manager_inject_latency_finalize_commit_ms=250")
.AddTabletServerFlag("--enable_txn_system_client_init=true")
.BuildHarnessAsync();
await using var client = harness.CreateClient();
var builder = ClientTestUtil.GetBasicSchema()
.SetTableName(nameof(TestPropagateTxnCommitTimestamp))
.AddHashPartitions(8, "key");
var table = await client.CreateTableAsync(builder);
// Make sure the commit timestamp for a transaction is propagated to the
// client upon committing a transaction.
using (var transaction = await client.NewTransactionAsync())
{
// Insert many rows: the goal is to get at least one row inserted into
// every tablet of the hash-partitioned test table, so every tablet would
// be a participant in the transaction, and most likely every tablet
// server would be involved.
var inserts = Enumerable
.Range(0, 128)
.Select(key => ClientTestUtil.CreateBasicSchemaInsert(table, key));
await transaction.WriteAsync(inserts);
await CommitAndVerifyTransactionAsync(client, transaction);
}
// Make sure the commit timestamp for a transaction is propagated to the
// client upon committing a transaction (using a session).
using (var transaction = await client.NewTransactionAsync())
{
await using var session = transaction.NewSession();
// Insert many rows: the goal is to get at least one row inserted into
// every tablet of the hash-partitioned test table, so every tablet would
// be a participant in the transaction, and most likely every tablet
// server would be involved.
for (int key = 128; key < 256; key++)
{
var insert = ClientTestUtil.CreateBasicSchemaInsert(table, key);
await session.EnqueueAsync(insert);
}
await session.FlushAsync();
await CommitAndVerifyTransactionAsync(client, transaction);
}
}
private static async ValueTask InsertRowsAsync(
IKuduSession session, KuduTable table, int startRow, int numRows)
{
var end = startRow + numRows;
for (var i = startRow; i < end; i++)
{
var insert = ClientTestUtil.CreateBasicSchemaInsert(table, i);
await session.EnqueueAsync(insert);
}
}
public async Task TestScanTokensNonCoveringRangePartitions()
{
var builder = ClientTestUtil.CreateManyStringsSchema()
.SetTableName(_tableName)
.AddHashPartitions(2, "key")
.CreateBasicRangePartition()
.AddRangePartition((lower, upper) =>
{
lower.SetString("key", "a");
upper.SetString("key", "f");
})
.AddRangePartition((lower, upper) =>
{
lower.SetString("key", "h");
upper.SetString("key", "z");
})
.AddSplitRow(row => row.SetString("key", "k"));
var table = await _client.CreateTableAsync(builder);
for (char c = 'a'; c < 'f'; c++)
{
var row = table.NewInsert();
row.SetString("key", $"{c}");
row.SetString("c1", $"c1_{c}");
row.SetString("c2", $"c2_{c}");
await _session.EnqueueAsync(row);
}
for (char c = 'h'; c < 'z'; c++)
{
var row = table.NewInsert();
row.SetString("key", $"{c}");
row.SetString("c1", $"c1_{c}");
row.SetString("c2", $"c2_{c}");
await _session.EnqueueAsync(row);
}
await _session.FlushAsync();
var tokenBuilder = _client.NewScanTokenBuilder(table)
.SetEmptyProjection();
List <KuduScanToken> tokens = await tokenBuilder.BuildAsync();
Assert.Equal(6, tokens.Count);
await using var newClient = _harness.CreateClient();
var rowCount = await CountScanTokenRowsAsync(newClient, tokens);
Assert.Equal('f' - 'a' + 'z' - 'h', rowCount);
}
public async Task TestRenameKeyColumn()
{
KuduTable table = await CreateTableAsync();
await InsertRowsAsync(table, 0, 100);
Assert.Equal(100, await ClientTestUtil.CountRowsAsync(_client, table));
await _client.AlterTableAsync(new AlterTableBuilder(table)
.RenameColumn("c0", "c0Key"));
var exception = await Assert.ThrowsAsync <NonRecoverableException>(async() =>
{
// Scanning with the old schema.
var scanner = _client.NewScanBuilder(table)
.SetProjectedColumns("c0", "c1")
.Build();
await foreach (var resultSet in scanner)
{
}
});
Assert.True(exception.Status.IsInvalidArgument);
Assert.Contains(
"Some columns are not present in the current schema: c0",
exception.Status.Message);
// Reopen table for the new schema.
table = await _client.OpenTableAsync(_tableName);
Assert.Equal("c0Key", table.Schema.GetColumn(0).Name);
Assert.Equal(2, table.Schema.Columns.Count);
// Add a row
var insert = table.NewInsert();
insert.SetInt32("c0Key", 101);
insert.SetInt32("c1", 101);
await _session.EnqueueAsync(insert);
await _session.FlushAsync();
var scanner2 = _client.NewScanBuilder(table)
.SetProjectedColumns("c0Key", "c1")
.Build();
var rows = await scanner2.ScanToListAsync <(int c0Key, int c1)>();
Assert.Equal(101, rows.Count);
Assert.All(rows, row => Assert.Equal(row.c0Key, row.c1));
}
public async Task TestAlterAddColumns()
{
KuduTable table = await CreateTableAsync();
await InsertRowsAsync(table, 0, 100);
Assert.Equal(100, await ClientTestUtil.CountRowsAsync(_client, table));
await _client.AlterTableAsync(new AlterTableBuilder(table)
.AddColumn("addNonNull", KuduType.Int32, opt => opt
.Nullable(false)
.DefaultValue(100))
.AddColumn("addNullable", KuduType.Int32)
.AddColumn("addNullableDef", KuduType.Int32, opt => opt
.DefaultValue(200)));
// Reopen table for the new schema.
table = await _client.OpenTableAsync(_tableName);
Assert.Equal(5, table.Schema.Columns.Count);
// Add a row with addNullableDef=null
var insert = table.NewInsert();
insert.SetInt32("c0", 101);
insert.SetInt32("c1", 101);
insert.SetInt32("addNonNull", 101);
insert.SetInt32("addNullable", 101);
insert.SetNull("addNullableDef");
await _session.EnqueueAsync(insert);
await _session.FlushAsync();
// Check defaults applied, and that row key=101
var results = await ClientTestUtil.ScanTableToStringsAsync(_client, table);
var expected = new List <string>(101);
for (int i = 0; i < 100; i++)
{
expected.Add($"INT32 c0={i}, INT32 c1={i}, INT32 addNonNull=100, " +
"INT32 addNullable=NULL, INT32 addNullableDef=200");
}
expected.Add("INT32 c0=101, INT32 c1=101, INT32 addNonNull=101, " +
"INT32 addNullable=101, INT32 addNullableDef=NULL");
Assert.Equal(
expected.OrderBy(r => r),
results.OrderBy(r => r));
}
public async Task TestKuduRequireEncryption()
{
await using var harness = await new MiniKuduClusterBuilder().BuildHarnessAsync();
await using var client = harness.CreateClientBuilder()
.SetEncryptionPolicy(EncryptionPolicy.Required)
.Build();
var builder = ClientTestUtil.GetBasicSchema()
.SetTableName(nameof(TestKuduRequireEncryption));
var table = await client.CreateTableAsync(builder);
Assert.NotNull(table.TableId);
}
public async Task TestGetTableStatistics()
{
await using var harness = await new MiniKuduClusterBuilder()
.AddTabletServerFlag("--update_tablet_stats_interval_ms=200")
.AddTabletServerFlag("--heartbeat_interval_ms=100")
.BuildHarnessAsync();
await using var client = harness.CreateClient();
// Create a table.
var builder = ClientTestUtil.GetBasicSchema().SetTableName(_tableName);
var table = await client.CreateTableAsync(builder);
// Insert some rows and test the statistics.
var prevStatistics = new KuduTableStatistics(-1, -1);
var currentStatistics = new KuduTableStatistics(-1, -1);
var session = client.NewSession();
int num = 100;
for (int i = 0; i < num; ++i)
{
// Get current table statistics.
currentStatistics = await client.GetTableStatisticsAsync(_tableName);
Assert.True(currentStatistics.OnDiskSize >= prevStatistics.OnDiskSize);
Assert.True(currentStatistics.LiveRowCount >= prevStatistics.LiveRowCount);
Assert.True(currentStatistics.LiveRowCount <= i + 1);
prevStatistics = currentStatistics;
// Insert row.
var insert = ClientTestUtil.CreateBasicSchemaInsert(table, i);
await session.EnqueueAsync(insert);
await session.FlushAsync();
long numRows = await ClientTestUtil.CountRowsAsync(client, table);
Assert.Equal(i + 1, numRows);
}
// Final accuracy test.
// Wait for master to aggregate table statistics.
await Task.Delay(200 * 6);
currentStatistics = await client.GetTableStatisticsAsync(_tableName);
Assert.True(currentStatistics.OnDiskSize >= prevStatistics.OnDiskSize);
Assert.True(currentStatistics.LiveRowCount >= prevStatistics.LiveRowCount);
Assert.Equal(num, currentStatistics.LiveRowCount);
}
public async Task TestSwitchToOtherTxnManagerInFlightCalls()
{
await using var harness = await new MiniKuduClusterBuilder()
.AddMasterServerFlag("--txn_manager_enabled")
// Set Raft heartbeat interval short for faster test runtime: speed up
// leader failure detection and new leader election.
.AddMasterServerFlag("--raft_heartbeat_interval_ms=100")
.AddTabletServerFlag("--enable_txn_system_client_init=true")
.BuildHarnessAsync();
await using var client = harness.CreateClient();
var builder = ClientTestUtil.GetBasicSchema()
.SetTableName(nameof(TestSwitchToOtherTxnManagerInFlightCalls))
.AddHashPartitions(2, "key");
var table = await client.CreateTableAsync(builder);
using var transaction = await client.NewTransactionAsync();
var insert = ClientTestUtil.CreateBasicSchemaInsert(table, 0);
await transaction.WriteAsync(new[] { insert });
await harness.KillAllMasterServersAsync();
var startMastersTask = Task.Run(async() =>
{
// Sleep for some time to allow the commit call
// below issue RPCs to non-running TxnManangers.
await Task.Delay(1000);
await harness.StartAllMasterServersAsync();
});
// It should be possible to commit the transaction.
await transaction.CommitAsync();
await transaction.WaitForCommitAsync();
await startMastersTask;
// An extra sanity check: read back the rows written into the table in the
// context of the transaction.
var scanner = client.NewScanBuilder(table)
.SetReadMode(ReadMode.ReadYourWrites)
.SetReplicaSelection(ReplicaSelection.LeaderOnly)
.Build();
Assert.Equal(1, await scanner.CountAsync());
}
public async Task TestAlterExtraConfigs()
{
KuduTable table = await CreateTableAsync();
await InsertRowsAsync(table, 0, 100);
Assert.Equal(100, await ClientTestUtil.CountRowsAsync(_client, table));
// 1. Check for expected defaults.
table = await _client.OpenTableAsync(_tableName);
Assert.DoesNotContain("kudu.table.history_max_age_sec", table.ExtraConfig);
// 2. Alter history max age second to 3600
var alterExtraConfigs = new Dictionary <string, string>
{
{ "kudu.table.history_max_age_sec", "3600" }
};
await _client.AlterTableAsync(new AlterTableBuilder(table)
.AlterExtraConfigs(alterExtraConfigs));
table = await _client.OpenTableAsync(_tableName);
Assert.Equal("3600", table.ExtraConfig["kudu.table.history_max_age_sec"]);
// 3. Alter history max age second to 7200
alterExtraConfigs = new Dictionary <string, string>
{
{ "kudu.table.history_max_age_sec", "7200" }
};
await _client.AlterTableAsync(new AlterTableBuilder(table)
.AlterExtraConfigs(alterExtraConfigs));
table = await _client.OpenTableAsync(_tableName);
Assert.Equal("7200", table.ExtraConfig["kudu.table.history_max_age_sec"]);
// 4. Reset history max age second to default
alterExtraConfigs = new Dictionary <string, string>
{
{ "kudu.table.history_max_age_sec", "" }
};
await _client.AlterTableAsync(new AlterTableBuilder(table)
.AlterExtraConfigs(alterExtraConfigs));
table = await _client.OpenTableAsync(_tableName);
Assert.Empty(table.ExtraConfig);
}
public async Task TestResourceMetrics()
{
// Scan one tablet and the whole table.
var oneTabletScanner = GetScanner("1", "1", "1", "4"); // Whole second tablet.
await using var oneTabletScanEnumerator = oneTabletScanner.GetAsyncEnumerator();
Assert.Equal(3, await ClientTestUtil.CountRowsInScanAsync(oneTabletScanEnumerator));
var fullTableScanner = GetScanner(null, null, null, null);
await using var fullTableScanEnumerator = fullTableScanner.GetAsyncEnumerator();
Assert.Equal(9, await ClientTestUtil.CountRowsInScanAsync(fullTableScanEnumerator));
// Both scans should take a positive amount of wait duration, total duration,
// cpu user and cpu system time.
ValidateResourceMetrics(oneTabletScanEnumerator.ResourceMetrics);
ValidateResourceMetrics(fullTableScanEnumerator.ResourceMetrics);
public async Task TestKuduRequireAuthenticationInsecureCluster()
{
await using var harness = await new MiniKuduClusterBuilder().BuildHarnessAsync();
await using var client = harness.CreateClientBuilder()
.RequireAuthentication(true)
.Build();
var builder = ClientTestUtil.GetBasicSchema()
.SetTableName(nameof(TestKuduRequireAuthenticationInsecureCluster));
var exception = await Assert.ThrowsAsync <NonRecoverableException>(
() => client.CreateTableAsync(builder));
Assert.Contains(
"Client requires authentication, but server does not have Kerberos enabled",
exception.Message);
}
public async Task TestExceptionCallback()
{
int numCallbacks = 0;
SessionExceptionContext sessionContext = null;
var builder = ClientTestUtil.GetBasicSchema()
.SetTableName(nameof(TestExceptionCallback));
var table = await _client.CreateTableAsync(builder);
var row1 = ClientTestUtil.CreateBasicSchemaInsert(table, 1);
var row2 = ClientTestUtil.CreateBasicSchemaInsert(table, 1);
var sessionOptions = new KuduSessionOptions
{
ExceptionHandler = HandleSessionExceptionAsync
};
await using var session = _client.NewSession(sessionOptions);
await session.EnqueueAsync(row1);
await session.FlushAsync();
await session.EnqueueAsync(row2);
await session.FlushAsync();
ValueTask HandleSessionExceptionAsync(SessionExceptionContext context)
{
numCallbacks++;
sessionContext = context;
return(new ValueTask());
}
Assert.Equal(1, numCallbacks);
var errorRow = Assert.Single(sessionContext.Rows);
Assert.Same(row2, errorRow);
var exception = Assert.IsType <KuduWriteException>(sessionContext.Exception);
var exceptionRow = Assert.Single(exception.PerRowErrors);
Assert.True(exceptionRow.IsAlreadyPresent);
}
public async Task TestAuthzTokensDuringElection()
{
await using var client = _harness.CreateClient();
await using var session = client.NewSession();
// Test sending various requests that require authorization.
var builder = ClientTestUtil.GetBasicSchema()
.SetTableName(_tableName)
.CreateBasicRangePartition()
.SetNumReplicas(1);
var table = await client.CreateTableAsync(builder);
// Restart the masters to trigger an election.
await _harness.KillAllMasterServersAsync();
await _harness.StartAllMasterServersAsync();
int numReqs = 10;
await InsertRowsAsync(session, table, 0, numReqs);
await session.FlushAsync();
// Do the same for batches of inserts.
await _harness.KillAllMasterServersAsync();
await _harness.StartAllMasterServersAsync();
await InsertRowsAsync(session, table, numReqs, numReqs);
await session.FlushAsync();
// And for scans.
await _harness.KillAllMasterServersAsync();
await _harness.StartAllMasterServersAsync();
for (int i = 0; i < numReqs; i++)
{
var numRows = await ClientTestUtil.CountRowsAsync(client, table);
Assert.Equal(2 * numReqs, numRows);
}
}
public async Task Test()
{
var testRuntime = TimeSpan.FromMinutes(1);
using var cts = new CancellationTokenSource(testRuntime);
var token = cts.Token;
var chaosTask = RunTaskAsync(() => DoChaosAsync(token), cts);
var writeTask = RunTaskAsync(() => WriteAsync(token), cts);
var scanTask = RunTaskAsync(() => ScanAsync(token), cts);
await Task.WhenAll(chaosTask, writeTask, scanTask);
// If the test passed, do some extra validation at the end.
var rowCount = await ClientTestUtil.CountRowsAsync(_client, _table);
Assert.True(rowCount > 0);
}
public async Task TestAlterRangePartitioningExclusiveInclusive()
{
// Create initial table with single range partition covering (-1, 99].
var builder = new TableBuilder(_tableName)
.SetNumReplicas(1)
.AddColumn("c0", KuduType.Int32, opt => opt.Key(true))
.AddColumn("c1", KuduType.Int32, opt => opt.Nullable(false))
.SetRangePartitionColumns("c0")
.AddRangePartition((lower, upper) =>
{
lower.SetInt32("c0", -1);
upper.SetInt32("c0", 99);
}, RangePartitionBound.Exclusive, RangePartitionBound.Inclusive);
KuduTable table = await _client.CreateTableAsync(builder);
await _client.AlterTableAsync(new AlterTableBuilder(table)
.AddRangePartition((lower, upper) =>
{
lower.SetInt32("c0", 199);
upper.SetInt32("c0", 299);
}, RangePartitionBound.Exclusive, RangePartitionBound.Inclusive));
// Insert some rows, and then drop the partition and ensure that the table is empty.
await InsertRowsAsync(table, 0, 100);
await InsertRowsAsync(table, 200, 300);
Assert.Equal(200, await ClientTestUtil.CountRowsAsync(_client, table));
await _client.AlterTableAsync(new AlterTableBuilder(table)
.DropRangePartition((lower, upper) =>
{
lower.SetInt32("c0", 0);
upper.SetInt32("c0", 100);
}, RangePartitionBound.Inclusive, RangePartitionBound.Exclusive)
.DropRangePartition((lower, upper) =>
{
lower.SetInt32("c0", 199);
upper.SetInt32("c0", 299);
}, RangePartitionBound.Exclusive, RangePartitionBound.Inclusive));
Assert.Equal(0, await ClientTestUtil.CountRowsAsync(_client, table));
}
public async Task TestScannerExpiration()
{
await using var miniCluster = await new MiniKuduClusterBuilder()
.AddTabletServerFlag($"--scanner_ttl_ms={ShortScannerTtlMs}")
.AddTabletServerFlag($"--scanner_gc_check_interval_us={ShortScannerGcUs}")
.BuildAsync();
await using var client = miniCluster.CreateClient();
var builder = ClientTestUtil.GetBasicSchema()
.SetTableName("TestScannerExpiration")
.AddHashPartitions(2, "key");
var table = await client.CreateTableAsync(builder);
int numRows = 1000;
var rows = Enumerable.Range(0, numRows)
.Select(i => ClientTestUtil.CreateBasicSchemaInsert(table, i));
await client.WriteAsync(rows);
var scanner = client.NewScanBuilder(table)
.SetReadMode(ReadMode.ReadYourWrites)
.SetReplicaSelection(ReplicaSelection.ClosestReplica)
.SetBatchSizeBytes(100) // Use a small batch size so we get many batches.
.Build();
var scanEnumerator = scanner.GetAsyncEnumerator();
// Initialize the scanner and verify we can read rows.
Assert.True(await scanEnumerator.MoveNextAsync());
Assert.True(scanEnumerator.Current.Count > 0);
// Wait for the scanner to time out.
await Task.Delay(ShortScannerTtlMs * 2);
var exception = await Assert.ThrowsAsync <NonRecoverableException>(async() =>
await scanEnumerator.MoveNextAsync());
Assert.Matches(".*Scanner .* not found.*", exception.Message);
// Closing an expired scanner shouldn't throw an exception.
await scanEnumerator.DisposeAsync();
}
