public async Task TestDifferentProjection()
{
var builder = new TableBuilder(nameof(TestDifferentProjection))
.AddColumn("key", KuduType.Int32, opt => opt.Key(true))
.AddColumn("extra_column", KuduType.Int32, opt => opt.Nullable(false));
var values = new[] { 0, 1, 2, 3, 4, 5 };
var table = await _client.CreateTableAsync(builder);
var rowsToInsert = values.Select(value =>
{
var insert = table.NewInsert();
insert.SetInt32("key", value);
insert.SetInt32("extra_column", value * 2);
return(insert);
});
await _client.WriteAsync(rowsToInsert);
var scanner = _client.NewScanBuilder(table)
.SetProjectedColumns("key")
.SetReadMode(ReadMode.ReadYourWrites)
.Build();
var rows = await scanner.ScanToListAsync <int>();
Assert.Equal(values, rows);
}
public async Task TestRenameKeyColumn()
{
KuduTable table = await CreateTableAsync();
await InsertRowsAsync(table, 0, 100);
Assert.Equal(100, await ClientTestUtil.CountRowsAsync(_client, table));
await _client.AlterTableAsync(new AlterTableBuilder(table)
.RenameColumn("c0", "c0Key"));
var exception = await Assert.ThrowsAsync <NonRecoverableException>(async() =>
{
// Scanning with the old schema.
var scanner = _client.NewScanBuilder(table)
.SetProjectedColumns("c0", "c1")
.Build();
await foreach (var resultSet in scanner)
{
}
});
Assert.True(exception.Status.IsInvalidArgument);
Assert.Contains(
"Some columns are not present in the current schema: c0",
exception.Status.Message);
// Reopen table for the new schema.
table = await _client.OpenTableAsync(_tableName);
Assert.Equal("c0Key", table.Schema.GetColumn(0).Name);
Assert.Equal(2, table.Schema.Columns.Count);
// Add a row
var insert = table.NewInsert();
insert.SetInt32("c0Key", 101);
insert.SetInt32("c1", 101);
await _session.EnqueueAsync(insert);
await _session.FlushAsync();
var scanner2 = _client.NewScanBuilder(table)
.SetProjectedColumns("c0Key", "c1")
.Build();
var rows = await scanner2.ScanToListAsync <(int c0Key, int c1)>();
Assert.Equal(101, rows.Count);
Assert.All(rows, row => Assert.Equal(row.c0Key, row.c1));
}
public static async Task <List <string> > ScanTableToStringsAsync(
KuduClient client, KuduTable table, params KuduPredicate[] predicates)
{
var rowStrings = new List <string>();
var scanBuilder = client.NewScanBuilder(table)
.SetReadMode(ReadMode.ReadYourWrites)
.SetReplicaSelection(ReplicaSelection.LeaderOnly);
foreach (var predicate in predicates)
{
scanBuilder.AddPredicate(predicate);
}
var scanner = scanBuilder.Build();
await foreach (var resultSet in scanner)
{
foreach (var row in resultSet)
{
rowStrings.Add(row.ToString());
}
}
rowStrings.Sort();
return(rowStrings);
}
/// <summary>
/// Checks the number of tablets and pruner ranges generated for a scan with
/// predicates and optional partition key bounds.
/// </summary>
/// <param name="expectedTablets">The expected number of tablets to satisfy the scan.</param>
/// <param name="expectedPrunerRanges">The expected number of generated partition pruner ranges.</param>
/// <param name="table">The table to scan.</param>
/// <param name="partitions">The partitions of the table.</param>
/// <param name="lowerBoundPartitionKey">An optional lower bound partition key.</param>
/// <param name="upperBoundPartitionKey">An optional upper bound partition key.</param>
/// <param name="predicates">The predicates to apply to the scan.</param>
private async Task CheckPartitionsAsync(
int expectedTablets,
int expectedPrunerRanges,
KuduTable table,
List <Partition> partitions,
byte[] lowerBoundPartitionKey,
byte[] upperBoundPartitionKey,
params KuduPredicate[] predicates)
{
// Partition key bounds can't be applied to the ScanTokenBuilder.
var scanBuilder = _client.NewScanBuilder(table);
foreach (var predicate in predicates)
{
scanBuilder.AddPredicate(predicate);
}
if (lowerBoundPartitionKey != null)
{
scanBuilder.LowerBoundPartitionKeyRaw(lowerBoundPartitionKey);
}
if (upperBoundPartitionKey != null)
{
scanBuilder.ExclusiveUpperBoundPartitionKeyRaw(upperBoundPartitionKey);
}
var pruner = PartitionPruner.Create(scanBuilder);
int scannedPartitions = 0;
foreach (var partition in partitions)
{
if (!pruner.ShouldPruneForTests(partition))
{
scannedPartitions++;
}
}
Assert.Equal(expectedTablets, scannedPartitions);
Assert.Equal(expectedPrunerRanges, pruner.NumRangesRemaining);
// Check that the scan token builder comes up with the same amount.
// The scan token builder does not allow for upper/lower partition keys.
if (lowerBoundPartitionKey == null && upperBoundPartitionKey == null)
{
var tokenBuilder = _client.NewScanTokenBuilder(table);
foreach (var predicate in predicates)
{
tokenBuilder.AddPredicate(predicate);
}
// Check that the number of ScanTokens built for the scan matches.
var tokens = await tokenBuilder.BuildAsync();
Assert.Equal(expectedTablets, tokens.Count);
}
}
public static ValueTask <long> CountRowsAsync(KuduClient client, KuduTable table)
{
var scanner = client.NewScanBuilder(table)
.SetReadMode(ReadMode.ReadYourWrites)
.SetReplicaSelection(ReplicaSelection.LeaderOnly)
.Build();
return(scanner.CountAsync());
}
public async Task TestHybridTime()
{
// Perform one write so we receive a timestamp from the server and can use
// it to propagate a modified timestamp back to the server. Following writes
// should force the servers to update their clocks to this value and increment
// the logical component of the timestamp.
await InsertRowAsync("0");
Assert.NotEqual(KuduClient.NoTimestamp, _client.LastPropagatedTimestamp);
var(timestampMicros, logicalValue) = HybridTimeUtil.HtTimestampToPhysicalAndLogical(
_client.LastPropagatedTimestamp);
Assert.Equal(0, logicalValue);
long futureTs = timestampMicros + 5000000;
_client.LastPropagatedTimestamp = HybridTimeUtil.ClockTimestampToHtTimestamp(futureTs);
var logicalValues = new List <long>();
var keys = new[] { "1", "2", "3", "11", "22", "33" };
for (int i = 0; i < keys.Length; i++)
{
await InsertRowAsync(keys[i]);
Assert.NotEqual(KuduClient.NoTimestamp, _client.LastPropagatedTimestamp);
(timestampMicros, logicalValue) = HybridTimeUtil.HtTimestampToPhysicalAndLogical(
_client.LastPropagatedTimestamp);
Assert.Equal(futureTs, timestampMicros);
logicalValues.Add(logicalValue);
Assert.Equal(logicalValues.OrderBy(v => v), logicalValues);
}
// Scan all rows with READ_LATEST (the default), which should retrieve all rows.
var scanner = _client.NewScanBuilder(_table)
.SetReadMode(ReadMode.ReadLatest)
.Build();
Assert.Equal(1 + keys.Length, await scanner.CountAsync());
// Now scan at multiple snapshots with READ_AT_SNAPSHOT. The logical timestamp
// from the 'i'th row (counted from 0) combined with the latest physical timestamp
// should observe 'i + 1' rows.
for (int i = 0; i < logicalValues.Count; i++)
{
logicalValue = logicalValues[i];
long snapshotTime = HybridTimeUtil.PhysicalAndLogicalToHtTimestamp(
futureTs, logicalValue);
int expected = i + 1;
long numRows = await ScanAtSnapshotAsync(snapshotTime);
Assert.Equal(expected, numRows);
}
// The last snapshots needs to be one into the future w.r.t. the last write's
// timestamp to get all rows, but the snapshot timestamp can't be bigger than
// the propagated timestamp. Ergo increase the propagated timestamp first.
long latestLogicalValue = logicalValues[^ 1];
private ValueTask <long> CountRowsAsync(KuduTable table, params KuduPredicate[] predicates)
{
var scanBuilder = _client.NewScanBuilder(table)
.SetReadMode(ReadMode.ReadYourWrites);
foreach (var predicate in predicates)
{
scanBuilder.AddPredicate(predicate);
}
var scanner = scanBuilder.Build();
return(scanner.CountAsync());
}
/// <summary>
/// Injecting failures (kill or restart TabletServer) while scanning, to verify:
/// fault tolerant scanner will continue scan and non-fault tolerant scanner will
/// throw <see cref="NonRecoverableException"/>. Also makes sure we pass all the
/// correct information down to the server by verifying we get rows in order from
/// 3 tablets. We detect those tablet boundaries when keys suddenly become smaller
/// than what was previously seen.
/// </summary>
/// <param name="restart">
/// If true restarts TabletServer, otherwise kills TabletServer.
/// </param>
/// <param name="isFaultTolerant">
/// If true uses fault tolerant scanner, otherwise uses non fault-tolerant one.
/// </param>
/// <param name="finishFirstScan">
/// If true injects failure before finishing first tablet scan, otherwise in the
/// middle of tablet scanning.
/// </param>
private async Task TestServerFaultInjectionAsync(
bool restart,
bool isFaultTolerant,
bool finishFirstScan)
{
var scanner = _client.NewScanBuilder(_table)
.SetFaultTolerant(isFaultTolerant)
.SetBatchSizeBytes(1)
.SetProjectedColumns(0)
.Build();
await using var scanEnumerator = scanner.GetAsyncEnumerator();
int previousRow = -1;
var keys = new List <int>();
if (await scanEnumerator.MoveNextAsync())
{
var resultSet = scanEnumerator.Current;
var results = resultSet.MapTo <int>();
keys.AddRange(results);
previousRow = keys[^ 1];
public async Task TestScannerBuilderFaultToleranceToggle()
{
var builder = ClientTestUtil.GetBasicSchema()
.SetTableName(_tableName);
var table = await _client.CreateTableAsync(builder);
var scanBuilder = _client.NewScanBuilder(table);
Assert.False(scanBuilder.IsFaultTolerant);
Assert.Equal(ReadMode.ReadLatest, scanBuilder.ReadMode);
scanBuilder.SetFaultTolerant(true);
Assert.True(scanBuilder.IsFaultTolerant);
Assert.Equal(ReadMode.ReadAtSnapshot, scanBuilder.ReadMode);
scanBuilder.SetFaultTolerant(false);
Assert.False(scanBuilder.IsFaultTolerant);
Assert.Equal(ReadMode.ReadAtSnapshot, scanBuilder.ReadMode);
scanBuilder.SetReadMode(ReadMode.ReadYourWrites);
Assert.False(scanBuilder.IsFaultTolerant);
Assert.Equal(ReadMode.ReadYourWrites, scanBuilder.ReadMode);
}
public async Task TestAllPrimaryKeyTypes()
{
var tableBuilder = new TableBuilder(nameof(TestAllPrimaryKeyTypes))
.AddColumn("int8", KuduType.Int8, opt => opt.Key(true))
.AddColumn("int16", KuduType.Int16, opt => opt.Key(true))
.AddColumn("int32", KuduType.Int32, opt => opt.Key(true))
.AddColumn("int64", KuduType.Int64, opt => opt.Key(true))
.AddColumn("string", KuduType.String, opt => opt.Key(true))
.AddColumn("binary", KuduType.Binary, opt => opt.Key(true))
.AddColumn("timestamp", KuduType.UnixtimeMicros, opt => opt.Key(true))
.AddColumn("decimal32", KuduType.Decimal32, opt => opt.Key(true)
.DecimalAttributes(DecimalUtil.MaxDecimal32Precision, 0))
.AddColumn("decimal64", KuduType.Decimal64, opt => opt.Key(true)
.DecimalAttributes(DecimalUtil.MaxDecimal64Precision, 0))
.AddColumn("decimal128", KuduType.Decimal128, opt => opt.Key(true)
.DecimalAttributes(DecimalUtil.MaxDecimal128Precision, 0))
.AddColumn("varchar", KuduType.Varchar, opt => opt.Key(true)
.VarcharAttributes(10))
.AddColumn("date", KuduType.Date, opt => opt.Key(true))
.AddColumn("bool", KuduType.Bool) // not primary key type
.AddColumn("float", KuduType.Float) // not primary key type
.AddColumn("double", KuduType.Double); // not primary key type
var table = await _client.CreateTableAsync(tableBuilder);
var insert = table.NewInsert();
insert.SetSByte("int8", 1);
insert.SetInt16("int16", 2);
insert.SetInt32("int32", 3);
insert.SetInt64("int64", 4);
insert.SetString("string", "foo");
insert.SetBinary("binary", "bar".ToUtf8ByteArray());
insert.SetInt64("timestamp", 6);
insert.SetDecimal("decimal32", DecimalUtil.MaxUnscaledDecimal32);
insert.SetDecimal("decimal64", DecimalUtil.MaxUnscaledDecimal64);
insert.SetDecimal("decimal128", decimal.Truncate(decimal.MaxValue));
insert.SetString("varchar", "varchar bar");
insert.SetDateTime("date", EpochTime.FromUnixTimeDays(0));
insert.SetBool("bool", true);
insert.SetFloat("float", 7.8f);
insert.SetDouble("double", 9.9);
await _client.WriteAsync(new[] { insert });
var scanner = _client.NewScanBuilder(table).Build();
var scannedRows = 0;
await foreach (var resultSet in scanner)
{
foreach (var row in resultSet)
{
scannedRows++;
Assert.Equal(1, row.GetSByte("int8"));
Assert.Equal(2, row.GetInt16("int16"));
Assert.Equal(3, row.GetInt32("int32"));
Assert.Equal(4, row.GetInt64("int64"));
Assert.Equal("foo", row.GetString("string"));
Assert.Equal("bar".ToUtf8ByteArray(), row.GetBinary("binary"));
Assert.Equal(6, row.GetInt64("timestamp"));
Assert.Equal(DecimalUtil.MaxUnscaledDecimal32, row.GetDecimal("decimal32"));
Assert.Equal(DecimalUtil.MaxUnscaledDecimal64, row.GetDecimal("decimal64"));
Assert.Equal(decimal.Truncate(decimal.MaxValue), row.GetDecimal("decimal128"));
Assert.Equal("varchar ba", row.GetString("varchar"));
Assert.Equal(0, row.GetInt32("date"));
Assert.True(row.GetBool("bool"));
Assert.Equal(7.8f, row.GetFloat("float"));
Assert.Equal(9.9, row.GetDouble("double"));
}
}
Assert.Equal(1, scannedRows);
}