/// <summary>
/// Counts the partitions touched by a scan with optional primary key bounds.
/// The table is assumed to have three INT8 columns as the primary key.
/// </summary>
/// <param name="expectedTablets">The expected number of tablets to satisfy the scan.</param>
/// <param name="table">The table to scan.</param>
/// <param name="partitions">The partitions of the table.</param>
/// <param name="lowerBoundPrimaryKey">The optional lower bound primary key.</param>
/// <param name="upperBoundPrimaryKey">The optional upper bound primary key.</param>
private async Task CheckPartitionsPrimaryKeyAsync(
int expectedTablets,
KuduTable table,
List <Partition> partitions,
sbyte[] lowerBoundPrimaryKey,
sbyte[] upperBoundPrimaryKey)
{
var scanBuilder = _client.NewScanTokenBuilder(table);
if (lowerBoundPrimaryKey != null)
{
var lower = new PartialRow(table.Schema);
for (int i = 0; i < 3; i++)
{
lower.SetSByte(i, lowerBoundPrimaryKey[i]);
}
scanBuilder.LowerBound(lower);
}
if (upperBoundPrimaryKey != null)
{
var upper = new PartialRow(table.Schema);
for (int i = 0; i < 3; i++)
{
upper.SetSByte(i, upperBoundPrimaryKey[i]);
}
scanBuilder.ExclusiveUpperBound(upper);
}
var pruner = PartitionPruner.Create(scanBuilder);
int scannedPartitions = 0;
foreach (var partition in partitions)
{
if (!pruner.ShouldPruneForTests(partition))
{
scannedPartitions++;
}
}
// Check that the number of ScanTokens built for the scan matches.
var tokens = await scanBuilder.BuildAsync();
Assert.Equal(expectedTablets, scannedPartitions);
Assert.Equal(scannedPartitions, tokens.Count);
Assert.Equal(expectedTablets == 0 ? 0 : 1, pruner.NumRangesRemaining);
}
public async Task TestScanTokens()
{
var builder = ClientTestUtil.CreateManyStringsSchema()
.SetTableName(_tableName)
.AddHashPartitions(8, "key")
.CreateBasicRangePartition()
.AddSplitRow(row => row.SetString("key", "key_50"));
var table = await _client.CreateTableAsync(builder);
for (int i = 0; i < 100; i++)
{
var row = table.NewInsert();
row.SetString("key", $"key_{i}");
row.SetString("c1", $"c1_{i}");
row.SetString("c2", $"c2_{i}");
await _session.EnqueueAsync(row);
}
await _session.FlushAsync();
var tokenBuilder = _client.NewScanTokenBuilder(table)
.SetEmptyProjection()
// For this test, make sure that we cover the case that not all tablets
// are returned in a single batch.
.SetFetchTabletsPerRangeLookup(4);
List <KuduScanToken> tokens = await tokenBuilder.BuildAsync();
Assert.Equal(16, tokens.Count);
await using var newClient = _harness.CreateClient();
var rowCount = await CountScanTokenRowsAsync(newClient, tokens);
Assert.Equal(100, rowCount);
}
public async Task TestFaultTolerantScannerRestartAfterSecondScanRequest()
{
// In fact, the test has TABLET_COUNT, default is 3.
// We check the rows' order, no dup rows and loss rows.
// And In the case, we need 2 times or more scan requests,
// so set a minimum batchSizeBytes 1.
var tokenBuilder = _client.NewScanTokenBuilder(_table)
.SetBatchSizeBytes(1)
.SetFaultTolerant(true)
.SetProjectedColumns(0);
var tokens = await tokenBuilder.BuildAsync();
Assert.Equal(_numTablets, tokens.Count);
var tabletScannerTasks = tokens
.Select((token, i) => ScanTokenAsync(token, i == 0));
var results = await Task.WhenAll(tabletScannerTasks);
var rowCount = results.Sum();
Assert.Equal(_numRows, rowCount);
async Task <int> ScanTokenAsync(KuduScanToken token, bool enableFaultInjection)
{
int rowCount = 0;
int previousRow = int.MinValue;
bool faultInjected = !enableFaultInjection;
int faultInjectionLowBound = (_numRows / _numTablets / 2);
bool firstScanRequest = true;
long firstScannedMetric = 0;
long firstPropagatedTimestamp = 0;
long lastScannedMetric = 0;
long lastPropagatedTimestamp = 0;
var scanBuilder = await _client.NewScanBuilderFromTokenAsync(token);
var scanner = scanBuilder.Build();
await using var scanEnumerator = scanner.GetAsyncEnumerator();
while (await scanEnumerator.MoveNextAsync())
{
foreach (var row in scanEnumerator.Current)
{
int key = row.GetInt32(0);
if (previousRow >= key)
{
throw new Exception(
$"Impossible results, previousKey: {previousRow} >= currentKey: {key}");
}
if (!faultInjected && rowCount > faultInjectionLowBound)
{
await _harness.RestartTabletServerAsync(scanEnumerator.Tablet);
faultInjected = true;
}
else
{
if (firstScanRequest)
{
firstScannedMetric = scanEnumerator.ResourceMetrics.TotalDurationNanos;
firstPropagatedTimestamp = _client.LastPropagatedTimestamp;
firstScanRequest = false;
}
lastScannedMetric = scanEnumerator.ResourceMetrics.TotalDurationNanos;
lastPropagatedTimestamp = _client.LastPropagatedTimestamp;
}
previousRow = key;
rowCount++;
}
}
Assert.NotEqual(lastScannedMetric, firstScannedMetric);
Assert.True(lastPropagatedTimestamp > firstPropagatedTimestamp,
$"Expected {lastPropagatedTimestamp} > {firstPropagatedTimestamp}");
return(rowCount);
}
}