public KuduScanEnumerator(
ILogger logger,
KuduClient client,
KuduTable table,
List <ColumnSchemaPB> projectedColumnsPb,
KuduSchema projectionSchema,
OrderMode orderMode,
ReadMode readMode,
ReplicaSelection replicaSelection,
bool isFaultTolerant,
Dictionary <string, KuduPredicate> predicates,
long limit,
bool cacheBlocks,
byte[] startPrimaryKey,
byte[] endPrimaryKey,
long startTimestamp,
long htTimestamp,
int batchSizeBytes,
PartitionPruner partitionPruner,
CancellationToken cancellationToken)
{
_logger = logger;
_client = client;
_table = table;
_partitionPruner = partitionPruner;
_orderMode = orderMode;
_readMode = readMode;
_columns = projectedColumnsPb;
_schema = projectionSchema;
_predicates = predicates;
_replicaSelection = replicaSelection;
_isFaultTolerant = isFaultTolerant;
_limit = limit;
_cacheBlocks = cacheBlocks;
_startPrimaryKey = startPrimaryKey ?? Array.Empty <byte>();
_endPrimaryKey = endPrimaryKey ?? Array.Empty <byte>();
_startTimestamp = startTimestamp;
SnapshotTimestamp = htTimestamp;
_batchSizeBytes = batchSizeBytes;
_scannerId = ByteString.Empty;
_lastPrimaryKey = ByteString.Empty;
_cancellationToken = cancellationToken;
ResourceMetrics = new ResourceMetrics();
// If the partition pruner has pruned all partitions, then the scan can be
// short circuited without contacting any tablet servers.
if (!_partitionPruner.HasMorePartitionKeyRanges)
{
_closed = true;
}
// For READ_YOUR_WRITES scan mode, get the latest observed timestamp
// and store it. Always use this one as the propagated timestamp for
// the duration of the scan to avoid unnecessary wait.
if (readMode == ReadMode.ReadYourWrites)
{
_lowerBoundPropagationTimestamp = client.LastPropagatedTimestamp;
}
}
/// <summary>
/// Helper function to centralize the calling of methods based on the
/// passed replica selection mechanism.
/// </summary>
/// <param name="replicaSelection">Replica selection mechanism to use.</param>
/// <param name="location">The location of the client.</param>
public ServerInfo?GetServerInfo(ReplicaSelection replicaSelection, string?location = null)
{
if (replicaSelection == ReplicaSelection.LeaderOnly)
{
return(GetLeaderServerInfo());
}
return(GetClosestServerInfo(location));
}
public KeepAliveRequest(
ByteString scannerId,
ReplicaSelection replicaSelection,
string tableId,
RemoteTablet?tablet,
byte[] partitionKey)
{
_request = new ScannerKeepAliveRequestPB {
ScannerId = scannerId
};
MethodName = "ScannerKeepAlive";
ReplicaSelection = replicaSelection;
TableId = tableId;
Tablet = tablet;
PartitionKey = partitionKey;
}
public async Task TestScanQuiescingTabletServer(ReplicaSelection replicaSelection)
{
await using var miniCluster = await new MiniKuduClusterBuilder().BuildAsync();
await using var client = miniCluster.CreateClient();
var builder = new TableBuilder(nameof(TestScanQuiescingTabletServer))
.AddColumn("key", KuduType.Int32, opt => opt.Key(true))
.SetRangePartitionColumns("key")
.SetNumReplicas(3);
var table = await client.CreateTableAsync(builder);
var numRows = 500;
var rows = Enumerable.Range(0, numRows).Select(i =>
{
var row = table.NewInsert();
row.SetInt32("key", i);
return(row);
});
await client.WriteAsync(rows);
// Quiesce a single tablet server.
var tservers = miniCluster.GetTabletServers();
await MiniKuduCluster.RunKuduToolAsync(
"tserver",
"quiesce",
"start",
tservers[0].ToString());
// Now start a scan. Even if the scan goes to the quiescing server, the
// scan request should eventually be routed to a non-quiescing server
// and complete. We aren't guaranteed to hit the quiescing server, but this
// test would frequently fail if we didn't handle quiescing servers properly.
var scanner = client.NewScanBuilder(table)
.SetReplicaSelection(replicaSelection)
.SetReadMode(ReadMode.ReadYourWrites)
.Build();
long scannedRows = await scanner.CountAsync();
Assert.Equal(numRows, scannedRows);
}
public ServerInfo?GetServerInfo(
ReplicaSelection replicaSelection, string?location = null)
{
return(_cache.GetServerInfo(replicaSelection, location));
}
public KuduScanner(
ILogger logger,
KuduClient client,
KuduTable table,
List <string>?projectedColumnNames,
List <int>?projectedColumnIndexes,
Dictionary <string, KuduPredicate> predicates,
ReadMode readMode,
ReplicaSelection replicaSelection,
bool isFaultTolerant,
int batchSizeBytes,
long limit,
bool cacheBlocks,
long startTimestamp,
long htTimestamp,
byte[] lowerBoundPrimaryKey,
byte[] upperBoundPrimaryKey,
byte[] lowerBoundPartitionKey,
byte[] upperBoundPartitionKey)
{
if (limit <= 0)
{
throw new ArgumentException($"Need a strictly positive number for the limit, got {limit}");
}
if (htTimestamp != KuduClient.NoTimestamp)
{
if (htTimestamp < 0)
{
throw new ArgumentException(
$"Need non-negative number for the scan, timestamp got {htTimestamp}");
}
if (readMode != ReadMode.ReadAtSnapshot)
{
throw new ArgumentException(
"When specifying a HybridClock timestamp, the read mode needs to be set to READ_AT_SNAPSHOT");
}
}
if (startTimestamp != KuduClient.NoTimestamp)
{
if (htTimestamp < 0)
{
throw new ArgumentException(
"Must have both start and end timestamps for a diff scan");
}
if (startTimestamp > htTimestamp)
{
throw new ArgumentException(
"Start timestamp must be less than or equal to end timestamp");
}
}
_isFaultTolerant = isFaultTolerant;
if (isFaultTolerant)
{
if (readMode != ReadMode.ReadAtSnapshot)
{
throw new ArgumentException("Use of fault tolerance scanner " +
"requires the read mode to be set to READ_AT_SNAPSHOT");
}
_orderMode = OrderMode.Ordered;
}
else
{
_orderMode = OrderMode.Unordered;
}
_logger = logger;
_client = client;
_table = table;
_predicates = predicates;
ReadMode = readMode;
ReplicaSelection = replicaSelection;
_isFaultTolerant = isFaultTolerant;
_limit = limit;
CacheBlocks = cacheBlocks;
_startTimestamp = startTimestamp;
_htTimestamp = htTimestamp;
_lowerBoundPrimaryKey = lowerBoundPrimaryKey;
_upperBoundPrimaryKey = upperBoundPrimaryKey;
_lowerBoundPartitionKey = lowerBoundPartitionKey;
_upperBoundPartitionKey = upperBoundPartitionKey;
// Add the IS_DELETED column for diff scans.
bool includeDeletedColumn = startTimestamp != KuduClient.NoTimestamp;
ProjectionSchema = GenerateProjectionSchema(
table.Schema,
projectedColumnNames,
projectedColumnIndexes,
includeDeletedColumn);
_projectedColumnsPb = ToColumnSchemaPbs(ProjectionSchema);
BatchSizeBytes = GetBatchSizeEstimate(batchSizeBytes);
}
/// <summary>
/// Sets the replica selection mechanism for this scanner.
/// The default is to read from the currently known leader.
/// </summary>
/// <param name="replicaSelection">Replication selection mechanism to use.</param>
public TBuilder SetReplicaSelection(ReplicaSelection replicaSelection)
{
ReplicaSelection = replicaSelection;
return((TBuilder)this);
}
