public static TBuilder AddIsNullPredicate <TBuilder>(
this TBuilder scanBuilder, string columnName)
where TBuilder : AbstractKuduScannerBuilder <TBuilder>
{
var column = scanBuilder.Table.Schema.GetColumn(columnName);
var predicate = KuduPredicate.NewIsNullPredicate(column);
return(scanBuilder.AddPredicate(predicate));
}
public static TBuilder AddInListPredicate <TBuilder, T>(
this TBuilder scanBuilder, string columnName, IEnumerable <T> values)
where TBuilder : AbstractKuduScannerBuilder <TBuilder>
{
var column = scanBuilder.Table.Schema.GetColumn(columnName);
var predicate = KuduPredicate.NewInListPredicate(column, values);
return(scanBuilder.AddPredicate(predicate));
}
public static TBuilder AddComparisonPredicate <TBuilder>(
this TBuilder scanBuilder, string columnName, ComparisonOp op, byte[] value)
where TBuilder : AbstractKuduScannerBuilder <TBuilder>
{
var column = scanBuilder.Table.Schema.GetColumn(columnName);
var predicate = KuduPredicate.NewComparisonPredicate(column, op, value);
return(scanBuilder.AddPredicate(predicate));
}
public async Task TestInListHashPartitionPruning()
{
// CREATE TABLE t
// (a INT8, b INT8, c INT8)
// PRIMARY KEY (a, b, c)
// PARTITION BY HASH (a) PARTITIONS 3,
// HASH (b) PARTITIONS 3,
// HASH (c) PARTITIONS 3;
var tableName = nameof(TestInListHashPartitionPruning);
var tableBuilder = new TableBuilder(tableName)
.AddColumn("a", KuduType.Int8, opt => opt.Key(true))
.AddColumn("b", KuduType.Int8, opt => opt.Key(true))
.AddColumn("c", KuduType.Int8, opt => opt.Key(true))
.AddHashPartitions(3, "a")
.AddHashPartitions(3, "b")
.AddHashPartitions(3, "c");
var table = await _client.CreateTableAsync(tableBuilder);
var partitions = await GetTablePartitionsAsync(table);
var a = table.Schema.GetColumn("a");
var b = table.Schema.GetColumn("b");
var c = table.Schema.GetColumn("c");
// a in [0, 1];
await CheckPartitionsAsync(18, 2, table, partitions,
KuduPredicate.NewInListPredicate(a, new byte[] { 0, 1 }));
// a in [0, 1, 8];
await CheckPartitionsAsync(27, 1, table, partitions,
KuduPredicate.NewInListPredicate(a, new byte[] { 0, 1, 8 }));
// b in [0, 1];
await CheckPartitionsAsync(18, 6, table, partitions,
KuduPredicate.NewInListPredicate(b, new byte[] { 0, 1 }));
// c in [0, 1];
await CheckPartitionsAsync(18, 18, table, partitions,
KuduPredicate.NewInListPredicate(c, new byte[] { 0, 1 }));
// b in [0, 1], c in [0, 1];
await CheckPartitionsAsync(12, 12, table, partitions,
KuduPredicate.NewInListPredicate(b, new byte[] { 0, 1 }),
KuduPredicate.NewInListPredicate(c, new byte[] { 0, 1 }));
// a in [0, 1], b in [0, 1], c in [0, 1];
await CheckPartitionsAsync(8, 8, table, partitions,
KuduPredicate.NewInListPredicate(a, new byte[] { 0, 1 }),
KuduPredicate.NewInListPredicate(b, new byte[] { 0, 1 }),
KuduPredicate.NewInListPredicate(c, new byte[] { 0, 1 }));
}
private static SortedSet <DateTime> CreateDateValues()
{
var values = new SortedSet <DateTime>();
for (long i = -50; i < 50; i++)
{
values.Add(EpochTime.UnixEpoch.AddDays(i));
}
values.Add(EpochTime.FromUnixTimeDays((int)KuduPredicate.MinIntValue(KuduType.Date)));
values.Add(EpochTime.FromUnixTimeDays((int)KuduPredicate.MinIntValue(KuduType.Date) + 1));
values.Add(EpochTime.FromUnixTimeDays((int)KuduPredicate.MaxIntValue(KuduType.Date) - 1));
values.Add(EpochTime.FromUnixTimeDays((int)KuduPredicate.MaxIntValue(KuduType.Date)));
return(values);
}
private static SortedSet <long> CreateIntegerValues(KuduType type)
{
var values = new SortedSet <long>();
for (long i = -50; i < 50; i++)
{
values.Add(i);
}
values.Add(KuduPredicate.MinIntValue(type));
values.Add(KuduPredicate.MinIntValue(type) + 1);
values.Add(KuduPredicate.MaxIntValue(type) - 1);
values.Add(KuduPredicate.MaxIntValue(type));
return(values);
}
private static List <DateTime> CreateDateTestValues()
{
return(new List <DateTime>
{
EpochTime.FromUnixTimeDays((int)KuduPredicate.MinIntValue(KuduType.Date)),
EpochTime.FromUnixTimeDays((int)KuduPredicate.MinIntValue(KuduType.Date) + 1),
EpochTime.UnixEpoch.AddDays(-51),
EpochTime.UnixEpoch.AddDays(-50),
EpochTime.UnixEpoch,
EpochTime.UnixEpoch.AddDays(49),
EpochTime.UnixEpoch.AddDays(50),
EpochTime.FromUnixTimeDays((int)KuduPredicate.MaxIntValue(KuduType.Date) - 1),
EpochTime.FromUnixTimeDays((int)KuduPredicate.MaxIntValue(KuduType.Date))
});
}
private static List <long> CreateIntegerTestValues(KuduType type)
{
return(new List <long>
{
KuduPredicate.MinIntValue(type),
KuduPredicate.MinIntValue(type) + 1,
-51L,
-50L,
0L,
49L,
50L,
KuduPredicate.MaxIntValue(type) - 1,
KuduPredicate.MaxIntValue(type)
});
}
/// <summary>
/// Adds a predicate to the scan.
/// </summary>
/// <param name="predicate">The predicate to add.</param>
public TBuilder AddPredicate(KuduPredicate predicate)
{
var column = predicate.Column;
var columnName = column.Name;
if (Predicates.TryGetValue(columnName, out var existing))
{
predicate = existing.Merge(predicate);
}
// KUDU-1652: Do not send an IS NOT NULL predicate to the server for a non-nullable column.
if (!column.IsNullable && predicate.Type == PredicateType.IsNotNull)
{
return((TBuilder)this);
}
Predicates[columnName] = predicate;
return((TBuilder)this);
}
public async Task TestScanTokensWithExtraPredicate()
{
int numRows = 100;
int predicateIndex = 0;
int predicateValue = 1;
var builder = ClientTestUtil.GetBasicSchema()
.SetTableName(_tableName)
.CreateBasicRangePartition();
var table = await _client.CreateTableAsync(builder);
await ClientTestUtil.LoadDefaultTableAsync(_client, table, numRows);
var tokens = await _client.NewScanTokenBuilder(table).BuildAsync();
var columnSchema = table.Schema.GetColumn(predicateIndex);
var predicate = KuduPredicate.NewComparisonPredicate(
columnSchema, ComparisonOp.Equal, predicateValue);
var resultKeys = new List <int>();
foreach (var token in tokens)
{
byte[] serialized = token.Serialize();
var scanBuilder = await _client.NewScanBuilderFromTokenAsync(serialized);
var scanner = scanBuilder
.AddPredicate(predicate)
.Build();
await foreach (var resultSet in scanner)
{
foreach (var row in resultSet)
{
resultKeys.Add(row.GetInt32(predicateIndex));
}
}
}
Assert.Collection(resultKeys, key => Assert.Equal(predicateValue, key));
}
private async Task CheckPredicatesAsync <T>(
KuduTable table,
SortedSet <T> values,
List <T> testValues)
{
var col = table.Schema.GetColumn("value");
Assert.Equal(values.Count + 1, await CountRowsAsync(table));
foreach (var v in testValues)
{
// value = v
var equal = KuduPredicate.NewComparisonPredicate(col, ComparisonOp.Equal, (dynamic)v);
Assert.Equal(values.Contains(v) ? 1 : 0, await CountRowsAsync(table, equal));
// value >= v
var greaterEqual = KuduPredicate.NewComparisonPredicate(col, ComparisonOp.GreaterEqual, (dynamic)v);
Assert.Equal(values.TailSet(v).Count, await CountRowsAsync(table, greaterEqual));
// value <= v
var lessEqual = KuduPredicate.NewComparisonPredicate(col, ComparisonOp.LessEqual, (dynamic)v);
Assert.Equal(values.HeadSet(v, true).Count, await CountRowsAsync(table, lessEqual));
// value > v
var greater = KuduPredicate.NewComparisonPredicate(col, ComparisonOp.Greater, (dynamic)v);
Assert.Equal(values.TailSet(v, false).Count, await CountRowsAsync(table, greater));
// value < v
var less = KuduPredicate.NewComparisonPredicate(col, ComparisonOp.Less, (dynamic)v);
Assert.Equal(values.HeadSet(v).Count, await CountRowsAsync(table, less));
}
var isNotNull = KuduPredicate.NewIsNotNullPredicate(col);
Assert.Equal(values.Count, await CountRowsAsync(table, isNotNull));
var isNull = KuduPredicate.NewIsNullPredicate(col);
Assert.Equal(1, await CountRowsAsync(table, isNull));
}
public async Task TestHashPartitionPruning()
{
// CREATE TABLE t
// (a INT8, b INT8, c INT8)
// PRIMARY KEY (a, b, c)
// PARTITION BY HASH (a) PARTITIONS 2,
// HASH (b, c) PARTITIONS 2;
var tableName = nameof(TestHashPartitionPruning);
var tableBuilder = new TableBuilder(tableName)
.AddColumn("a", KuduType.Int8, opt => opt.Key(true))
.AddColumn("b", KuduType.Int8, opt => opt.Key(true))
.AddColumn("c", KuduType.Int8, opt => opt.Key(true))
.AddHashPartitions(2, "a")
.AddHashPartitions(2, "b", "c");
var table = await _client.CreateTableAsync(tableBuilder);
var partitions = await GetTablePartitionsAsync(table);
var a = table.Schema.GetColumn("a");
var b = table.Schema.GetColumn("b");
var c = table.Schema.GetColumn("c");
// No Predicates
await CheckPartitionsAsync(4, 1, table, partitions);
// a = 0;
await CheckPartitionsAsync(2, 1, table, partitions,
KuduPredicate.NewComparisonPredicate(a, ComparisonOp.Equal, 0));
// a >= 0;
await CheckPartitionsAsync(4, 1, table, partitions,
KuduPredicate.NewComparisonPredicate(a, ComparisonOp.GreaterEqual, 0));
// a >= 0;
// a < 1;
await CheckPartitionsAsync(2, 1, table, partitions,
KuduPredicate.NewComparisonPredicate(a, ComparisonOp.GreaterEqual, 0),
KuduPredicate.NewComparisonPredicate(a, ComparisonOp.Less, 1));
// a >= 0;
// a < 2;
await CheckPartitionsAsync(4, 1, table, partitions,
KuduPredicate.NewComparisonPredicate(a, ComparisonOp.GreaterEqual, 0),
KuduPredicate.NewComparisonPredicate(a, ComparisonOp.Less, 2));
// b = 1;
await CheckPartitionsAsync(4, 1, table, partitions,
KuduPredicate.NewComparisonPredicate(b, ComparisonOp.Equal, 1));
// b = 1;
// c = 2;
await CheckPartitionsAsync(2, 2, table, partitions,
KuduPredicate.NewComparisonPredicate(b, ComparisonOp.Equal, 1),
KuduPredicate.NewComparisonPredicate(c, ComparisonOp.Equal, 2));
// a = 0;
// b = 1;
// c = 2;
await CheckPartitionsAsync(1, 1, table, partitions,
KuduPredicate.NewComparisonPredicate(a, ComparisonOp.Equal, 0),
KuduPredicate.NewComparisonPredicate(b, ComparisonOp.Equal, 1),
KuduPredicate.NewComparisonPredicate(c, ComparisonOp.Equal, 2));
// a IN (0, 10)
await CheckPartitionsAsync(4, 1, table, partitions,
KuduPredicate.NewInListPredicate(c, new byte[] { 0, 10 }));
}
public async Task TestRangePartitionPruning()
{
// CREATE TABLE t
// (a INT8, b STRING, c INT8)
// PRIMARY KEY (a, b, c))
// PARTITION BY RANGE (c, b)
// (PARTITION VALUES < (0, "m"),
// PARTITION (0, "m") <= VALUES < (10, "r")
// PARTITION (10, "r") <= VALUES);
var tableName = nameof(TestRangePartitionPruning);
var tableBuilder = new TableBuilder(tableName)
.AddColumn("a", KuduType.Int8, opt => opt.Key(true))
.AddColumn("b", KuduType.String, opt => opt.Key(true))
.AddColumn("c", KuduType.Int8, opt => opt.Key(true))
.SetRangePartitionColumns("c", "b")
.AddSplitRow(row =>
{
row.SetByte("c", 0);
row.SetString("b", "m");
})
.AddSplitRow(row =>
{
row.SetByte("c", 10);
row.SetString("b", "r");
});
var table = await _client.CreateTableAsync(tableBuilder);
var partitions = await GetTablePartitionsAsync(table);
var b = table.Schema.GetColumn("b");
var c = table.Schema.GetColumn("c");
// No Predicates
await CheckPartitionsAsync(3, 1, table, partitions);
// c < -10
await CheckPartitionsAsync(1, 1, table, partitions,
KuduPredicate.NewComparisonPredicate(c, ComparisonOp.Less, -10));
// c = -10
await CheckPartitionsAsync(1, 1, table, partitions,
KuduPredicate.NewComparisonPredicate(c, ComparisonOp.Equal, -10));
// c < 10
await CheckPartitionsAsync(2, 1, table, partitions,
KuduPredicate.NewComparisonPredicate(c, ComparisonOp.Less, 10));
// c < 100
await CheckPartitionsAsync(3, 1, table, partitions,
KuduPredicate.NewComparisonPredicate(c, ComparisonOp.Less, 100));
// c < MIN
await CheckPartitionsAsync(0, 0, table, partitions,
KuduPredicate.NewComparisonPredicate(c, ComparisonOp.Less, sbyte.MinValue));
// c < MAX
await CheckPartitionsAsync(3, 1, table, partitions,
KuduPredicate.NewComparisonPredicate(c, ComparisonOp.Less, sbyte.MaxValue));
// c >= -10
await CheckPartitionsAsync(3, 1, table, partitions,
KuduPredicate.NewComparisonPredicate(c, ComparisonOp.GreaterEqual, -10));
// c >= 0
await CheckPartitionsAsync(3, 1, table, partitions,
KuduPredicate.NewComparisonPredicate(c, ComparisonOp.GreaterEqual, -10));
// c >= 5
await CheckPartitionsAsync(2, 1, table, partitions,
KuduPredicate.NewComparisonPredicate(c, ComparisonOp.GreaterEqual, 5));
// c >= 10
await CheckPartitionsAsync(2, 1, table, partitions,
KuduPredicate.NewComparisonPredicate(c, ComparisonOp.GreaterEqual, 10));
// c >= 100
await CheckPartitionsAsync(1, 1, table, partitions,
KuduPredicate.NewComparisonPredicate(c, ComparisonOp.GreaterEqual, 100));
// c >= MIN
await CheckPartitionsAsync(3, 1, table, partitions,
KuduPredicate.NewComparisonPredicate(c, ComparisonOp.GreaterEqual, sbyte.MinValue));
// c >= MAX
await CheckPartitionsAsync(1, 1, table, partitions,
KuduPredicate.NewComparisonPredicate(c, ComparisonOp.GreaterEqual, sbyte.MaxValue));
// c >= -10
// c < 0
await CheckPartitionsAsync(1, 1, table, partitions,
KuduPredicate.NewComparisonPredicate(c, ComparisonOp.GreaterEqual, -10),
KuduPredicate.NewComparisonPredicate(c, ComparisonOp.Less, 0));
// c >= 5
// c < 100
await CheckPartitionsAsync(2, 1, table, partitions,
KuduPredicate.NewComparisonPredicate(c, ComparisonOp.GreaterEqual, 5),
KuduPredicate.NewComparisonPredicate(c, ComparisonOp.Less, 100));
// b = ""
await CheckPartitionsAsync(3, 1, table, partitions,
KuduPredicate.NewComparisonPredicate(b, ComparisonOp.Equal, ""));
// b >= "z"
await CheckPartitionsAsync(3, 1, table, partitions,
KuduPredicate.NewComparisonPredicate(b, ComparisonOp.GreaterEqual, "z"));
// b < "a"
await CheckPartitionsAsync(3, 1, table, partitions,
KuduPredicate.NewComparisonPredicate(b, ComparisonOp.Less, "a"));
// b >= "m"
// b < "z"
await CheckPartitionsAsync(3, 1, table, partitions,
KuduPredicate.NewComparisonPredicate(b, ComparisonOp.GreaterEqual, "m"),
KuduPredicate.NewComparisonPredicate(b, ComparisonOp.Less, "z"));
// c >= 10
// b >= "r"
await CheckPartitionsAsync(1, 1, table, partitions,
KuduPredicate.NewComparisonPredicate(c, ComparisonOp.GreaterEqual, 10),
KuduPredicate.NewComparisonPredicate(b, ComparisonOp.GreaterEqual, "r"));
// c >= 10
// b < "r"
await CheckPartitionsAsync(2, 1, table, partitions,
KuduPredicate.NewComparisonPredicate(c, ComparisonOp.GreaterEqual, 10),
KuduPredicate.NewComparisonPredicate(b, ComparisonOp.Less, "r"));
// c = 10
// b < "r"
await CheckPartitionsAsync(1, 1, table, partitions,
KuduPredicate.NewComparisonPredicate(c, ComparisonOp.Equal, 10),
KuduPredicate.NewComparisonPredicate(b, ComparisonOp.Less, "r"));
// c < 0
// b < "m"
await CheckPartitionsAsync(1, 1, table, partitions,
KuduPredicate.NewComparisonPredicate(c, ComparisonOp.Equal, 0),
KuduPredicate.NewComparisonPredicate(b, ComparisonOp.Less, "m"));
// c < 0
// b < "z"
await CheckPartitionsAsync(1, 1, table, partitions,
KuduPredicate.NewComparisonPredicate(c, ComparisonOp.Less, 0),
KuduPredicate.NewComparisonPredicate(b, ComparisonOp.Less, "z"));
// c = 0
// b = "m\0"
await CheckPartitionsAsync(1, 1, table, partitions,
KuduPredicate.NewComparisonPredicate(c, ComparisonOp.Equal, 0),
KuduPredicate.NewComparisonPredicate(b, ComparisonOp.Equal, "m\0"));
// c = 0
// b < "m"
await CheckPartitionsAsync(1, 1, table, partitions,
KuduPredicate.NewComparisonPredicate(c, ComparisonOp.Equal, 0),
KuduPredicate.NewComparisonPredicate(b, ComparisonOp.Less, "m"));
// c = 0
// b < "m\0"
await CheckPartitionsAsync(2, 1, table, partitions,
KuduPredicate.NewComparisonPredicate(c, ComparisonOp.Equal, 0),
KuduPredicate.NewComparisonPredicate(b, ComparisonOp.Less, "m\0"));
// c = 0
// c = 2
await CheckPartitionsAsync(0, 0, table, partitions,
KuduPredicate.NewComparisonPredicate(c, ComparisonOp.Equal, 0),
KuduPredicate.NewComparisonPredicate(c, ComparisonOp.Equal, 2));
// c = MIN
await CheckPartitionsAsync(1, 1, table, partitions,
KuduPredicate.NewComparisonPredicate(c, ComparisonOp.Equal, sbyte.MinValue));
// c = MAX
await CheckPartitionsAsync(1, 1, table, partitions,
KuduPredicate.NewComparisonPredicate(c, ComparisonOp.Equal, sbyte.MaxValue));
// c IN (1, 2)
await CheckPartitionsAsync(1, 1, table, partitions,
KuduPredicate.NewInListPredicate(c, new byte[] { 1, 2 }));
// c IN (0, 1, 2)
await CheckPartitionsAsync(2, 1, table, partitions,
KuduPredicate.NewInListPredicate(c, new byte[] { 0, 1, 2 }));
// c IN (-10, 0)
// b < "m"
await CheckPartitionsAsync(1, 1, table, partitions,
KuduPredicate.NewInListPredicate(c, new sbyte[] { -10, 0 }),
KuduPredicate.NewComparisonPredicate(b, ComparisonOp.Less, "m"));
// c IN (-10, 0)
// b < "m\0"
await CheckPartitionsAsync(2, 1, table, partitions,
KuduPredicate.NewInListPredicate(c, new sbyte[] { -10, 0 }),
KuduPredicate.NewComparisonPredicate(b, ComparisonOp.Less, "m\0"));
}
public async Task TestBinaryPredicates()
{
var builder = GetDefaultTableBuilder()
.SetTableName("binary-table")
.AddColumn("value", KuduType.Binary);
var table = await _client.CreateTableAsync(builder);
var values = CreateStringValues();
var testValues = CreateStringTestValues();
long i = 0;
foreach (var value in values)
{
var insert = table.NewInsert();
insert.SetInt64("key", i++);
insert.SetBinary("value", value.ToUtf8ByteArray());
await _session.EnqueueAsync(insert);
}
var nullInsert = table.NewInsert();
nullInsert.SetInt64("key", i);
nullInsert.SetNull("value");
await _session.EnqueueAsync(nullInsert);
await _session.FlushAsync();
var col = table.Schema.GetColumn("value");
Assert.Equal(values.Count + 1, await CountRowsAsync(table));
foreach (var s in testValues)
{
var v = s.ToUtf8ByteArray();
// value = v
var equal = KuduPredicate.NewComparisonPredicate(col, ComparisonOp.Equal, v);
Assert.Equal(values.GetViewBetween(s, s).Count, await CountRowsAsync(table, equal));
// value >= v
var greaterEqual = KuduPredicate.NewComparisonPredicate(col, ComparisonOp.GreaterEqual, v);
Assert.Equal(values.TailSet(s).Count, await CountRowsAsync(table, greaterEqual));
// value <= v
var lessEqual = KuduPredicate.NewComparisonPredicate(col, ComparisonOp.LessEqual, v);
Assert.Equal(values.HeadSet(s, true).Count, await CountRowsAsync(table, lessEqual));
// value > v
var greater = KuduPredicate.NewComparisonPredicate(col, ComparisonOp.Greater, v);
Assert.Equal(values.TailSet(s, false).Count, await CountRowsAsync(table, greater));
// value < v
var less = KuduPredicate.NewComparisonPredicate(col, ComparisonOp.Less, v);
Assert.Equal(values.HeadSet(s).Count, await CountRowsAsync(table, less));
}
var isNotNull = KuduPredicate.NewIsNotNullPredicate(col);
Assert.Equal(values.Count, await CountRowsAsync(table, isNotNull));
var isNull = KuduPredicate.NewIsNullPredicate(col);
Assert.Equal(1, await CountRowsAsync(table, isNull));
}
public async Task TestMultiColumnInListHashPruning()
{
// CREATE TABLE t
// (a INT8, b INT8, c INT8)
// PRIMARY KEY (a, b, c)
// PARTITION BY HASH (a) PARTITIONS 3,
// HASH (b, c) PARTITIONS 3;
var tableName = nameof(TestMultiColumnInListHashPruning);
var tableBuilder = new TableBuilder(tableName)
.AddColumn("a", KuduType.Int8, opt => opt.Key(true))
.AddColumn("b", KuduType.Int8, opt => opt.Key(true))
.AddColumn("c", KuduType.Int8, opt => opt.Key(true))
.AddHashPartitions(3, "a")
.AddHashPartitions(3, "b", "c");
var table = await _client.CreateTableAsync(tableBuilder);
var partitions = await GetTablePartitionsAsync(table);
var a = table.Schema.GetColumn("a");
var b = table.Schema.GetColumn("b");
var c = table.Schema.GetColumn("c");
// a in [0, 1];
await CheckPartitionsAsync(6, 2, table, partitions,
KuduPredicate.NewInListPredicate(a, new byte[] { 0, 1 }));
// a in [0, 1, 8];
await CheckPartitionsAsync(9, 1, table, partitions,
KuduPredicate.NewInListPredicate(a, new byte[] { 0, 1, 8 }));
// b in [0, 1];
await CheckPartitionsAsync(9, 1, table, partitions,
KuduPredicate.NewInListPredicate(b, new byte[] { 0, 1 }));
// c in [0, 1];
await CheckPartitionsAsync(9, 1, table, partitions,
KuduPredicate.NewInListPredicate(c, new byte[] { 0, 1 }));
// b in [0, 1], c in [0, 1]
// (0, 0) in bucket 2
// (0, 1) in bucket 2
// (1, 0) in bucket 1
// (1, 1) in bucket 0
await CheckPartitionsAsync(9, 1, table, partitions,
KuduPredicate.NewInListPredicate(b, new byte[] { 0, 1 }),
KuduPredicate.NewInListPredicate(c, new byte[] { 0, 1 }));
// b = 0, c in [0, 1]
await CheckPartitionsAsync(3, 3, table, partitions,
KuduPredicate.NewComparisonPredicate(b, ComparisonOp.Equal, 0),
KuduPredicate.NewInListPredicate(c, new byte[] { 0, 1 }));
// b = 1, c in [0, 1]
await CheckPartitionsAsync(6, 6, table, partitions,
KuduPredicate.NewComparisonPredicate(b, ComparisonOp.Equal, 1),
KuduPredicate.NewInListPredicate(c, new byte[] { 0, 1 }));
// a in [0, 1], b in [0, 1], c in [0, 1];
await CheckPartitionsAsync(6, 2, table, partitions,
KuduPredicate.NewInListPredicate(a, new byte[] { 0, 1 }),
KuduPredicate.NewInListPredicate(b, new byte[] { 0, 1 }),
KuduPredicate.NewInListPredicate(c, new byte[] { 0, 1 }));
}
public async Task TestPruning()
{
// CREATE TABLE timeseries
// (host STRING, metric STRING, timestamp UNIXTIME_MICROS, value DOUBLE)
// PRIMARY KEY (host, metric, time)
// DISTRIBUTE BY
// RANGE(time)
// (PARTITION VALUES < 10,
// PARTITION VALUES >= 10);
// HASH (host, metric) 2 PARTITIONS;
var tableName = nameof(TestPruning);
var tableBuilder = new TableBuilder(tableName)
.AddColumn("host", KuduType.String, opt => opt.Key(true))
.AddColumn("metric", KuduType.String, opt => opt.Key(true))
.AddColumn("timestamp", KuduType.UnixtimeMicros, opt => opt.Key(true))
.AddColumn("value", KuduType.Double)
.SetRangePartitionColumns("timestamp")
.AddSplitRow(row => row.SetInt64("timestamp", 10))
.AddHashPartitions(2, "host", "metric");
var table = await _client.CreateTableAsync(tableBuilder);
var partitions = await GetTablePartitionsAsync(table);
var host = table.Schema.GetColumn("host");
var metric = table.Schema.GetColumn("metric");
var timestamp = table.Schema.GetColumn("timestamp");
// No Predicates
await CheckPartitionsAsync(4, 1, table, partitions);
// host = "a"
await CheckPartitionsAsync(4, 1, table, partitions,
KuduPredicate.NewComparisonPredicate(host, ComparisonOp.Equal, "a"));
// host = "a"
// metric = "a"
await CheckPartitionsAsync(2, 1, table, partitions,
KuduPredicate.NewComparisonPredicate(host, ComparisonOp.Equal, "a"),
KuduPredicate.NewComparisonPredicate(metric, ComparisonOp.Equal, "a"));
// host = "a"
// metric = "a"
// timestamp >= 9;
await CheckPartitionsAsync(2, 1, table, partitions,
KuduPredicate.NewComparisonPredicate(host, ComparisonOp.Equal, "a"),
KuduPredicate.NewComparisonPredicate(metric, ComparisonOp.Equal, "a"),
KuduPredicate.NewComparisonPredicate(timestamp, ComparisonOp.GreaterEqual, 9));
// host = "a"
// metric = "a"
// timestamp >= 10;
// timestamp < 20;
await CheckPartitionsAsync(1, 1, table, partitions,
KuduPredicate.NewComparisonPredicate(host, ComparisonOp.Equal, "a"),
KuduPredicate.NewComparisonPredicate(metric, ComparisonOp.Equal, "a"),
KuduPredicate.NewComparisonPredicate(timestamp, ComparisonOp.GreaterEqual, 10),
KuduPredicate.NewComparisonPredicate(timestamp, ComparisonOp.Less, 20));
// host = "a"
// metric = "a"
// timestamp < 10;
await CheckPartitionsAsync(1, 1, table, partitions,
KuduPredicate.NewComparisonPredicate(host, ComparisonOp.Equal, "a"),
KuduPredicate.NewComparisonPredicate(metric, ComparisonOp.Equal, "a"),
KuduPredicate.NewComparisonPredicate(timestamp, ComparisonOp.Less, 10));
// host = "a"
// metric = "a"
// timestamp >= 10;
await CheckPartitionsAsync(1, 1, table, partitions,
KuduPredicate.NewComparisonPredicate(host, ComparisonOp.Equal, "a"),
KuduPredicate.NewComparisonPredicate(metric, ComparisonOp.Equal, "a"),
KuduPredicate.NewComparisonPredicate(timestamp, ComparisonOp.GreaterEqual, 10));
// host = "a"
// metric = "a"
// timestamp = 10;
await CheckPartitionsAsync(1, 1, table, partitions,
KuduPredicate.NewComparisonPredicate(host, ComparisonOp.Equal, "a"),
KuduPredicate.NewComparisonPredicate(metric, ComparisonOp.Equal, "a"),
KuduPredicate.NewComparisonPredicate(timestamp, ComparisonOp.Equal, 10));
byte[] hash1 = new byte[] { 0, 0, 0, 1 };
// partition key < (hash=1)
await CheckPartitionsAsync(2, 1, table, partitions, null, hash1);
// partition key >= (hash=1)
await CheckPartitionsAsync(2, 1, table, partitions, hash1, null);
// timestamp = 10
// partition key < (hash=1)
await CheckPartitionsAsync(1, 1, table, partitions, null, hash1,
KuduPredicate.NewComparisonPredicate(timestamp, ComparisonOp.Equal, 10));
// timestamp = 10
// partition key >= (hash=1)
await CheckPartitionsAsync(1, 1, table, partitions, hash1, null,
KuduPredicate.NewComparisonPredicate(timestamp, ComparisonOp.Equal, 10));
// timestamp IN (0, 9)
// host = "a"
// metric IN ("foo", "baz")
await CheckPartitionsAsync(1, 1, table, partitions,
KuduPredicate.NewInListPredicate(timestamp, new long[] { 0, 9 }),
KuduPredicate.NewComparisonPredicate(host, ComparisonOp.Equal, "a"),
KuduPredicate.NewInListPredicate(metric, new string[] { "foo", "baz" }));
// timestamp IN (10, 100)
await CheckPartitionsAsync(2, 2, table, partitions,
KuduPredicate.NewInListPredicate(timestamp, new long[] { 10, 100 }));
// timestamp IN (9, 10)
await CheckPartitionsAsync(4, 2, table, partitions,
KuduPredicate.NewInListPredicate(timestamp, new long[] { 9, 10 }));
// timestamp IS NOT NULL
await CheckPartitionsAsync(4, 1, table, partitions,
KuduPredicate.NewIsNotNullPredicate(timestamp));
// timestamp IS NULL
await CheckPartitionsAsync(0, 0, table, partitions,
KuduPredicate.NewIsNullPredicate(timestamp));
}
internal static void PbIntoScanner <TBuilder>(
TBuilder builder, ScanTokenPB scanTokenPb)
where TBuilder : AbstractKuduScannerBuilder <TBuilder>
{
var table = builder.Table;
var schema = table.Schema;
if (scanTokenPb.FeatureFlags.Contains(ScanTokenPB.Types.Feature.Unknown))
{
throw new Exception("Scan token requires an unsupported feature. " +
"This Kudu client must be updated.");
}
builder.SetProjectedColumns(
ComputeProjectedColumnIndexes(scanTokenPb, schema));
foreach (var predicate in scanTokenPb.ColumnPredicates)
{
builder.AddPredicate(KuduPredicate.FromPb(schema, predicate));
}
if (scanTokenPb.HasLowerBoundPrimaryKey)
{
builder.LowerBoundRaw(scanTokenPb.LowerBoundPrimaryKey.ToByteArray());
}
if (scanTokenPb.HasUpperBoundPrimaryKey)
{
builder.ExclusiveUpperBoundRaw(scanTokenPb.UpperBoundPrimaryKey.ToByteArray());
}
if (scanTokenPb.HasLowerBoundPartitionKey)
{
builder.LowerBoundPartitionKeyRaw(scanTokenPb.LowerBoundPartitionKey.ToByteArray());
}
if (scanTokenPb.HasUpperBoundPartitionKey)
{
builder.ExclusiveUpperBoundPartitionKeyRaw(scanTokenPb.UpperBoundPartitionKey.ToByteArray());
}
if (scanTokenPb.HasLimit)
{
builder.SetLimit((long)scanTokenPb.Limit);
}
if (scanTokenPb.HasReadMode)
{
switch (scanTokenPb.ReadMode)
{
case Protobuf.ReadMode.ReadAtSnapshot:
{
builder.SetReadMode(ReadMode.ReadAtSnapshot);
if (scanTokenPb.HasSnapTimestamp)
{
builder.SnapshotTimestampRaw((long)scanTokenPb.SnapTimestamp);
}
// Set the diff scan timestamps if they are set.
if (scanTokenPb.HasSnapStartTimestamp)
{
builder.DiffScan(
(long)scanTokenPb.SnapStartTimestamp,
(long)scanTokenPb.SnapTimestamp);
}
break;
}
case Protobuf.ReadMode.ReadLatest:
{
builder.SetReadMode(ReadMode.ReadLatest);
break;
}
case Protobuf.ReadMode.ReadYourWrites:
{
builder.SetReadMode(ReadMode.ReadYourWrites);
break;
}
default:
throw new Exception("Unknown read mode");
}
}
if (scanTokenPb.HasReplicaSelection)
{
switch (scanTokenPb.ReplicaSelection)
{
case Protobuf.ReplicaSelection.LeaderOnly:
builder.SetReplicaSelection(ReplicaSelection.LeaderOnly);
break;
case Protobuf.ReplicaSelection.ClosestReplica:
builder.SetReplicaSelection(ReplicaSelection.ClosestReplica);
break;
default:
throw new Exception("Unknown replica selection policy");
}
}
if (scanTokenPb.HasPropagatedTimestamp &&
(long)scanTokenPb.PropagatedTimestamp != KuduClient.NoTimestamp)
{
builder.Client.LastPropagatedTimestamp = (long)scanTokenPb.PropagatedTimestamp;
}
if (scanTokenPb.HasCacheBlocks)
{
builder.SetCacheBlocks(scanTokenPb.CacheBlocks);
}
if (scanTokenPb.HasFaultTolerant)
{
builder.SetFaultTolerant(scanTokenPb.FaultTolerant);
}
if (scanTokenPb.HasBatchSizeBytes)
{
builder.SetBatchSizeBytes((int)scanTokenPb.BatchSizeBytes);
}
if (scanTokenPb.HasScanRequestTimeoutMs)
{
// TODO
}
if (scanTokenPb.HasKeepAlivePeriodMs)
{
// TODO
}
}