public static TBuilder AddIsNotNullPredicate <TBuilder>(
this TBuilder scanBuilder, string columnName)
where TBuilder : AbstractKuduScannerBuilder <TBuilder>
{
var column = scanBuilder.Table.Schema.GetColumn(columnName);
var predicate = KuduPredicate.NewIsNotNullPredicate(column);
return(scanBuilder.AddPredicate(predicate));
}
private async Task CheckPredicatesAsync <T>(
KuduTable table,
SortedSet <T> values,
List <T> testValues)
{
var col = table.Schema.GetColumn("value");
Assert.Equal(values.Count + 1, await CountRowsAsync(table));
foreach (var v in testValues)
{
// value = v
var equal = KuduPredicate.NewComparisonPredicate(col, ComparisonOp.Equal, (dynamic)v);
Assert.Equal(values.Contains(v) ? 1 : 0, await CountRowsAsync(table, equal));
// value >= v
var greaterEqual = KuduPredicate.NewComparisonPredicate(col, ComparisonOp.GreaterEqual, (dynamic)v);
Assert.Equal(values.TailSet(v).Count, await CountRowsAsync(table, greaterEqual));
// value <= v
var lessEqual = KuduPredicate.NewComparisonPredicate(col, ComparisonOp.LessEqual, (dynamic)v);
Assert.Equal(values.HeadSet(v, true).Count, await CountRowsAsync(table, lessEqual));
// value > v
var greater = KuduPredicate.NewComparisonPredicate(col, ComparisonOp.Greater, (dynamic)v);
Assert.Equal(values.TailSet(v, false).Count, await CountRowsAsync(table, greater));
// value < v
var less = KuduPredicate.NewComparisonPredicate(col, ComparisonOp.Less, (dynamic)v);
Assert.Equal(values.HeadSet(v).Count, await CountRowsAsync(table, less));
}
var isNotNull = KuduPredicate.NewIsNotNullPredicate(col);
Assert.Equal(values.Count, await CountRowsAsync(table, isNotNull));
var isNull = KuduPredicate.NewIsNullPredicate(col);
Assert.Equal(1, await CountRowsAsync(table, isNull));
}
public async Task TestPruning()
{
// CREATE TABLE timeseries
// (host STRING, metric STRING, timestamp UNIXTIME_MICROS, value DOUBLE)
// PRIMARY KEY (host, metric, time)
// DISTRIBUTE BY
// RANGE(time)
// (PARTITION VALUES < 10,
// PARTITION VALUES >= 10);
// HASH (host, metric) 2 PARTITIONS;
var tableName = nameof(TestPruning);
var tableBuilder = new TableBuilder(tableName)
.AddColumn("host", KuduType.String, opt => opt.Key(true))
.AddColumn("metric", KuduType.String, opt => opt.Key(true))
.AddColumn("timestamp", KuduType.UnixtimeMicros, opt => opt.Key(true))
.AddColumn("value", KuduType.Double)
.SetRangePartitionColumns("timestamp")
.AddSplitRow(row => row.SetInt64("timestamp", 10))
.AddHashPartitions(2, "host", "metric");
var table = await _client.CreateTableAsync(tableBuilder);
var partitions = await GetTablePartitionsAsync(table);
var host = table.Schema.GetColumn("host");
var metric = table.Schema.GetColumn("metric");
var timestamp = table.Schema.GetColumn("timestamp");
// No Predicates
await CheckPartitionsAsync(4, 1, table, partitions);
// host = "a"
await CheckPartitionsAsync(4, 1, table, partitions,
KuduPredicate.NewComparisonPredicate(host, ComparisonOp.Equal, "a"));
// host = "a"
// metric = "a"
await CheckPartitionsAsync(2, 1, table, partitions,
KuduPredicate.NewComparisonPredicate(host, ComparisonOp.Equal, "a"),
KuduPredicate.NewComparisonPredicate(metric, ComparisonOp.Equal, "a"));
// host = "a"
// metric = "a"
// timestamp >= 9;
await CheckPartitionsAsync(2, 1, table, partitions,
KuduPredicate.NewComparisonPredicate(host, ComparisonOp.Equal, "a"),
KuduPredicate.NewComparisonPredicate(metric, ComparisonOp.Equal, "a"),
KuduPredicate.NewComparisonPredicate(timestamp, ComparisonOp.GreaterEqual, 9));
// host = "a"
// metric = "a"
// timestamp >= 10;
// timestamp < 20;
await CheckPartitionsAsync(1, 1, table, partitions,
KuduPredicate.NewComparisonPredicate(host, ComparisonOp.Equal, "a"),
KuduPredicate.NewComparisonPredicate(metric, ComparisonOp.Equal, "a"),
KuduPredicate.NewComparisonPredicate(timestamp, ComparisonOp.GreaterEqual, 10),
KuduPredicate.NewComparisonPredicate(timestamp, ComparisonOp.Less, 20));
// host = "a"
// metric = "a"
// timestamp < 10;
await CheckPartitionsAsync(1, 1, table, partitions,
KuduPredicate.NewComparisonPredicate(host, ComparisonOp.Equal, "a"),
KuduPredicate.NewComparisonPredicate(metric, ComparisonOp.Equal, "a"),
KuduPredicate.NewComparisonPredicate(timestamp, ComparisonOp.Less, 10));
// host = "a"
// metric = "a"
// timestamp >= 10;
await CheckPartitionsAsync(1, 1, table, partitions,
KuduPredicate.NewComparisonPredicate(host, ComparisonOp.Equal, "a"),
KuduPredicate.NewComparisonPredicate(metric, ComparisonOp.Equal, "a"),
KuduPredicate.NewComparisonPredicate(timestamp, ComparisonOp.GreaterEqual, 10));
// host = "a"
// metric = "a"
// timestamp = 10;
await CheckPartitionsAsync(1, 1, table, partitions,
KuduPredicate.NewComparisonPredicate(host, ComparisonOp.Equal, "a"),
KuduPredicate.NewComparisonPredicate(metric, ComparisonOp.Equal, "a"),
KuduPredicate.NewComparisonPredicate(timestamp, ComparisonOp.Equal, 10));
byte[] hash1 = new byte[] { 0, 0, 0, 1 };
// partition key < (hash=1)
await CheckPartitionsAsync(2, 1, table, partitions, null, hash1);
// partition key >= (hash=1)
await CheckPartitionsAsync(2, 1, table, partitions, hash1, null);
// timestamp = 10
// partition key < (hash=1)
await CheckPartitionsAsync(1, 1, table, partitions, null, hash1,
KuduPredicate.NewComparisonPredicate(timestamp, ComparisonOp.Equal, 10));
// timestamp = 10
// partition key >= (hash=1)
await CheckPartitionsAsync(1, 1, table, partitions, hash1, null,
KuduPredicate.NewComparisonPredicate(timestamp, ComparisonOp.Equal, 10));
// timestamp IN (0, 9)
// host = "a"
// metric IN ("foo", "baz")
await CheckPartitionsAsync(1, 1, table, partitions,
KuduPredicate.NewInListPredicate(timestamp, new long[] { 0, 9 }),
KuduPredicate.NewComparisonPredicate(host, ComparisonOp.Equal, "a"),
KuduPredicate.NewInListPredicate(metric, new string[] { "foo", "baz" }));
// timestamp IN (10, 100)
await CheckPartitionsAsync(2, 2, table, partitions,
KuduPredicate.NewInListPredicate(timestamp, new long[] { 10, 100 }));
// timestamp IN (9, 10)
await CheckPartitionsAsync(4, 2, table, partitions,
KuduPredicate.NewInListPredicate(timestamp, new long[] { 9, 10 }));
// timestamp IS NOT NULL
await CheckPartitionsAsync(4, 1, table, partitions,
KuduPredicate.NewIsNotNullPredicate(timestamp));
// timestamp IS NULL
await CheckPartitionsAsync(0, 0, table, partitions,
KuduPredicate.NewIsNullPredicate(timestamp));
}
public async Task TestBinaryPredicates()
{
var builder = GetDefaultTableBuilder()
.SetTableName("binary-table")
.AddColumn("value", KuduType.Binary);
var table = await _client.CreateTableAsync(builder);
var values = CreateStringValues();
var testValues = CreateStringTestValues();
long i = 0;
foreach (var value in values)
{
var insert = table.NewInsert();
insert.SetInt64("key", i++);
insert.SetBinary("value", value.ToUtf8ByteArray());
await _session.EnqueueAsync(insert);
}
var nullInsert = table.NewInsert();
nullInsert.SetInt64("key", i);
nullInsert.SetNull("value");
await _session.EnqueueAsync(nullInsert);
await _session.FlushAsync();
var col = table.Schema.GetColumn("value");
Assert.Equal(values.Count + 1, await CountRowsAsync(table));
foreach (var s in testValues)
{
var v = s.ToUtf8ByteArray();
// value = v
var equal = KuduPredicate.NewComparisonPredicate(col, ComparisonOp.Equal, v);
Assert.Equal(values.GetViewBetween(s, s).Count, await CountRowsAsync(table, equal));
// value >= v
var greaterEqual = KuduPredicate.NewComparisonPredicate(col, ComparisonOp.GreaterEqual, v);
Assert.Equal(values.TailSet(s).Count, await CountRowsAsync(table, greaterEqual));
// value <= v
var lessEqual = KuduPredicate.NewComparisonPredicate(col, ComparisonOp.LessEqual, v);
Assert.Equal(values.HeadSet(s, true).Count, await CountRowsAsync(table, lessEqual));
// value > v
var greater = KuduPredicate.NewComparisonPredicate(col, ComparisonOp.Greater, v);
Assert.Equal(values.TailSet(s, false).Count, await CountRowsAsync(table, greater));
// value < v
var less = KuduPredicate.NewComparisonPredicate(col, ComparisonOp.Less, v);
Assert.Equal(values.HeadSet(s).Count, await CountRowsAsync(table, less));
}
var isNotNull = KuduPredicate.NewIsNotNullPredicate(col);
Assert.Equal(values.Count, await CountRowsAsync(table, isNotNull));
var isNull = KuduPredicate.NewIsNullPredicate(col);
Assert.Equal(1, await CountRowsAsync(table, isNull));
}