コード例 #1
0
    public static TBuilder AddIsNullPredicate <TBuilder>(
        this TBuilder scanBuilder, string columnName)
        where TBuilder : AbstractKuduScannerBuilder <TBuilder>
    {
        var column    = scanBuilder.Table.Schema.GetColumn(columnName);
        var predicate = KuduPredicate.NewIsNullPredicate(column);

        return(scanBuilder.AddPredicate(predicate));
    }
コード例 #2
0
    public static TBuilder AddInListPredicate <TBuilder, T>(
        this TBuilder scanBuilder, string columnName, IEnumerable <T> values)
        where TBuilder : AbstractKuduScannerBuilder <TBuilder>
    {
        var column    = scanBuilder.Table.Schema.GetColumn(columnName);
        var predicate = KuduPredicate.NewInListPredicate(column, values);

        return(scanBuilder.AddPredicate(predicate));
    }
コード例 #3
0
    public static TBuilder AddComparisonPredicate <TBuilder>(
        this TBuilder scanBuilder, string columnName, ComparisonOp op, byte[] value)
        where TBuilder : AbstractKuduScannerBuilder <TBuilder>
    {
        var column    = scanBuilder.Table.Schema.GetColumn(columnName);
        var predicate = KuduPredicate.NewComparisonPredicate(column, op, value);

        return(scanBuilder.AddPredicate(predicate));
    }
コード例 #4
0
    public async Task TestInListHashPartitionPruning()
    {
        // CREATE TABLE t
        // (a INT8, b INT8, c INT8)
        // PRIMARY KEY (a, b, c)
        // PARTITION BY HASH (a) PARTITIONS 3,
        //              HASH (b) PARTITIONS 3,
        //              HASH (c) PARTITIONS 3;

        var tableName = nameof(TestInListHashPartitionPruning);

        var tableBuilder = new TableBuilder(tableName)
                           .AddColumn("a", KuduType.Int8, opt => opt.Key(true))
                           .AddColumn("b", KuduType.Int8, opt => opt.Key(true))
                           .AddColumn("c", KuduType.Int8, opt => opt.Key(true))
                           .AddHashPartitions(3, "a")
                           .AddHashPartitions(3, "b")
                           .AddHashPartitions(3, "c");

        var table = await _client.CreateTableAsync(tableBuilder);

        var partitions = await GetTablePartitionsAsync(table);

        var a = table.Schema.GetColumn("a");
        var b = table.Schema.GetColumn("b");
        var c = table.Schema.GetColumn("c");

        // a in [0, 1];
        await CheckPartitionsAsync(18, 2, table, partitions,
                                   KuduPredicate.NewInListPredicate(a, new byte[] { 0, 1 }));

        // a in [0, 1, 8];
        await CheckPartitionsAsync(27, 1, table, partitions,
                                   KuduPredicate.NewInListPredicate(a, new byte[] { 0, 1, 8 }));

        // b in [0, 1];
        await CheckPartitionsAsync(18, 6, table, partitions,
                                   KuduPredicate.NewInListPredicate(b, new byte[] { 0, 1 }));

        // c in [0, 1];
        await CheckPartitionsAsync(18, 18, table, partitions,
                                   KuduPredicate.NewInListPredicate(c, new byte[] { 0, 1 }));

        // b in [0, 1], c in [0, 1];
        await CheckPartitionsAsync(12, 12, table, partitions,
                                   KuduPredicate.NewInListPredicate(b, new byte[] { 0, 1 }),
                                   KuduPredicate.NewInListPredicate(c, new byte[] { 0, 1 }));

        // a in [0, 1], b in [0, 1], c in [0, 1];
        await CheckPartitionsAsync(8, 8, table, partitions,
                                   KuduPredicate.NewInListPredicate(a, new byte[] { 0, 1 }),
                                   KuduPredicate.NewInListPredicate(b, new byte[] { 0, 1 }),
                                   KuduPredicate.NewInListPredicate(c, new byte[] { 0, 1 }));
    }
コード例 #5
0
    private static SortedSet <DateTime> CreateDateValues()
    {
        var values = new SortedSet <DateTime>();

        for (long i = -50; i < 50; i++)
        {
            values.Add(EpochTime.UnixEpoch.AddDays(i));
        }
        values.Add(EpochTime.FromUnixTimeDays((int)KuduPredicate.MinIntValue(KuduType.Date)));
        values.Add(EpochTime.FromUnixTimeDays((int)KuduPredicate.MinIntValue(KuduType.Date) + 1));
        values.Add(EpochTime.FromUnixTimeDays((int)KuduPredicate.MaxIntValue(KuduType.Date) - 1));
        values.Add(EpochTime.FromUnixTimeDays((int)KuduPredicate.MaxIntValue(KuduType.Date)));
        return(values);
    }
コード例 #6
0
    private static SortedSet <long> CreateIntegerValues(KuduType type)
    {
        var values = new SortedSet <long>();

        for (long i = -50; i < 50; i++)
        {
            values.Add(i);
        }
        values.Add(KuduPredicate.MinIntValue(type));
        values.Add(KuduPredicate.MinIntValue(type) + 1);
        values.Add(KuduPredicate.MaxIntValue(type) - 1);
        values.Add(KuduPredicate.MaxIntValue(type));
        return(values);
    }
コード例 #7
0
 private static List <DateTime> CreateDateTestValues()
 {
     return(new List <DateTime>
     {
         EpochTime.FromUnixTimeDays((int)KuduPredicate.MinIntValue(KuduType.Date)),
         EpochTime.FromUnixTimeDays((int)KuduPredicate.MinIntValue(KuduType.Date) + 1),
         EpochTime.UnixEpoch.AddDays(-51),
         EpochTime.UnixEpoch.AddDays(-50),
         EpochTime.UnixEpoch,
         EpochTime.UnixEpoch.AddDays(49),
         EpochTime.UnixEpoch.AddDays(50),
         EpochTime.FromUnixTimeDays((int)KuduPredicate.MaxIntValue(KuduType.Date) - 1),
         EpochTime.FromUnixTimeDays((int)KuduPredicate.MaxIntValue(KuduType.Date))
     });
 }
コード例 #8
0
 private static List <long> CreateIntegerTestValues(KuduType type)
 {
     return(new List <long>
     {
         KuduPredicate.MinIntValue(type),
         KuduPredicate.MinIntValue(type) + 1,
         -51L,
         -50L,
         0L,
         49L,
         50L,
         KuduPredicate.MaxIntValue(type) - 1,
         KuduPredicate.MaxIntValue(type)
     });
 }
コード例 #9
0
    /// <summary>
    /// Adds a predicate to the scan.
    /// </summary>
    /// <param name="predicate">The predicate to add.</param>
    public TBuilder AddPredicate(KuduPredicate predicate)
    {
        var column     = predicate.Column;
        var columnName = column.Name;

        if (Predicates.TryGetValue(columnName, out var existing))
        {
            predicate = existing.Merge(predicate);
        }

        // KUDU-1652: Do not send an IS NOT NULL predicate to the server for a non-nullable column.
        if (!column.IsNullable && predicate.Type == PredicateType.IsNotNull)
        {
            return((TBuilder)this);
        }

        Predicates[columnName] = predicate;
        return((TBuilder)this);
    }
コード例 #10
0
    public async Task TestScanTokensWithExtraPredicate()
    {
        int numRows        = 100;
        int predicateIndex = 0;
        int predicateValue = 1;

        var builder = ClientTestUtil.GetBasicSchema()
                      .SetTableName(_tableName)
                      .CreateBasicRangePartition();

        var table = await _client.CreateTableAsync(builder);

        await ClientTestUtil.LoadDefaultTableAsync(_client, table, numRows);

        var tokens = await _client.NewScanTokenBuilder(table).BuildAsync();

        var columnSchema = table.Schema.GetColumn(predicateIndex);
        var predicate    = KuduPredicate.NewComparisonPredicate(
            columnSchema, ComparisonOp.Equal, predicateValue);

        var resultKeys = new List <int>();

        foreach (var token in tokens)
        {
            byte[] serialized  = token.Serialize();
            var    scanBuilder = await _client.NewScanBuilderFromTokenAsync(serialized);

            var scanner = scanBuilder
                          .AddPredicate(predicate)
                          .Build();

            await foreach (var resultSet in scanner)
            {
                foreach (var row in resultSet)
                {
                    resultKeys.Add(row.GetInt32(predicateIndex));
                }
            }
        }

        Assert.Collection(resultKeys, key => Assert.Equal(predicateValue, key));
    }
コード例 #11
0
    private async Task CheckPredicatesAsync <T>(
        KuduTable table,
        SortedSet <T> values,
        List <T> testValues)
    {
        var col = table.Schema.GetColumn("value");

        Assert.Equal(values.Count + 1, await CountRowsAsync(table));

        foreach (var v in testValues)
        {
            // value = v
            var equal = KuduPredicate.NewComparisonPredicate(col, ComparisonOp.Equal, (dynamic)v);
            Assert.Equal(values.Contains(v) ? 1 : 0, await CountRowsAsync(table, equal));

            // value >= v
            var greaterEqual = KuduPredicate.NewComparisonPredicate(col, ComparisonOp.GreaterEqual, (dynamic)v);
            Assert.Equal(values.TailSet(v).Count, await CountRowsAsync(table, greaterEqual));

            // value <= v
            var lessEqual = KuduPredicate.NewComparisonPredicate(col, ComparisonOp.LessEqual, (dynamic)v);
            Assert.Equal(values.HeadSet(v, true).Count, await CountRowsAsync(table, lessEqual));

            // value > v
            var greater = KuduPredicate.NewComparisonPredicate(col, ComparisonOp.Greater, (dynamic)v);
            Assert.Equal(values.TailSet(v, false).Count, await CountRowsAsync(table, greater));

            // value < v
            var less = KuduPredicate.NewComparisonPredicate(col, ComparisonOp.Less, (dynamic)v);
            Assert.Equal(values.HeadSet(v).Count, await CountRowsAsync(table, less));
        }

        var isNotNull = KuduPredicate.NewIsNotNullPredicate(col);

        Assert.Equal(values.Count, await CountRowsAsync(table, isNotNull));

        var isNull = KuduPredicate.NewIsNullPredicate(col);

        Assert.Equal(1, await CountRowsAsync(table, isNull));
    }
コード例 #12
0
    public async Task TestHashPartitionPruning()
    {
        // CREATE TABLE t
        // (a INT8, b INT8, c INT8)
        // PRIMARY KEY (a, b, c)
        // PARTITION BY HASH (a) PARTITIONS 2,
        //              HASH (b, c) PARTITIONS 2;

        var tableName = nameof(TestHashPartitionPruning);

        var tableBuilder = new TableBuilder(tableName)
                           .AddColumn("a", KuduType.Int8, opt => opt.Key(true))
                           .AddColumn("b", KuduType.Int8, opt => opt.Key(true))
                           .AddColumn("c", KuduType.Int8, opt => opt.Key(true))
                           .AddHashPartitions(2, "a")
                           .AddHashPartitions(2, "b", "c");

        var table = await _client.CreateTableAsync(tableBuilder);

        var partitions = await GetTablePartitionsAsync(table);

        var a = table.Schema.GetColumn("a");
        var b = table.Schema.GetColumn("b");
        var c = table.Schema.GetColumn("c");

        // No Predicates
        await CheckPartitionsAsync(4, 1, table, partitions);

        // a = 0;
        await CheckPartitionsAsync(2, 1, table, partitions,
                                   KuduPredicate.NewComparisonPredicate(a, ComparisonOp.Equal, 0));

        // a >= 0;
        await CheckPartitionsAsync(4, 1, table, partitions,
                                   KuduPredicate.NewComparisonPredicate(a, ComparisonOp.GreaterEqual, 0));

        // a >= 0;
        // a < 1;
        await CheckPartitionsAsync(2, 1, table, partitions,
                                   KuduPredicate.NewComparisonPredicate(a, ComparisonOp.GreaterEqual, 0),
                                   KuduPredicate.NewComparisonPredicate(a, ComparisonOp.Less, 1));

        // a >= 0;
        // a < 2;
        await CheckPartitionsAsync(4, 1, table, partitions,
                                   KuduPredicate.NewComparisonPredicate(a, ComparisonOp.GreaterEqual, 0),
                                   KuduPredicate.NewComparisonPredicate(a, ComparisonOp.Less, 2));

        // b = 1;
        await CheckPartitionsAsync(4, 1, table, partitions,
                                   KuduPredicate.NewComparisonPredicate(b, ComparisonOp.Equal, 1));

        // b = 1;
        // c = 2;
        await CheckPartitionsAsync(2, 2, table, partitions,
                                   KuduPredicate.NewComparisonPredicate(b, ComparisonOp.Equal, 1),
                                   KuduPredicate.NewComparisonPredicate(c, ComparisonOp.Equal, 2));

        // a = 0;
        // b = 1;
        // c = 2;
        await CheckPartitionsAsync(1, 1, table, partitions,
                                   KuduPredicate.NewComparisonPredicate(a, ComparisonOp.Equal, 0),
                                   KuduPredicate.NewComparisonPredicate(b, ComparisonOp.Equal, 1),
                                   KuduPredicate.NewComparisonPredicate(c, ComparisonOp.Equal, 2));

        // a IN (0, 10)
        await CheckPartitionsAsync(4, 1, table, partitions,
                                   KuduPredicate.NewInListPredicate(c, new byte[] { 0, 10 }));
    }
コード例 #13
0
    public async Task TestRangePartitionPruning()
    {
        // CREATE TABLE t
        // (a INT8, b STRING, c INT8)
        // PRIMARY KEY (a, b, c))
        // PARTITION BY RANGE (c, b)
        //    (PARTITION              VALUES < (0, "m"),
        //     PARTITION  (0, "m") <= VALUES < (10, "r")
        //     PARTITION (10, "r") <= VALUES);

        var tableName = nameof(TestRangePartitionPruning);

        var tableBuilder = new TableBuilder(tableName)
                           .AddColumn("a", KuduType.Int8, opt => opt.Key(true))
                           .AddColumn("b", KuduType.String, opt => opt.Key(true))
                           .AddColumn("c", KuduType.Int8, opt => opt.Key(true))
                           .SetRangePartitionColumns("c", "b")
                           .AddSplitRow(row =>
        {
            row.SetByte("c", 0);
            row.SetString("b", "m");
        })
                           .AddSplitRow(row =>
        {
            row.SetByte("c", 10);
            row.SetString("b", "r");
        });

        var table = await _client.CreateTableAsync(tableBuilder);

        var partitions = await GetTablePartitionsAsync(table);

        var b = table.Schema.GetColumn("b");
        var c = table.Schema.GetColumn("c");

        // No Predicates
        await CheckPartitionsAsync(3, 1, table, partitions);

        // c < -10
        await CheckPartitionsAsync(1, 1, table, partitions,
                                   KuduPredicate.NewComparisonPredicate(c, ComparisonOp.Less, -10));

        // c = -10
        await CheckPartitionsAsync(1, 1, table, partitions,
                                   KuduPredicate.NewComparisonPredicate(c, ComparisonOp.Equal, -10));

        // c < 10
        await CheckPartitionsAsync(2, 1, table, partitions,
                                   KuduPredicate.NewComparisonPredicate(c, ComparisonOp.Less, 10));

        // c < 100
        await CheckPartitionsAsync(3, 1, table, partitions,
                                   KuduPredicate.NewComparisonPredicate(c, ComparisonOp.Less, 100));

        // c < MIN
        await CheckPartitionsAsync(0, 0, table, partitions,
                                   KuduPredicate.NewComparisonPredicate(c, ComparisonOp.Less, sbyte.MinValue));

        // c < MAX
        await CheckPartitionsAsync(3, 1, table, partitions,
                                   KuduPredicate.NewComparisonPredicate(c, ComparisonOp.Less, sbyte.MaxValue));

        // c >= -10
        await CheckPartitionsAsync(3, 1, table, partitions,
                                   KuduPredicate.NewComparisonPredicate(c, ComparisonOp.GreaterEqual, -10));

        // c >= 0
        await CheckPartitionsAsync(3, 1, table, partitions,
                                   KuduPredicate.NewComparisonPredicate(c, ComparisonOp.GreaterEqual, -10));

        // c >= 5
        await CheckPartitionsAsync(2, 1, table, partitions,
                                   KuduPredicate.NewComparisonPredicate(c, ComparisonOp.GreaterEqual, 5));

        // c >= 10
        await CheckPartitionsAsync(2, 1, table, partitions,
                                   KuduPredicate.NewComparisonPredicate(c, ComparisonOp.GreaterEqual, 10));

        // c >= 100
        await CheckPartitionsAsync(1, 1, table, partitions,
                                   KuduPredicate.NewComparisonPredicate(c, ComparisonOp.GreaterEqual, 100));

        // c >= MIN
        await CheckPartitionsAsync(3, 1, table, partitions,
                                   KuduPredicate.NewComparisonPredicate(c, ComparisonOp.GreaterEqual, sbyte.MinValue));

        // c >= MAX
        await CheckPartitionsAsync(1, 1, table, partitions,
                                   KuduPredicate.NewComparisonPredicate(c, ComparisonOp.GreaterEqual, sbyte.MaxValue));

        // c >= -10
        // c < 0
        await CheckPartitionsAsync(1, 1, table, partitions,
                                   KuduPredicate.NewComparisonPredicate(c, ComparisonOp.GreaterEqual, -10),
                                   KuduPredicate.NewComparisonPredicate(c, ComparisonOp.Less, 0));

        // c >= 5
        // c < 100
        await CheckPartitionsAsync(2, 1, table, partitions,
                                   KuduPredicate.NewComparisonPredicate(c, ComparisonOp.GreaterEqual, 5),
                                   KuduPredicate.NewComparisonPredicate(c, ComparisonOp.Less, 100));

        // b = ""
        await CheckPartitionsAsync(3, 1, table, partitions,
                                   KuduPredicate.NewComparisonPredicate(b, ComparisonOp.Equal, ""));

        // b >= "z"
        await CheckPartitionsAsync(3, 1, table, partitions,
                                   KuduPredicate.NewComparisonPredicate(b, ComparisonOp.GreaterEqual, "z"));

        // b < "a"
        await CheckPartitionsAsync(3, 1, table, partitions,
                                   KuduPredicate.NewComparisonPredicate(b, ComparisonOp.Less, "a"));

        // b >= "m"
        // b < "z"
        await CheckPartitionsAsync(3, 1, table, partitions,
                                   KuduPredicate.NewComparisonPredicate(b, ComparisonOp.GreaterEqual, "m"),
                                   KuduPredicate.NewComparisonPredicate(b, ComparisonOp.Less, "z"));

        // c >= 10
        // b >= "r"
        await CheckPartitionsAsync(1, 1, table, partitions,
                                   KuduPredicate.NewComparisonPredicate(c, ComparisonOp.GreaterEqual, 10),
                                   KuduPredicate.NewComparisonPredicate(b, ComparisonOp.GreaterEqual, "r"));

        // c >= 10
        // b < "r"
        await CheckPartitionsAsync(2, 1, table, partitions,
                                   KuduPredicate.NewComparisonPredicate(c, ComparisonOp.GreaterEqual, 10),
                                   KuduPredicate.NewComparisonPredicate(b, ComparisonOp.Less, "r"));

        // c = 10
        // b < "r"
        await CheckPartitionsAsync(1, 1, table, partitions,
                                   KuduPredicate.NewComparisonPredicate(c, ComparisonOp.Equal, 10),
                                   KuduPredicate.NewComparisonPredicate(b, ComparisonOp.Less, "r"));

        // c < 0
        // b < "m"
        await CheckPartitionsAsync(1, 1, table, partitions,
                                   KuduPredicate.NewComparisonPredicate(c, ComparisonOp.Equal, 0),
                                   KuduPredicate.NewComparisonPredicate(b, ComparisonOp.Less, "m"));

        // c < 0
        // b < "z"
        await CheckPartitionsAsync(1, 1, table, partitions,
                                   KuduPredicate.NewComparisonPredicate(c, ComparisonOp.Less, 0),
                                   KuduPredicate.NewComparisonPredicate(b, ComparisonOp.Less, "z"));

        // c = 0
        // b = "m\0"
        await CheckPartitionsAsync(1, 1, table, partitions,
                                   KuduPredicate.NewComparisonPredicate(c, ComparisonOp.Equal, 0),
                                   KuduPredicate.NewComparisonPredicate(b, ComparisonOp.Equal, "m\0"));

        // c = 0
        // b < "m"
        await CheckPartitionsAsync(1, 1, table, partitions,
                                   KuduPredicate.NewComparisonPredicate(c, ComparisonOp.Equal, 0),
                                   KuduPredicate.NewComparisonPredicate(b, ComparisonOp.Less, "m"));

        // c = 0
        // b < "m\0"
        await CheckPartitionsAsync(2, 1, table, partitions,
                                   KuduPredicate.NewComparisonPredicate(c, ComparisonOp.Equal, 0),
                                   KuduPredicate.NewComparisonPredicate(b, ComparisonOp.Less, "m\0"));

        // c = 0
        // c = 2
        await CheckPartitionsAsync(0, 0, table, partitions,
                                   KuduPredicate.NewComparisonPredicate(c, ComparisonOp.Equal, 0),
                                   KuduPredicate.NewComparisonPredicate(c, ComparisonOp.Equal, 2));

        // c = MIN
        await CheckPartitionsAsync(1, 1, table, partitions,
                                   KuduPredicate.NewComparisonPredicate(c, ComparisonOp.Equal, sbyte.MinValue));

        // c = MAX
        await CheckPartitionsAsync(1, 1, table, partitions,
                                   KuduPredicate.NewComparisonPredicate(c, ComparisonOp.Equal, sbyte.MaxValue));

        // c IN (1, 2)
        await CheckPartitionsAsync(1, 1, table, partitions,
                                   KuduPredicate.NewInListPredicate(c, new byte[] { 1, 2 }));

        // c IN (0, 1, 2)
        await CheckPartitionsAsync(2, 1, table, partitions,
                                   KuduPredicate.NewInListPredicate(c, new byte[] { 0, 1, 2 }));

        // c IN (-10, 0)
        // b < "m"
        await CheckPartitionsAsync(1, 1, table, partitions,
                                   KuduPredicate.NewInListPredicate(c, new sbyte[] { -10, 0 }),
                                   KuduPredicate.NewComparisonPredicate(b, ComparisonOp.Less, "m"));

        // c IN (-10, 0)
        // b < "m\0"
        await CheckPartitionsAsync(2, 1, table, partitions,
                                   KuduPredicate.NewInListPredicate(c, new sbyte[] { -10, 0 }),
                                   KuduPredicate.NewComparisonPredicate(b, ComparisonOp.Less, "m\0"));
    }
コード例 #14
0
    public async Task TestBinaryPredicates()
    {
        var builder = GetDefaultTableBuilder()
                      .SetTableName("binary-table")
                      .AddColumn("value", KuduType.Binary);

        var table = await _client.CreateTableAsync(builder);

        var values     = CreateStringValues();
        var testValues = CreateStringTestValues();

        long i = 0;

        foreach (var value in values)
        {
            var insert = table.NewInsert();
            insert.SetInt64("key", i++);
            insert.SetBinary("value", value.ToUtf8ByteArray());
            await _session.EnqueueAsync(insert);
        }

        var nullInsert = table.NewInsert();

        nullInsert.SetInt64("key", i);
        nullInsert.SetNull("value");
        await _session.EnqueueAsync(nullInsert);

        await _session.FlushAsync();

        var col = table.Schema.GetColumn("value");

        Assert.Equal(values.Count + 1, await CountRowsAsync(table));

        foreach (var s in testValues)
        {
            var v = s.ToUtf8ByteArray();

            // value = v
            var equal = KuduPredicate.NewComparisonPredicate(col, ComparisonOp.Equal, v);
            Assert.Equal(values.GetViewBetween(s, s).Count, await CountRowsAsync(table, equal));

            // value >= v
            var greaterEqual = KuduPredicate.NewComparisonPredicate(col, ComparisonOp.GreaterEqual, v);
            Assert.Equal(values.TailSet(s).Count, await CountRowsAsync(table, greaterEqual));

            // value <= v
            var lessEqual = KuduPredicate.NewComparisonPredicate(col, ComparisonOp.LessEqual, v);
            Assert.Equal(values.HeadSet(s, true).Count, await CountRowsAsync(table, lessEqual));

            // value > v
            var greater = KuduPredicate.NewComparisonPredicate(col, ComparisonOp.Greater, v);
            Assert.Equal(values.TailSet(s, false).Count, await CountRowsAsync(table, greater));

            // value < v
            var less = KuduPredicate.NewComparisonPredicate(col, ComparisonOp.Less, v);
            Assert.Equal(values.HeadSet(s).Count, await CountRowsAsync(table, less));
        }

        var isNotNull = KuduPredicate.NewIsNotNullPredicate(col);

        Assert.Equal(values.Count, await CountRowsAsync(table, isNotNull));

        var isNull = KuduPredicate.NewIsNullPredicate(col);

        Assert.Equal(1, await CountRowsAsync(table, isNull));
    }
コード例 #15
0
    public async Task TestMultiColumnInListHashPruning()
    {
        // CREATE TABLE t
        // (a INT8, b INT8, c INT8)
        // PRIMARY KEY (a, b, c)
        // PARTITION BY HASH (a) PARTITIONS 3,
        //              HASH (b, c) PARTITIONS 3;

        var tableName = nameof(TestMultiColumnInListHashPruning);

        var tableBuilder = new TableBuilder(tableName)
                           .AddColumn("a", KuduType.Int8, opt => opt.Key(true))
                           .AddColumn("b", KuduType.Int8, opt => opt.Key(true))
                           .AddColumn("c", KuduType.Int8, opt => opt.Key(true))
                           .AddHashPartitions(3, "a")
                           .AddHashPartitions(3, "b", "c");

        var table = await _client.CreateTableAsync(tableBuilder);

        var partitions = await GetTablePartitionsAsync(table);

        var a = table.Schema.GetColumn("a");
        var b = table.Schema.GetColumn("b");
        var c = table.Schema.GetColumn("c");

        // a in [0, 1];
        await CheckPartitionsAsync(6, 2, table, partitions,
                                   KuduPredicate.NewInListPredicate(a, new byte[] { 0, 1 }));

        // a in [0, 1, 8];
        await CheckPartitionsAsync(9, 1, table, partitions,
                                   KuduPredicate.NewInListPredicate(a, new byte[] { 0, 1, 8 }));

        // b in [0, 1];
        await CheckPartitionsAsync(9, 1, table, partitions,
                                   KuduPredicate.NewInListPredicate(b, new byte[] { 0, 1 }));

        // c in [0, 1];
        await CheckPartitionsAsync(9, 1, table, partitions,
                                   KuduPredicate.NewInListPredicate(c, new byte[] { 0, 1 }));

        // b in [0, 1], c in [0, 1]
        // (0, 0) in bucket 2
        // (0, 1) in bucket 2
        // (1, 0) in bucket 1
        // (1, 1) in bucket 0
        await CheckPartitionsAsync(9, 1, table, partitions,
                                   KuduPredicate.NewInListPredicate(b, new byte[] { 0, 1 }),
                                   KuduPredicate.NewInListPredicate(c, new byte[] { 0, 1 }));

        // b = 0, c in [0, 1]
        await CheckPartitionsAsync(3, 3, table, partitions,
                                   KuduPredicate.NewComparisonPredicate(b, ComparisonOp.Equal, 0),
                                   KuduPredicate.NewInListPredicate(c, new byte[] { 0, 1 }));

        // b = 1, c in [0, 1]
        await CheckPartitionsAsync(6, 6, table, partitions,
                                   KuduPredicate.NewComparisonPredicate(b, ComparisonOp.Equal, 1),
                                   KuduPredicate.NewInListPredicate(c, new byte[] { 0, 1 }));

        // a in [0, 1], b in [0, 1], c in [0, 1];
        await CheckPartitionsAsync(6, 2, table, partitions,
                                   KuduPredicate.NewInListPredicate(a, new byte[] { 0, 1 }),
                                   KuduPredicate.NewInListPredicate(b, new byte[] { 0, 1 }),
                                   KuduPredicate.NewInListPredicate(c, new byte[] { 0, 1 }));
    }
コード例 #16
0
    public async Task TestPruning()
    {
        // CREATE TABLE timeseries
        // (host STRING, metric STRING, timestamp UNIXTIME_MICROS, value DOUBLE)
        // PRIMARY KEY (host, metric, time)
        // DISTRIBUTE BY
        //    RANGE(time)
        //        (PARTITION VALUES < 10,
        //         PARTITION VALUES >= 10);
        //    HASH (host, metric) 2 PARTITIONS;

        var tableName = nameof(TestPruning);

        var tableBuilder = new TableBuilder(tableName)
                           .AddColumn("host", KuduType.String, opt => opt.Key(true))
                           .AddColumn("metric", KuduType.String, opt => opt.Key(true))
                           .AddColumn("timestamp", KuduType.UnixtimeMicros, opt => opt.Key(true))
                           .AddColumn("value", KuduType.Double)
                           .SetRangePartitionColumns("timestamp")
                           .AddSplitRow(row => row.SetInt64("timestamp", 10))
                           .AddHashPartitions(2, "host", "metric");

        var table = await _client.CreateTableAsync(tableBuilder);

        var partitions = await GetTablePartitionsAsync(table);

        var host      = table.Schema.GetColumn("host");
        var metric    = table.Schema.GetColumn("metric");
        var timestamp = table.Schema.GetColumn("timestamp");

        // No Predicates
        await CheckPartitionsAsync(4, 1, table, partitions);

        // host = "a"
        await CheckPartitionsAsync(4, 1, table, partitions,
                                   KuduPredicate.NewComparisonPredicate(host, ComparisonOp.Equal, "a"));

        // host = "a"
        // metric = "a"
        await CheckPartitionsAsync(2, 1, table, partitions,
                                   KuduPredicate.NewComparisonPredicate(host, ComparisonOp.Equal, "a"),
                                   KuduPredicate.NewComparisonPredicate(metric, ComparisonOp.Equal, "a"));

        // host = "a"
        // metric = "a"
        // timestamp >= 9;
        await CheckPartitionsAsync(2, 1, table, partitions,
                                   KuduPredicate.NewComparisonPredicate(host, ComparisonOp.Equal, "a"),
                                   KuduPredicate.NewComparisonPredicate(metric, ComparisonOp.Equal, "a"),
                                   KuduPredicate.NewComparisonPredicate(timestamp, ComparisonOp.GreaterEqual, 9));

        // host = "a"
        // metric = "a"
        // timestamp >= 10;
        // timestamp < 20;
        await CheckPartitionsAsync(1, 1, table, partitions,
                                   KuduPredicate.NewComparisonPredicate(host, ComparisonOp.Equal, "a"),
                                   KuduPredicate.NewComparisonPredicate(metric, ComparisonOp.Equal, "a"),
                                   KuduPredicate.NewComparisonPredicate(timestamp, ComparisonOp.GreaterEqual, 10),
                                   KuduPredicate.NewComparisonPredicate(timestamp, ComparisonOp.Less, 20));

        // host = "a"
        // metric = "a"
        // timestamp < 10;
        await CheckPartitionsAsync(1, 1, table, partitions,
                                   KuduPredicate.NewComparisonPredicate(host, ComparisonOp.Equal, "a"),
                                   KuduPredicate.NewComparisonPredicate(metric, ComparisonOp.Equal, "a"),
                                   KuduPredicate.NewComparisonPredicate(timestamp, ComparisonOp.Less, 10));

        // host = "a"
        // metric = "a"
        // timestamp >= 10;
        await CheckPartitionsAsync(1, 1, table, partitions,
                                   KuduPredicate.NewComparisonPredicate(host, ComparisonOp.Equal, "a"),
                                   KuduPredicate.NewComparisonPredicate(metric, ComparisonOp.Equal, "a"),
                                   KuduPredicate.NewComparisonPredicate(timestamp, ComparisonOp.GreaterEqual, 10));

        // host = "a"
        // metric = "a"
        // timestamp = 10;
        await CheckPartitionsAsync(1, 1, table, partitions,
                                   KuduPredicate.NewComparisonPredicate(host, ComparisonOp.Equal, "a"),
                                   KuduPredicate.NewComparisonPredicate(metric, ComparisonOp.Equal, "a"),
                                   KuduPredicate.NewComparisonPredicate(timestamp, ComparisonOp.Equal, 10));

        byte[] hash1 = new byte[] { 0, 0, 0, 1 };

        // partition key < (hash=1)
        await CheckPartitionsAsync(2, 1, table, partitions, null, hash1);

        // partition key >= (hash=1)
        await CheckPartitionsAsync(2, 1, table, partitions, hash1, null);

        // timestamp = 10
        // partition key < (hash=1)
        await CheckPartitionsAsync(1, 1, table, partitions, null, hash1,
                                   KuduPredicate.NewComparisonPredicate(timestamp, ComparisonOp.Equal, 10));

        // timestamp = 10
        // partition key >= (hash=1)
        await CheckPartitionsAsync(1, 1, table, partitions, hash1, null,
                                   KuduPredicate.NewComparisonPredicate(timestamp, ComparisonOp.Equal, 10));

        // timestamp IN (0, 9)
        // host = "a"
        // metric IN ("foo", "baz")
        await CheckPartitionsAsync(1, 1, table, partitions,
                                   KuduPredicate.NewInListPredicate(timestamp, new long[] { 0, 9 }),
                                   KuduPredicate.NewComparisonPredicate(host, ComparisonOp.Equal, "a"),
                                   KuduPredicate.NewInListPredicate(metric, new string[] { "foo", "baz" }));

        // timestamp IN (10, 100)
        await CheckPartitionsAsync(2, 2, table, partitions,
                                   KuduPredicate.NewInListPredicate(timestamp, new long[] { 10, 100 }));

        // timestamp IN (9, 10)
        await CheckPartitionsAsync(4, 2, table, partitions,
                                   KuduPredicate.NewInListPredicate(timestamp, new long[] { 9, 10 }));

        // timestamp IS NOT NULL
        await CheckPartitionsAsync(4, 1, table, partitions,
                                   KuduPredicate.NewIsNotNullPredicate(timestamp));

        // timestamp IS NULL
        await CheckPartitionsAsync(0, 0, table, partitions,
                                   KuduPredicate.NewIsNullPredicate(timestamp));
    }
コード例 #17
0
    internal static void PbIntoScanner <TBuilder>(
        TBuilder builder, ScanTokenPB scanTokenPb)
        where TBuilder : AbstractKuduScannerBuilder <TBuilder>
    {
        var table  = builder.Table;
        var schema = table.Schema;

        if (scanTokenPb.FeatureFlags.Contains(ScanTokenPB.Types.Feature.Unknown))
        {
            throw new Exception("Scan token requires an unsupported feature. " +
                                "This Kudu client must be updated.");
        }

        builder.SetProjectedColumns(
            ComputeProjectedColumnIndexes(scanTokenPb, schema));

        foreach (var predicate in scanTokenPb.ColumnPredicates)
        {
            builder.AddPredicate(KuduPredicate.FromPb(schema, predicate));
        }

        if (scanTokenPb.HasLowerBoundPrimaryKey)
        {
            builder.LowerBoundRaw(scanTokenPb.LowerBoundPrimaryKey.ToByteArray());
        }

        if (scanTokenPb.HasUpperBoundPrimaryKey)
        {
            builder.ExclusiveUpperBoundRaw(scanTokenPb.UpperBoundPrimaryKey.ToByteArray());
        }

        if (scanTokenPb.HasLowerBoundPartitionKey)
        {
            builder.LowerBoundPartitionKeyRaw(scanTokenPb.LowerBoundPartitionKey.ToByteArray());
        }

        if (scanTokenPb.HasUpperBoundPartitionKey)
        {
            builder.ExclusiveUpperBoundPartitionKeyRaw(scanTokenPb.UpperBoundPartitionKey.ToByteArray());
        }

        if (scanTokenPb.HasLimit)
        {
            builder.SetLimit((long)scanTokenPb.Limit);
        }

        if (scanTokenPb.HasReadMode)
        {
            switch (scanTokenPb.ReadMode)
            {
            case Protobuf.ReadMode.ReadAtSnapshot:
            {
                builder.SetReadMode(ReadMode.ReadAtSnapshot);

                if (scanTokenPb.HasSnapTimestamp)
                {
                    builder.SnapshotTimestampRaw((long)scanTokenPb.SnapTimestamp);
                }

                // Set the diff scan timestamps if they are set.
                if (scanTokenPb.HasSnapStartTimestamp)
                {
                    builder.DiffScan(
                        (long)scanTokenPb.SnapStartTimestamp,
                        (long)scanTokenPb.SnapTimestamp);
                }

                break;
            }

            case Protobuf.ReadMode.ReadLatest:
            {
                builder.SetReadMode(ReadMode.ReadLatest);
                break;
            }

            case Protobuf.ReadMode.ReadYourWrites:
            {
                builder.SetReadMode(ReadMode.ReadYourWrites);
                break;
            }

            default:
                throw new Exception("Unknown read mode");
            }
        }

        if (scanTokenPb.HasReplicaSelection)
        {
            switch (scanTokenPb.ReplicaSelection)
            {
            case Protobuf.ReplicaSelection.LeaderOnly:
                builder.SetReplicaSelection(ReplicaSelection.LeaderOnly);
                break;

            case Protobuf.ReplicaSelection.ClosestReplica:
                builder.SetReplicaSelection(ReplicaSelection.ClosestReplica);
                break;

            default:
                throw new Exception("Unknown replica selection policy");
            }
        }

        if (scanTokenPb.HasPropagatedTimestamp &&
            (long)scanTokenPb.PropagatedTimestamp != KuduClient.NoTimestamp)
        {
            builder.Client.LastPropagatedTimestamp = (long)scanTokenPb.PropagatedTimestamp;
        }

        if (scanTokenPb.HasCacheBlocks)
        {
            builder.SetCacheBlocks(scanTokenPb.CacheBlocks);
        }

        if (scanTokenPb.HasFaultTolerant)
        {
            builder.SetFaultTolerant(scanTokenPb.FaultTolerant);
        }

        if (scanTokenPb.HasBatchSizeBytes)
        {
            builder.SetBatchSizeBytes((int)scanTokenPb.BatchSizeBytes);
        }

        if (scanTokenPb.HasScanRequestTimeoutMs)
        {
            // TODO
        }

        if (scanTokenPb.HasKeepAlivePeriodMs)
        {
            // TODO
        }
    }