private static async Task <long> CountScanTokenRowsAsync(
        KuduClient client, List <KuduScanToken> tokens)
    {
        var tasks = new List <Task <long> >();

        foreach (var token in tokens)
        {
            var task = Task.Run(async() =>
            {
                var tokenBytes = token.Serialize();

                var scanBuilder = await client.NewScanBuilderFromTokenAsync(tokenBytes);
                var scanner     = scanBuilder.Build();

                return(await scanner.CountAsync());
            });

            tasks.Add(task);
        }

        var results = await Task.WhenAll(tasks);

        var rowCount = results.Sum();

        return(rowCount);
    }
    /// <summary>
    /// Load a table of default schema with the specified number of records, in ascending key order.
    /// </summary>
    public static async Task LoadDefaultTableAsync(KuduClient client, KuduTable table, int numRows)
    {
        var rows = Enumerable.Range(0, numRows)
                   .Select(i => CreateBasicSchemaInsert(table, i));

        await client.WriteAsync(rows);
    }
Beispiel #3
0
    public KuduSession(
        KuduClient client,
        KuduSessionOptions options,
        ILoggerFactory loggerFactory,
        long txnId)
    {
        _client  = client;
        _options = options;
        _txnId   = txnId;
        _logger  = loggerFactory.CreateLogger <KuduSession>();

        var channelOptions = new BoundedChannelOptions(options.Capacity)
        {
            SingleWriter = false,
            SingleReader = true,
            FullMode     = BoundedChannelFullMode.Wait,
            AllowSynchronousContinuations = false
        };

        var channel = Channel.CreateBounded <KuduOperation>(channelOptions);

        _writer = channel.Writer;
        _reader = channel.Reader;

        _singleFlush = new SemaphoreSlim(1, 1);
        _flushCts    = new CancellationTokenSource();
        _flushTcs    = new TaskCompletionSource(
            TaskCreationOptions.RunContinuationsAsynchronously);

        _consumeTask = ConsumeAsync();
    }
    private static async Task CommitAndVerifyTransactionAsync(
        KuduClient client, KuduTransaction transaction)
    {
        var tsBeforeCommit = client.LastPropagatedTimestamp;
        await transaction.CommitAsync();

        Assert.Equal(tsBeforeCommit, client.LastPropagatedTimestamp);

        await transaction.WaitForCommitAsync();

        var tsAfterCommit = client.LastPropagatedTimestamp;

        Assert.True(tsAfterCommit > tsBeforeCommit);

        // A sanity check: calling WaitForCommitAsync() again after the commit phase
        // has been finalized doesn't change last propagated timestamp at the
        // client side.
        for (int i = 0; i < 10; ++i)
        {
            await transaction.WaitForCommitAsync();

            Assert.Equal(tsAfterCommit, client.LastPropagatedTimestamp);
            await Task.Delay(10);
        }
    }
    public async Task InitializeAsync()
    {
        _harness = await new MiniKuduClusterBuilder().BuildHarnessAsync();

        var options           = _harness.CreateClientBuilder().BuildOptions();
        var securityContext   = new SecurityContext();
        var systemClock       = new SystemClock();
        var connectionFactory = new KuduConnectionFactory(
            options, securityContext, NullLoggerFactory.Instance);

        _testConnectionFactory = new TestConnectionFactory(connectionFactory);

        _client = new KuduClient(
            options,
            securityContext,
            _testConnectionFactory,
            systemClock,
            NullLoggerFactory.Instance);

        var builder = ClientTestUtil.GetBasicSchema()
                      .SetTableName("chaos_test_table")
                      .SetNumReplicas(3)
                      .CreateBasicRangePartition();

        _table = await _client.CreateTableAsync(builder);
    }
Beispiel #6
0
    /// <summary>
    /// Applies a list of operations and returns the final change type for each key.
    /// </summary>
    private async Task <Dictionary <int, RowOperation> > ApplyOperationsAsync(
        KuduClient client, List <KuduOperation> operations)
    {
        var results = new Dictionary <int, RowOperation>();

        // If there are no operations, return early.
        if (operations.Count == 0)
        {
            return(results);
        }

        // On some runs, wait long enough to flush at the start.
        if (_random.NextBool())
        {
            await Task.Delay(2000);
        }

        // Pick an int as a flush indicator so we flush once on average while applying operations.
        int flushInt = _random.Next(operations.Count);

        foreach (var operation in operations)
        {
            // On some runs, wait long enough to flush while applying operations.
            if (_random.Next(operations.Count) == flushInt)
            {
                await Task.Delay(2000);
            }

            await client.WriteAsync(new[] { operation });

            results[operation.GetInt32(0)] = operation.Operation;
        }

        return(results);
    }
    public static async Task <List <string> > ScanTableToStringsAsync(
        KuduClient client, KuduTable table, params KuduPredicate[] predicates)
    {
        var rowStrings  = new List <string>();
        var scanBuilder = client.NewScanBuilder(table)
                          .SetReadMode(ReadMode.ReadYourWrites)
                          .SetReplicaSelection(ReplicaSelection.LeaderOnly);

        foreach (var predicate in predicates)
        {
            scanBuilder.AddPredicate(predicate);
        }

        var scanner = scanBuilder.Build();

        await foreach (var resultSet in scanner)
        {
            foreach (var row in resultSet)
            {
                rowStrings.Add(row.ToString());
            }
        }

        rowStrings.Sort();

        return(rowStrings);
    }
 public AbstractKuduScannerBuilder(KuduClient client, KuduTable table)
 {
     Client             = client;
     Table              = table;
     Predicates         = new Dictionary <string, KuduPredicate>();
     ScanRequestTimeout = -1; // TODO: Pull this from the client.
 }
Beispiel #9
0
    public KuduScanEnumerator(
        ILogger logger,
        KuduClient client,
        KuduTable table,
        List <ColumnSchemaPB> projectedColumnsPb,
        KuduSchema projectionSchema,
        OrderMode orderMode,
        ReadMode readMode,
        ReplicaSelection replicaSelection,
        bool isFaultTolerant,
        Dictionary <string, KuduPredicate> predicates,
        long limit,
        bool cacheBlocks,
        byte[] startPrimaryKey,
        byte[] endPrimaryKey,
        long startTimestamp,
        long htTimestamp,
        int batchSizeBytes,
        PartitionPruner partitionPruner,
        CancellationToken cancellationToken)
    {
        _logger            = logger;
        _client            = client;
        _table             = table;
        _partitionPruner   = partitionPruner;
        _orderMode         = orderMode;
        _readMode          = readMode;
        _columns           = projectedColumnsPb;
        _schema            = projectionSchema;
        _predicates        = predicates;
        _replicaSelection  = replicaSelection;
        _isFaultTolerant   = isFaultTolerant;
        _limit             = limit;
        _cacheBlocks       = cacheBlocks;
        _startPrimaryKey   = startPrimaryKey ?? Array.Empty <byte>();
        _endPrimaryKey     = endPrimaryKey ?? Array.Empty <byte>();
        _startTimestamp    = startTimestamp;
        SnapshotTimestamp  = htTimestamp;
        _batchSizeBytes    = batchSizeBytes;
        _scannerId         = ByteString.Empty;
        _lastPrimaryKey    = ByteString.Empty;
        _cancellationToken = cancellationToken;
        ResourceMetrics    = new ResourceMetrics();

        // If the partition pruner has pruned all partitions, then the scan can be
        // short circuited without contacting any tablet servers.
        if (!_partitionPruner.HasMorePartitionKeyRanges)
        {
            _closed = true;
        }

        // For READ_YOUR_WRITES scan mode, get the latest observed timestamp
        // and store it. Always use this one as the propagated timestamp for
        // the duration of the scan to avoid unnecessary wait.
        if (readMode == ReadMode.ReadYourWrites)
        {
            _lowerBoundPropagationTimestamp = client.LastPropagatedTimestamp;
        }
    }
Beispiel #10
0
 public KuduTestHarness(
     MiniKuduCluster miniCluster,
     bool disposeMiniCluster = false)
 {
     _miniCluster        = miniCluster;
     _disposeMiniCluster = disposeMiniCluster;
     _client             = miniCluster.CreateClient();
 }
Beispiel #11
0
    public static ValueTask <long> CountRowsAsync(KuduClient client, KuduTable table)
    {
        var scanner = client.NewScanBuilder(table)
                      .SetReadMode(ReadMode.ReadYourWrites)
                      .SetReplicaSelection(ReplicaSelection.LeaderOnly)
                      .Build();

        return(scanner.CountAsync());
    }
Beispiel #12
0
    public async Task InitializeAsync()
    {
        _harness = await new MiniKuduClusterBuilder().BuildHarnessAsync();
        _client  = _harness.CreateClient();
        await using var session = _client.NewSession();

        // Create a 4-tablets table for scanning.
        var builder = new TableBuilder(_tableName)
                      .AddColumn("key1", KuduType.String, opt => opt.Key(true))
                      .AddColumn("key2", KuduType.String, opt => opt.Key(true))
                      .AddColumn("val", KuduType.String)
                      .SetRangePartitionColumns("key1", "key2");

        for (int i = 1; i < 4; i++)
        {
            builder.AddSplitRow(splitRow =>
            {
                splitRow.SetString("key1", i.ToString());
                splitRow.SetString("key2", "");
            });
        }

        var table = await _client.CreateTableAsync(builder);

        // The data layout ends up like this:
        // tablet '', '1': no rows
        // tablet '1', '2': '111', '122', '133'
        // tablet '2', '3': '211', '222', '233'
        // tablet '3', '': '311', '322', '333'
        var keys = new[] { "1", "2", "3" };

        foreach (var key1 in keys)
        {
            foreach (var key2 in keys)
            {
                var insert = table.NewInsert();
                insert.SetString(0, key1);
                insert.SetString(1, key2);
                insert.SetString(2, key2);
                await session.EnqueueAsync(insert);

                await session.FlushAsync();
            }
        }

        _beforeWriteTimestamp = _client.LastPropagatedTimestamp;

        // Reset the client in order to clear the propagated timestamp.
        _newClient = _harness.CreateClient();

        // Reopen the table using the new client.
        _table = await _newClient.OpenTableAsync(_tableName);

        _schema = _table.Schema;
    }
Beispiel #13
0
    public async Task InitializeAsync()
    {
        _harness = await new MiniKuduClusterBuilder().BuildHarnessAsync();
        _client  = _harness.CreateClient();

        // Use one tablet because multiple tablets don't work: we could jump
        // from one tablet to another which could change the logical clock.
        var builder = new TableBuilder("HybridTimeTest")
                      .AddColumn("key", KuduType.String, opt => opt.Key(true))
                      .SetRangePartitionColumns("key");

        _table = await _client.CreateTableAsync(builder);
    }
    public async Task InitializeAsync()
    {
        _harness = await new MiniKuduClusterBuilder().BuildHarnessAsync();
        _client  = _harness.CreateClient();

        var builder = ClientTestUtil.GetBasicSchema()
                      .SetTableName("ScannerFaultToleranceTests")
                      .AddHashPartitions(_numTablets, "key");

        _table = await _client.CreateTableAsync(builder);

        var random = new Random();

        _keys = new HashSet <int>();
        while (_keys.Count < _numRows)
        {
            _keys.Add(random.Next());
        }

        var rows = _keys
                   .Select((key, i) =>
        {
            var insert = _table.NewInsert();
            insert.SetInt32(0, key);
            insert.SetInt32(1, i);
            insert.SetInt32(2, i++);
            insert.SetString(3, DataGenerator.RandomString(1024, random));
            insert.SetBool(4, true);

            return(insert);
        })
                   .Chunk(1000);

        foreach (var batch in rows)
        {
            await _client.WriteAsync(batch);
        }
    }
    private static KuduTransaction MakeFakeTransaction(
        KuduClient client, KuduTransaction transaction)
    {
        var buf = transaction.Serialize();
        var pb  = TxnTokenPB.Parser.ParseFrom(buf);

        Assert.True(pb.HasTxnId);
        var txnId = pb.TxnId;

        Assert.True(txnId > KuduClient.InvalidTxnId);

        var fakeTxnId = txnId + 123;
        var message   = new TxnTokenPB
        {
            TxnId           = fakeTxnId,
            EnableKeepalive = false,
            KeepaliveMillis = 0
        };

        var fakeTxnBuf = ProtobufHelper.ToByteArray(message);

        return(client.NewTransactionFromToken(fakeTxnBuf));
    }
Beispiel #16
0
    static async Task Main()
    {
        var masterAddresses = "localhost:7051,localhost:7151,localhost:7251";
        var tableName       = $"test_table_{Guid.NewGuid():N}";
        var numRows         = 10000;

        var loggerFactory = LoggerFactory.Create(builder => builder
                                                 .SetMinimumLevel(LogLevel.Trace)
                                                 .AddConsole());

        await using var client = KuduClient.NewBuilder(masterAddresses)
                                 .SetLoggerFactory(loggerFactory)
                                 .Build();

        var tableBuilder = new TableBuilder(tableName)
                           .AddColumn("host", KuduType.String, opt => opt.Key(true))
                           .AddColumn("metric", KuduType.String, opt => opt.Key(true))
                           .AddColumn("timestamp", KuduType.UnixtimeMicros, opt => opt.Key(true))
                           .AddColumn("value", KuduType.Double)
                           .SetNumReplicas(1)
                           .SetRangePartitionColumns("host", "metric", "timestamp");

        var table = await client.CreateTableAsync(tableBuilder);

        Console.WriteLine($"Created table {tableName}");

        var batches     = CreateRows(table, numRows).Chunk(2000);
        var writtenRows = 0;

        foreach (var batch in batches)
        {
            await client.WriteAsync(batch);

            writtenRows += batch.Length;
            Console.WriteLine($"Wrote {writtenRows} rows");
        }
    }
Beispiel #17
0
 public async Task InitializeAsync()
 {
     _harness = await new MiniKuduClusterBuilder().BuildHarnessAsync();
     _client  = _harness.CreateClient();
 }
Beispiel #18
0
        // LINQPad script entry point
        // when deployed as an Azure Function this method is not used
        public static async Task <int> MainAsync(object userQuery, string[] args)
        {
            var hasConsoleInput = false;

            if ("LPRun.exe".Equals(Process.GetCurrentProcess().MainModule.ModuleName, StringComparison.OrdinalIgnoreCase))
            {
                hasConsoleInput = Environment.UserInteractive;

                // pipe trace to console
                Trace.Listeners.Add(new ConsoleTraceListener());
            }

            if (userQuery == null)
            {
                throw new ArgumentNullException("User query cannot be null. You should pass 'this' here.", nameof(userQuery));
            }
            var userQueryTypeInfo = new UserQueryTypeInfo(userQuery);

            var currentQuery = Util.CurrentQuery;

            if (currentQuery == null)
            {
                throw new InvalidOperationException("This script must be run from wthin a LINQPad context (either via LINQPad or LPRun).");
            }
            var currentQueryInfo = new QueryInfo(currentQuery);

            var currentQueryPath = Util.CurrentQueryPath;

            if (currentQueryPath == null)
            {
                throw new InvalidOperationException("A file name is required (save your LINQPad query to disk). Without it, we cannot establish a context for your functions.");
            }
            var currentQueryPathInfo = new QueryPathInfo(currentQueryPath);

            // ========

            args = args ?? new string[0]; // note: `args` can be null
            if (args.Length == 0)
            {
                if (FirstRun.ShouldPrompt())
                {
                    FirstRun.Prompt();
                }

                // ================================

                var workingDirectory = Path.Combine(Env.GetLocalAppDataDirectory(), currentQueryPathInfo.InstanceId);
                Debug.WriteLine($"workingDirectory: {workingDirectory}");
                FunctionApp.Deploy(workingDirectory);

                // ================================

                Compiler.Compile(new UserQueryTypeInfo(userQuery), currentQueryInfo, new CompilationOptions(currentQueryPath)
                {
                    OutDir = workingDirectory,
                }, currentQueryInfo);

                // ================================

                StorageEmulator.StartOrInstall();

                // todo: if AzureWebJobsStorage or AzureWebJobsDashboard is set elsewhere, like app.config we shouldn't override them like this

                if (string.IsNullOrEmpty(Environment.GetEnvironmentVariable("AzureWebJobsStorage")))
                {
                    Environment.SetEnvironmentVariable("AzureWebJobsStorage", "UseDevelopmentStorage=true");
                }

                if (string.IsNullOrEmpty(Environment.GetEnvironmentVariable("AzureWebJobsDashboard")))
                {
                    Environment.SetEnvironmentVariable("AzureWebJobsDashboard", "UseDevelopmentStorage=true");
                }

                // ================================

                var lastWriteTime = File.GetLastWriteTimeUtc(currentQueryPathInfo.QueryPath);

                using (var fs = new FileSystemWatcher(currentQueryPathInfo.QueryDirectoryName, currentQueryPathInfo.QueryFileName))
                {
                    var stream = new BlockingCollection <int>();

                    ThreadPool.QueueUserWorkItem(_ =>
                    {
                        foreach (var x in stream.GetConsumingEnumerable())
                        {
                            Thread.Sleep(250); // hold, for just a moment...

                            var lastWriteTime2 = File.GetLastWriteTimeUtc(currentQueryPathInfo.QueryPath);
                            if (lastWriteTime < lastWriteTime2)
                            {
                                Debug.WriteLine("Recompiling...");

                                Util.RunAndWait(currentQueryPathInfo.QueryPath, new[] { "-compile", "-out-dir", workingDirectory });

                                Debug.WriteLine("Recompiled!");

                                lastWriteTime = lastWriteTime2;
                            }
                        }
                    });

                    fs.Changed += (sender, e) => stream.Add(0);

                    fs.EnableRaisingEvents = true;

                    await JobHost.LaunchAsync(workingDirectory);

                    stream.CompleteAdding();
                }

                // ================================

                return(0);
            }
            else
            {
                try
                {
                    var options = CommandLine.Parse(args, new Options {
                    });
                    if (options.compile)
                    {
                        var compilationOptions = new CompilationOptions(currentQueryPath);
                        compilationOptions.OutDir = options.out_dir == null?Path.Combine(compilationOptions.QueryDirectoryName, compilationOptions.QueryName + "_" + userQueryTypeInfo.Id) : Path.GetFullPath(options.out_dir);

                        Compiler.Compile(userQueryTypeInfo, currentQueryInfo, compilationOptions, currentQueryInfo);
                        Trace.WriteLine($"Done. Output written to '{compilationOptions.OutDir}'");
                        return(0);
                    }
                    else if (options.publish)
                    {
                        var compilationOptions = new CompilationOptions(currentQueryPath);
                        compilationOptions.OutDir = Path.Combine(compilationOptions.QueryDirectoryName, compilationOptions.QueryName + "_" + userQueryTypeInfo.Id + "_" + Environment.TickCount);
                        try
                        {
                            Compiler.Compile(userQueryTypeInfo, currentQueryInfo, compilationOptions, currentQueryInfo);

                            var publishSettingsFileNames = FileUtil.ResolveSearchPatternUpDirectoryTree(compilationOptions.QueryDirectoryName, "*.PublishSettings").ToList();
                            if (1 != publishSettingsFileNames.Count)
                            {
                                if (1 < publishSettingsFileNames.Count)
                                {
                                    throw new InvalidOperationException($"Aborted. Found two or more '*.PublishSettings' files. " + string.Join(", ", publishSettingsFileNames));
                                }
                                else
                                {
                                    throw new InvalidOperationException($"Aborted. Cannot find a '*.PublishSettings' file in '{compilationOptions.QueryDirectoryName}' or any of it's parents");
                                }
                            }

                            var kudu = KuduClient.FromPublishProfile(publishSettingsFileNames[0]);

                            var appSettings = kudu.GetSettings();
                            if (appSettings.TryGetValue("WEBSITE_SITE_NAME", out var siteName) && appSettings.TryGetValue("WEBSITE_SLOT_NAME", out var slotName))
                            {
                                Trace.WriteLine($"Publishing to '{siteName}' ({slotName})...");
                            }
                            else
                            {
                                Trace.WriteLine($"Site '{kudu.Host}' metadata is missing");
                            }

                            // this setting needs to be changed using the az command line
                            // https://docs.microsoft.com/en-us/azure/azure-functions/set-runtime-version#view-and-update-the-runtime-version-using-azure-cli
                            // using the Kudu settings API doesn't update the application app settings

                            if (appSettings.TryGetValue("FUNCTIONS_EXTENSION_VERSION", out var functionsExtensionVersion))
                            {
                                if (functionsExtensionVersion != "~1")
                                {
                                    var text    = "The Azure Functions runtime version 1.x is required. Would you like to change the FUNCTIONS_EXTENSION_VERSION setting to ~1?";
                                    var caption = "Azure Functions Runtime";

                                    if (MessageBox.ShowYesNoQuestion(text, caption))
                                    {
                                        var result = await Az.RunAsunc("functionapp", "list", "--query", $"[?name=='{siteName}']");

                                        if (result.Output.Count() != 1)
                                        {
                                            throw new InvalidOperationException($"Aborted. Cannot find Azure Function App '{siteName}'");
                                        }

                                        var functionApp = result.Output[0];

                                        var g = (string)functionApp["resourceGroup"];

                                        var result2 = await Az.RunAsunc("functionapp", "config", "appsettings", "set", "-g", g, "-n", siteName, "--settings", "FUNCTIONS_EXTENSION_VERSION=~1");

                                        if (result2.ExitCode != 0)
                                        {
                                            throw new InvalidOperationException($"Aborted. Cannot configure Azure Function App '{siteName}'");
                                        }
                                    }
                                    else
                                    {
                                        throw new InvalidOperationException("Aborted. Azure Functions runtime version 1.x is required");
                                    }
                                }
                            }

                            // need to check for cloud pad function app runtime
                            // if not found, offer to deploy it

                            if (!kudu.VfsExists("site/wwwroot/bin/CloudPad.FunctionApp.dll"))
                            {
                                if (hasConsoleInput)
                                {
                                    var text    = "It looks like the CloudPad.FunctionApp (runtime) has not been deployed yet. Would you like to deploy this now (you only do this once per Azure Function App)?";
                                    var caption = "Deployment";

                                    if (MessageBox.ShowYesNoQuestion(text, caption))
                                    {
                                        Trace.WriteLine($"Deploying runtime... (this will just take a minute)");
                                        kudu.ZipDeployPackage(FunctionApp.PackageUri);
                                    }
                                }
                                else
                                {
                                    Trace.WriteLine($"Oh no. You have to deploy the CloudPad.FunctionApp runtime first");
                                    return(1);
                                }
                            }

                            Trace.WriteLine($"Deploying script '{currentQueryPathInfo.QueryFileNameWithoutExtension}' ({userQueryTypeInfo.AssemblyName})...");

                            kudu.ZipUpload(compilationOptions.OutDir);
                        }
                        finally
                        {
                            if (Directory.Exists(compilationOptions.OutDir))
                            {
                                Directory.Delete(compilationOptions.OutDir, true);
                            }
                        }
                        Trace.WriteLine("Done.");
                        if (options.interactive)
                        {
                            if (hasConsoleInput)
                            {
                                Console.WriteLine("Press any key to continue...");
                                Console.ReadKey();
                            }
                        }
                        return(0);
                    }
                    else if (options.pack)
                    {
                        var compilationOptions = new CompilationOptions(currentQueryPath);
                        compilationOptions.OutDir = options.out_dir == null?Path.Combine(compilationOptions.QueryDirectoryName, compilationOptions.QueryName + "_publish") : Path.GetFullPath(options.out_dir);

                        FunctionApp.Deploy(compilationOptions.OutDir);
                        Compiler.Compile(userQueryTypeInfo, currentQueryInfo, compilationOptions, currentQueryInfo);
                        Trace.WriteLine($"Done. Output written to '{compilationOptions.OutDir}'");
                        return(0);
                    }
                    else if (options.install)
                    {
                        FirstRun.Install();
                        return(0);
                    }
                }
                catch (Exception ex)
                {
                    Console.WriteLine(ex.Message);
                    Console.WriteLine(ex.StackTrace);

                    if (hasConsoleInput)
                    {
                        Console.WriteLine("Press any key to continue...");
                        Console.ReadKey();
                    }

                    throw;
                }
                return(1);
            }
        }
 public KuduScannerBuilder(KuduClient client, KuduTable table, ILogger logger)
     : base(client, table)
 {
     Logger = logger;
 }
Beispiel #20
0
    public KuduScanner(
        ILogger logger,
        KuduClient client,
        KuduTable table,
        List <string>?projectedColumnNames,
        List <int>?projectedColumnIndexes,
        Dictionary <string, KuduPredicate> predicates,
        ReadMode readMode,
        ReplicaSelection replicaSelection,
        bool isFaultTolerant,
        int batchSizeBytes,
        long limit,
        bool cacheBlocks,
        long startTimestamp,
        long htTimestamp,
        byte[] lowerBoundPrimaryKey,
        byte[] upperBoundPrimaryKey,
        byte[] lowerBoundPartitionKey,
        byte[] upperBoundPartitionKey)
    {
        if (limit <= 0)
        {
            throw new ArgumentException($"Need a strictly positive number for the limit, got {limit}");
        }

        if (htTimestamp != KuduClient.NoTimestamp)
        {
            if (htTimestamp < 0)
            {
                throw new ArgumentException(
                          $"Need non-negative number for the scan, timestamp got {htTimestamp}");
            }

            if (readMode != ReadMode.ReadAtSnapshot)
            {
                throw new ArgumentException(
                          "When specifying a HybridClock timestamp, the read mode needs to be set to READ_AT_SNAPSHOT");
            }
        }

        if (startTimestamp != KuduClient.NoTimestamp)
        {
            if (htTimestamp < 0)
            {
                throw new ArgumentException(
                          "Must have both start and end timestamps for a diff scan");
            }

            if (startTimestamp > htTimestamp)
            {
                throw new ArgumentException(
                          "Start timestamp must be less than or equal to end timestamp");
            }
        }

        _isFaultTolerant = isFaultTolerant;
        if (isFaultTolerant)
        {
            if (readMode != ReadMode.ReadAtSnapshot)
            {
                throw new ArgumentException("Use of fault tolerance scanner " +
                                            "requires the read mode to be set to READ_AT_SNAPSHOT");
            }

            _orderMode = OrderMode.Ordered;
        }
        else
        {
            _orderMode = OrderMode.Unordered;
        }

        _logger                 = logger;
        _client                 = client;
        _table                  = table;
        _predicates             = predicates;
        ReadMode                = readMode;
        ReplicaSelection        = replicaSelection;
        _isFaultTolerant        = isFaultTolerant;
        _limit                  = limit;
        CacheBlocks             = cacheBlocks;
        _startTimestamp         = startTimestamp;
        _htTimestamp            = htTimestamp;
        _lowerBoundPrimaryKey   = lowerBoundPrimaryKey;
        _upperBoundPrimaryKey   = upperBoundPrimaryKey;
        _lowerBoundPartitionKey = lowerBoundPartitionKey;
        _upperBoundPartitionKey = upperBoundPartitionKey;

        // Add the IS_DELETED column for diff scans.
        bool includeDeletedColumn = startTimestamp != KuduClient.NoTimestamp;

        ProjectionSchema = GenerateProjectionSchema(
            table.Schema,
            projectedColumnNames,
            projectedColumnIndexes,
            includeDeletedColumn);

        _projectedColumnsPb = ToColumnSchemaPbs(ProjectionSchema);
        BatchSizeBytes      = GetBatchSizeEstimate(batchSizeBytes);
    }
Beispiel #21
0
 public KuduScanTokenBuilder(KuduClient client, KuduTable table, ISystemClock systemClock)
     : base(client, table)
 {
     _systemClock = systemClock;
 }