private static async Task <long> CountScanTokenRowsAsync( KuduClient client, List <KuduScanToken> tokens) { var tasks = new List <Task <long> >(); foreach (var token in tokens) { var task = Task.Run(async() => { var tokenBytes = token.Serialize(); var scanBuilder = await client.NewScanBuilderFromTokenAsync(tokenBytes); var scanner = scanBuilder.Build(); return(await scanner.CountAsync()); }); tasks.Add(task); } var results = await Task.WhenAll(tasks); var rowCount = results.Sum(); return(rowCount); }
/// <summary> /// Load a table of default schema with the specified number of records, in ascending key order. /// </summary> public static async Task LoadDefaultTableAsync(KuduClient client, KuduTable table, int numRows) { var rows = Enumerable.Range(0, numRows) .Select(i => CreateBasicSchemaInsert(table, i)); await client.WriteAsync(rows); }
public KuduSession( KuduClient client, KuduSessionOptions options, ILoggerFactory loggerFactory, long txnId) { _client = client; _options = options; _txnId = txnId; _logger = loggerFactory.CreateLogger <KuduSession>(); var channelOptions = new BoundedChannelOptions(options.Capacity) { SingleWriter = false, SingleReader = true, FullMode = BoundedChannelFullMode.Wait, AllowSynchronousContinuations = false }; var channel = Channel.CreateBounded <KuduOperation>(channelOptions); _writer = channel.Writer; _reader = channel.Reader; _singleFlush = new SemaphoreSlim(1, 1); _flushCts = new CancellationTokenSource(); _flushTcs = new TaskCompletionSource( TaskCreationOptions.RunContinuationsAsynchronously); _consumeTask = ConsumeAsync(); }
private static async Task CommitAndVerifyTransactionAsync( KuduClient client, KuduTransaction transaction) { var tsBeforeCommit = client.LastPropagatedTimestamp; await transaction.CommitAsync(); Assert.Equal(tsBeforeCommit, client.LastPropagatedTimestamp); await transaction.WaitForCommitAsync(); var tsAfterCommit = client.LastPropagatedTimestamp; Assert.True(tsAfterCommit > tsBeforeCommit); // A sanity check: calling WaitForCommitAsync() again after the commit phase // has been finalized doesn't change last propagated timestamp at the // client side. for (int i = 0; i < 10; ++i) { await transaction.WaitForCommitAsync(); Assert.Equal(tsAfterCommit, client.LastPropagatedTimestamp); await Task.Delay(10); } }
public async Task InitializeAsync() { _harness = await new MiniKuduClusterBuilder().BuildHarnessAsync(); var options = _harness.CreateClientBuilder().BuildOptions(); var securityContext = new SecurityContext(); var systemClock = new SystemClock(); var connectionFactory = new KuduConnectionFactory( options, securityContext, NullLoggerFactory.Instance); _testConnectionFactory = new TestConnectionFactory(connectionFactory); _client = new KuduClient( options, securityContext, _testConnectionFactory, systemClock, NullLoggerFactory.Instance); var builder = ClientTestUtil.GetBasicSchema() .SetTableName("chaos_test_table") .SetNumReplicas(3) .CreateBasicRangePartition(); _table = await _client.CreateTableAsync(builder); }
/// <summary> /// Applies a list of operations and returns the final change type for each key. /// </summary> private async Task <Dictionary <int, RowOperation> > ApplyOperationsAsync( KuduClient client, List <KuduOperation> operations) { var results = new Dictionary <int, RowOperation>(); // If there are no operations, return early. if (operations.Count == 0) { return(results); } // On some runs, wait long enough to flush at the start. if (_random.NextBool()) { await Task.Delay(2000); } // Pick an int as a flush indicator so we flush once on average while applying operations. int flushInt = _random.Next(operations.Count); foreach (var operation in operations) { // On some runs, wait long enough to flush while applying operations. if (_random.Next(operations.Count) == flushInt) { await Task.Delay(2000); } await client.WriteAsync(new[] { operation }); results[operation.GetInt32(0)] = operation.Operation; } return(results); }
public static async Task <List <string> > ScanTableToStringsAsync( KuduClient client, KuduTable table, params KuduPredicate[] predicates) { var rowStrings = new List <string>(); var scanBuilder = client.NewScanBuilder(table) .SetReadMode(ReadMode.ReadYourWrites) .SetReplicaSelection(ReplicaSelection.LeaderOnly); foreach (var predicate in predicates) { scanBuilder.AddPredicate(predicate); } var scanner = scanBuilder.Build(); await foreach (var resultSet in scanner) { foreach (var row in resultSet) { rowStrings.Add(row.ToString()); } } rowStrings.Sort(); return(rowStrings); }
public AbstractKuduScannerBuilder(KuduClient client, KuduTable table) { Client = client; Table = table; Predicates = new Dictionary <string, KuduPredicate>(); ScanRequestTimeout = -1; // TODO: Pull this from the client. }
public KuduScanEnumerator( ILogger logger, KuduClient client, KuduTable table, List <ColumnSchemaPB> projectedColumnsPb, KuduSchema projectionSchema, OrderMode orderMode, ReadMode readMode, ReplicaSelection replicaSelection, bool isFaultTolerant, Dictionary <string, KuduPredicate> predicates, long limit, bool cacheBlocks, byte[] startPrimaryKey, byte[] endPrimaryKey, long startTimestamp, long htTimestamp, int batchSizeBytes, PartitionPruner partitionPruner, CancellationToken cancellationToken) { _logger = logger; _client = client; _table = table; _partitionPruner = partitionPruner; _orderMode = orderMode; _readMode = readMode; _columns = projectedColumnsPb; _schema = projectionSchema; _predicates = predicates; _replicaSelection = replicaSelection; _isFaultTolerant = isFaultTolerant; _limit = limit; _cacheBlocks = cacheBlocks; _startPrimaryKey = startPrimaryKey ?? Array.Empty <byte>(); _endPrimaryKey = endPrimaryKey ?? Array.Empty <byte>(); _startTimestamp = startTimestamp; SnapshotTimestamp = htTimestamp; _batchSizeBytes = batchSizeBytes; _scannerId = ByteString.Empty; _lastPrimaryKey = ByteString.Empty; _cancellationToken = cancellationToken; ResourceMetrics = new ResourceMetrics(); // If the partition pruner has pruned all partitions, then the scan can be // short circuited without contacting any tablet servers. if (!_partitionPruner.HasMorePartitionKeyRanges) { _closed = true; } // For READ_YOUR_WRITES scan mode, get the latest observed timestamp // and store it. Always use this one as the propagated timestamp for // the duration of the scan to avoid unnecessary wait. if (readMode == ReadMode.ReadYourWrites) { _lowerBoundPropagationTimestamp = client.LastPropagatedTimestamp; } }
public KuduTestHarness( MiniKuduCluster miniCluster, bool disposeMiniCluster = false) { _miniCluster = miniCluster; _disposeMiniCluster = disposeMiniCluster; _client = miniCluster.CreateClient(); }
public static ValueTask <long> CountRowsAsync(KuduClient client, KuduTable table) { var scanner = client.NewScanBuilder(table) .SetReadMode(ReadMode.ReadYourWrites) .SetReplicaSelection(ReplicaSelection.LeaderOnly) .Build(); return(scanner.CountAsync()); }
public async Task InitializeAsync() { _harness = await new MiniKuduClusterBuilder().BuildHarnessAsync(); _client = _harness.CreateClient(); await using var session = _client.NewSession(); // Create a 4-tablets table for scanning. var builder = new TableBuilder(_tableName) .AddColumn("key1", KuduType.String, opt => opt.Key(true)) .AddColumn("key2", KuduType.String, opt => opt.Key(true)) .AddColumn("val", KuduType.String) .SetRangePartitionColumns("key1", "key2"); for (int i = 1; i < 4; i++) { builder.AddSplitRow(splitRow => { splitRow.SetString("key1", i.ToString()); splitRow.SetString("key2", ""); }); } var table = await _client.CreateTableAsync(builder); // The data layout ends up like this: // tablet '', '1': no rows // tablet '1', '2': '111', '122', '133' // tablet '2', '3': '211', '222', '233' // tablet '3', '': '311', '322', '333' var keys = new[] { "1", "2", "3" }; foreach (var key1 in keys) { foreach (var key2 in keys) { var insert = table.NewInsert(); insert.SetString(0, key1); insert.SetString(1, key2); insert.SetString(2, key2); await session.EnqueueAsync(insert); await session.FlushAsync(); } } _beforeWriteTimestamp = _client.LastPropagatedTimestamp; // Reset the client in order to clear the propagated timestamp. _newClient = _harness.CreateClient(); // Reopen the table using the new client. _table = await _newClient.OpenTableAsync(_tableName); _schema = _table.Schema; }
public async Task InitializeAsync() { _harness = await new MiniKuduClusterBuilder().BuildHarnessAsync(); _client = _harness.CreateClient(); // Use one tablet because multiple tablets don't work: we could jump // from one tablet to another which could change the logical clock. var builder = new TableBuilder("HybridTimeTest") .AddColumn("key", KuduType.String, opt => opt.Key(true)) .SetRangePartitionColumns("key"); _table = await _client.CreateTableAsync(builder); }
public async Task InitializeAsync() { _harness = await new MiniKuduClusterBuilder().BuildHarnessAsync(); _client = _harness.CreateClient(); var builder = ClientTestUtil.GetBasicSchema() .SetTableName("ScannerFaultToleranceTests") .AddHashPartitions(_numTablets, "key"); _table = await _client.CreateTableAsync(builder); var random = new Random(); _keys = new HashSet <int>(); while (_keys.Count < _numRows) { _keys.Add(random.Next()); } var rows = _keys .Select((key, i) => { var insert = _table.NewInsert(); insert.SetInt32(0, key); insert.SetInt32(1, i); insert.SetInt32(2, i++); insert.SetString(3, DataGenerator.RandomString(1024, random)); insert.SetBool(4, true); return(insert); }) .Chunk(1000); foreach (var batch in rows) { await _client.WriteAsync(batch); } }
private static KuduTransaction MakeFakeTransaction( KuduClient client, KuduTransaction transaction) { var buf = transaction.Serialize(); var pb = TxnTokenPB.Parser.ParseFrom(buf); Assert.True(pb.HasTxnId); var txnId = pb.TxnId; Assert.True(txnId > KuduClient.InvalidTxnId); var fakeTxnId = txnId + 123; var message = new TxnTokenPB { TxnId = fakeTxnId, EnableKeepalive = false, KeepaliveMillis = 0 }; var fakeTxnBuf = ProtobufHelper.ToByteArray(message); return(client.NewTransactionFromToken(fakeTxnBuf)); }
static async Task Main() { var masterAddresses = "localhost:7051,localhost:7151,localhost:7251"; var tableName = $"test_table_{Guid.NewGuid():N}"; var numRows = 10000; var loggerFactory = LoggerFactory.Create(builder => builder .SetMinimumLevel(LogLevel.Trace) .AddConsole()); await using var client = KuduClient.NewBuilder(masterAddresses) .SetLoggerFactory(loggerFactory) .Build(); var tableBuilder = new TableBuilder(tableName) .AddColumn("host", KuduType.String, opt => opt.Key(true)) .AddColumn("metric", KuduType.String, opt => opt.Key(true)) .AddColumn("timestamp", KuduType.UnixtimeMicros, opt => opt.Key(true)) .AddColumn("value", KuduType.Double) .SetNumReplicas(1) .SetRangePartitionColumns("host", "metric", "timestamp"); var table = await client.CreateTableAsync(tableBuilder); Console.WriteLine($"Created table {tableName}"); var batches = CreateRows(table, numRows).Chunk(2000); var writtenRows = 0; foreach (var batch in batches) { await client.WriteAsync(batch); writtenRows += batch.Length; Console.WriteLine($"Wrote {writtenRows} rows"); } }
public async Task InitializeAsync() { _harness = await new MiniKuduClusterBuilder().BuildHarnessAsync(); _client = _harness.CreateClient(); }
// LINQPad script entry point // when deployed as an Azure Function this method is not used public static async Task <int> MainAsync(object userQuery, string[] args) { var hasConsoleInput = false; if ("LPRun.exe".Equals(Process.GetCurrentProcess().MainModule.ModuleName, StringComparison.OrdinalIgnoreCase)) { hasConsoleInput = Environment.UserInteractive; // pipe trace to console Trace.Listeners.Add(new ConsoleTraceListener()); } if (userQuery == null) { throw new ArgumentNullException("User query cannot be null. You should pass 'this' here.", nameof(userQuery)); } var userQueryTypeInfo = new UserQueryTypeInfo(userQuery); var currentQuery = Util.CurrentQuery; if (currentQuery == null) { throw new InvalidOperationException("This script must be run from wthin a LINQPad context (either via LINQPad or LPRun)."); } var currentQueryInfo = new QueryInfo(currentQuery); var currentQueryPath = Util.CurrentQueryPath; if (currentQueryPath == null) { throw new InvalidOperationException("A file name is required (save your LINQPad query to disk). Without it, we cannot establish a context for your functions."); } var currentQueryPathInfo = new QueryPathInfo(currentQueryPath); // ======== args = args ?? new string[0]; // note: `args` can be null if (args.Length == 0) { if (FirstRun.ShouldPrompt()) { FirstRun.Prompt(); } // ================================ var workingDirectory = Path.Combine(Env.GetLocalAppDataDirectory(), currentQueryPathInfo.InstanceId); Debug.WriteLine($"workingDirectory: {workingDirectory}"); FunctionApp.Deploy(workingDirectory); // ================================ Compiler.Compile(new UserQueryTypeInfo(userQuery), currentQueryInfo, new CompilationOptions(currentQueryPath) { OutDir = workingDirectory, }, currentQueryInfo); // ================================ StorageEmulator.StartOrInstall(); // todo: if AzureWebJobsStorage or AzureWebJobsDashboard is set elsewhere, like app.config we shouldn't override them like this if (string.IsNullOrEmpty(Environment.GetEnvironmentVariable("AzureWebJobsStorage"))) { Environment.SetEnvironmentVariable("AzureWebJobsStorage", "UseDevelopmentStorage=true"); } if (string.IsNullOrEmpty(Environment.GetEnvironmentVariable("AzureWebJobsDashboard"))) { Environment.SetEnvironmentVariable("AzureWebJobsDashboard", "UseDevelopmentStorage=true"); } // ================================ var lastWriteTime = File.GetLastWriteTimeUtc(currentQueryPathInfo.QueryPath); using (var fs = new FileSystemWatcher(currentQueryPathInfo.QueryDirectoryName, currentQueryPathInfo.QueryFileName)) { var stream = new BlockingCollection <int>(); ThreadPool.QueueUserWorkItem(_ => { foreach (var x in stream.GetConsumingEnumerable()) { Thread.Sleep(250); // hold, for just a moment... var lastWriteTime2 = File.GetLastWriteTimeUtc(currentQueryPathInfo.QueryPath); if (lastWriteTime < lastWriteTime2) { Debug.WriteLine("Recompiling..."); Util.RunAndWait(currentQueryPathInfo.QueryPath, new[] { "-compile", "-out-dir", workingDirectory }); Debug.WriteLine("Recompiled!"); lastWriteTime = lastWriteTime2; } } }); fs.Changed += (sender, e) => stream.Add(0); fs.EnableRaisingEvents = true; await JobHost.LaunchAsync(workingDirectory); stream.CompleteAdding(); } // ================================ return(0); } else { try { var options = CommandLine.Parse(args, new Options { }); if (options.compile) { var compilationOptions = new CompilationOptions(currentQueryPath); compilationOptions.OutDir = options.out_dir == null?Path.Combine(compilationOptions.QueryDirectoryName, compilationOptions.QueryName + "_" + userQueryTypeInfo.Id) : Path.GetFullPath(options.out_dir); Compiler.Compile(userQueryTypeInfo, currentQueryInfo, compilationOptions, currentQueryInfo); Trace.WriteLine($"Done. Output written to '{compilationOptions.OutDir}'"); return(0); } else if (options.publish) { var compilationOptions = new CompilationOptions(currentQueryPath); compilationOptions.OutDir = Path.Combine(compilationOptions.QueryDirectoryName, compilationOptions.QueryName + "_" + userQueryTypeInfo.Id + "_" + Environment.TickCount); try { Compiler.Compile(userQueryTypeInfo, currentQueryInfo, compilationOptions, currentQueryInfo); var publishSettingsFileNames = FileUtil.ResolveSearchPatternUpDirectoryTree(compilationOptions.QueryDirectoryName, "*.PublishSettings").ToList(); if (1 != publishSettingsFileNames.Count) { if (1 < publishSettingsFileNames.Count) { throw new InvalidOperationException($"Aborted. Found two or more '*.PublishSettings' files. " + string.Join(", ", publishSettingsFileNames)); } else { throw new InvalidOperationException($"Aborted. Cannot find a '*.PublishSettings' file in '{compilationOptions.QueryDirectoryName}' or any of it's parents"); } } var kudu = KuduClient.FromPublishProfile(publishSettingsFileNames[0]); var appSettings = kudu.GetSettings(); if (appSettings.TryGetValue("WEBSITE_SITE_NAME", out var siteName) && appSettings.TryGetValue("WEBSITE_SLOT_NAME", out var slotName)) { Trace.WriteLine($"Publishing to '{siteName}' ({slotName})..."); } else { Trace.WriteLine($"Site '{kudu.Host}' metadata is missing"); } // this setting needs to be changed using the az command line // https://docs.microsoft.com/en-us/azure/azure-functions/set-runtime-version#view-and-update-the-runtime-version-using-azure-cli // using the Kudu settings API doesn't update the application app settings if (appSettings.TryGetValue("FUNCTIONS_EXTENSION_VERSION", out var functionsExtensionVersion)) { if (functionsExtensionVersion != "~1") { var text = "The Azure Functions runtime version 1.x is required. Would you like to change the FUNCTIONS_EXTENSION_VERSION setting to ~1?"; var caption = "Azure Functions Runtime"; if (MessageBox.ShowYesNoQuestion(text, caption)) { var result = await Az.RunAsunc("functionapp", "list", "--query", $"[?name=='{siteName}']"); if (result.Output.Count() != 1) { throw new InvalidOperationException($"Aborted. Cannot find Azure Function App '{siteName}'"); } var functionApp = result.Output[0]; var g = (string)functionApp["resourceGroup"]; var result2 = await Az.RunAsunc("functionapp", "config", "appsettings", "set", "-g", g, "-n", siteName, "--settings", "FUNCTIONS_EXTENSION_VERSION=~1"); if (result2.ExitCode != 0) { throw new InvalidOperationException($"Aborted. Cannot configure Azure Function App '{siteName}'"); } } else { throw new InvalidOperationException("Aborted. Azure Functions runtime version 1.x is required"); } } } // need to check for cloud pad function app runtime // if not found, offer to deploy it if (!kudu.VfsExists("site/wwwroot/bin/CloudPad.FunctionApp.dll")) { if (hasConsoleInput) { var text = "It looks like the CloudPad.FunctionApp (runtime) has not been deployed yet. Would you like to deploy this now (you only do this once per Azure Function App)?"; var caption = "Deployment"; if (MessageBox.ShowYesNoQuestion(text, caption)) { Trace.WriteLine($"Deploying runtime... (this will just take a minute)"); kudu.ZipDeployPackage(FunctionApp.PackageUri); } } else { Trace.WriteLine($"Oh no. You have to deploy the CloudPad.FunctionApp runtime first"); return(1); } } Trace.WriteLine($"Deploying script '{currentQueryPathInfo.QueryFileNameWithoutExtension}' ({userQueryTypeInfo.AssemblyName})..."); kudu.ZipUpload(compilationOptions.OutDir); } finally { if (Directory.Exists(compilationOptions.OutDir)) { Directory.Delete(compilationOptions.OutDir, true); } } Trace.WriteLine("Done."); if (options.interactive) { if (hasConsoleInput) { Console.WriteLine("Press any key to continue..."); Console.ReadKey(); } } return(0); } else if (options.pack) { var compilationOptions = new CompilationOptions(currentQueryPath); compilationOptions.OutDir = options.out_dir == null?Path.Combine(compilationOptions.QueryDirectoryName, compilationOptions.QueryName + "_publish") : Path.GetFullPath(options.out_dir); FunctionApp.Deploy(compilationOptions.OutDir); Compiler.Compile(userQueryTypeInfo, currentQueryInfo, compilationOptions, currentQueryInfo); Trace.WriteLine($"Done. Output written to '{compilationOptions.OutDir}'"); return(0); } else if (options.install) { FirstRun.Install(); return(0); } } catch (Exception ex) { Console.WriteLine(ex.Message); Console.WriteLine(ex.StackTrace); if (hasConsoleInput) { Console.WriteLine("Press any key to continue..."); Console.ReadKey(); } throw; } return(1); } }
public KuduScannerBuilder(KuduClient client, KuduTable table, ILogger logger) : base(client, table) { Logger = logger; }
public KuduScanner( ILogger logger, KuduClient client, KuduTable table, List <string>?projectedColumnNames, List <int>?projectedColumnIndexes, Dictionary <string, KuduPredicate> predicates, ReadMode readMode, ReplicaSelection replicaSelection, bool isFaultTolerant, int batchSizeBytes, long limit, bool cacheBlocks, long startTimestamp, long htTimestamp, byte[] lowerBoundPrimaryKey, byte[] upperBoundPrimaryKey, byte[] lowerBoundPartitionKey, byte[] upperBoundPartitionKey) { if (limit <= 0) { throw new ArgumentException($"Need a strictly positive number for the limit, got {limit}"); } if (htTimestamp != KuduClient.NoTimestamp) { if (htTimestamp < 0) { throw new ArgumentException( $"Need non-negative number for the scan, timestamp got {htTimestamp}"); } if (readMode != ReadMode.ReadAtSnapshot) { throw new ArgumentException( "When specifying a HybridClock timestamp, the read mode needs to be set to READ_AT_SNAPSHOT"); } } if (startTimestamp != KuduClient.NoTimestamp) { if (htTimestamp < 0) { throw new ArgumentException( "Must have both start and end timestamps for a diff scan"); } if (startTimestamp > htTimestamp) { throw new ArgumentException( "Start timestamp must be less than or equal to end timestamp"); } } _isFaultTolerant = isFaultTolerant; if (isFaultTolerant) { if (readMode != ReadMode.ReadAtSnapshot) { throw new ArgumentException("Use of fault tolerance scanner " + "requires the read mode to be set to READ_AT_SNAPSHOT"); } _orderMode = OrderMode.Ordered; } else { _orderMode = OrderMode.Unordered; } _logger = logger; _client = client; _table = table; _predicates = predicates; ReadMode = readMode; ReplicaSelection = replicaSelection; _isFaultTolerant = isFaultTolerant; _limit = limit; CacheBlocks = cacheBlocks; _startTimestamp = startTimestamp; _htTimestamp = htTimestamp; _lowerBoundPrimaryKey = lowerBoundPrimaryKey; _upperBoundPrimaryKey = upperBoundPrimaryKey; _lowerBoundPartitionKey = lowerBoundPartitionKey; _upperBoundPartitionKey = upperBoundPartitionKey; // Add the IS_DELETED column for diff scans. bool includeDeletedColumn = startTimestamp != KuduClient.NoTimestamp; ProjectionSchema = GenerateProjectionSchema( table.Schema, projectedColumnNames, projectedColumnIndexes, includeDeletedColumn); _projectedColumnsPb = ToColumnSchemaPbs(ProjectionSchema); BatchSizeBytes = GetBatchSizeEstimate(batchSizeBytes); }
public KuduScanTokenBuilder(KuduClient client, KuduTable table, ISystemClock systemClock) : base(client, table) { _systemClock = systemClock; }