public async Task IngestFromCsvAsync( string csv, ServicePrincipalOptions servicePrincipal, string cluster, string database, string table, bool isDryRun) { KustoConnectionStringBuilder connectionBuilder = new KustoConnectionStringBuilder($"https://{cluster}.kusto.windows.net") .WithAadApplicationKeyAuthentication( servicePrincipal.ClientId, servicePrincipal.Secret, servicePrincipal.Tenant); using (IKustoIngestClient client = KustoIngestFactory.CreateDirectIngestClient(connectionBuilder)) { KustoIngestionProperties properties = new(database, table) { Format = DataSourceFormat.csv }; StreamSourceOptions sourceOptions = new() { SourceId = Guid.NewGuid() }; if (!isDryRun) { AsyncRetryPolicy retryPolicy = Policy .Handle <Kusto.Data.Exceptions.KustoException>() .Or <Kusto.Ingest.Exceptions.KustoException>() .WaitAndRetryAsync( Backoff.DecorrelatedJitterBackoffV2(TimeSpan.FromSeconds(10), RetryHelper.MaxRetries), RetryHelper.GetOnRetryDelegate(RetryHelper.MaxRetries, _loggerService)); IKustoIngestionResult result = await retryPolicy.ExecuteAsync( () => IngestFromStreamAsync(csv, client, properties, sourceOptions)); IngestionStatus ingestionStatus = result.GetIngestionStatusBySourceId(sourceOptions.SourceId); for (int i = 0; i < 10 && ingestionStatus.Status == Status.Pending; i++) { await Task.Delay(TimeSpan.FromSeconds(30)); ingestionStatus = result.GetIngestionStatusBySourceId(sourceOptions.SourceId); } if (ingestionStatus.Status == Status.Pending) { throw new InvalidOperationException($"Timeout while ingesting Kusto data."); } else if (ingestionStatus.Status != Status.Succeeded) { throw new InvalidOperationException( $"Failed to ingest Kusto data.{Environment.NewLine}{ingestionStatus.Details}"); } } } }
public async Task IngestFromCsvStreamAsync(Stream csv, IngestKustoImageInfoOptions options) { KustoConnectionStringBuilder connectionBuilder = new KustoConnectionStringBuilder($"https://{options.Cluster}.kusto.windows.net") .WithAadApplicationKeyAuthentication( options.ServicePrincipal.ClientId, options.ServicePrincipal.Secret, options.ServicePrincipal.Tenant); using (IKustoIngestClient client = KustoIngestFactory.CreateDirectIngestClient(connectionBuilder)) { KustoIngestionProperties properties = new KustoIngestionProperties(options.Database, options.Table) { Format = DataSourceFormat.csv }; StreamSourceOptions sourceOptions = new StreamSourceOptions { SourceId = Guid.NewGuid() }; if (!options.IsDryRun) { IKustoIngestionResult result = await client.IngestFromStreamAsync(csv, properties, sourceOptions); IngestionStatus ingestionStatus = result.GetIngestionStatusBySourceId(sourceOptions.SourceId); for (int i = 0; i < 10 && ingestionStatus.Status == Status.Pending; i++) { await Task.Delay(TimeSpan.FromSeconds(30)); ingestionStatus = result.GetIngestionStatusBySourceId(sourceOptions.SourceId); } if (ingestionStatus.Status != Status.Succeeded) { throw new InvalidOperationException( $"Failed to ingest Kusto data.{Environment.NewLine}{ingestionStatus.Details}"); } else if (ingestionStatus.Status == Status.Pending) { throw new InvalidOperationException($"Timeout while ingesting Kusto data."); } } } }
public static void Main(string[] args) { KustoConnectionStringBuilder readStringBuilder; KustoConnectionStringBuilder ingestStringBuilder; readStringBuilder = new KustoConnectionStringBuilder(@"https://kuskusops.kustomfa.windows.net") { FederatedSecurity = true, InitialCatalog = "testdb", Authority = "72f988bf-86f1-41af-91ab-2d7cd011db47" }; ingestStringBuilder = new KustoConnectionStringBuilder($"https://ingest-kuskusops.kustomfa.windows.net") { FederatedSecurity = true, InitialCatalog = "testdb", Authority = "72f988bf-86f1-41af-91ab-2d7cd011db47" }; using (IKustoIngestClient kustoIngestClient = KustoIngestFactory.CreateQueuedIngestClient(ingestStringBuilder)) { var kustoIngestionProperties = new KustoQueuedIngestionProperties("testdb", "test_table"); var dataReader = GetDataAsIDataReader(); IKustoIngestionResult ingestionResult = kustoIngestClient.IngestFromDataReader(dataReader, kustoIngestionProperties); var ingestionStatus = ingestionResult.GetIngestionStatusCollection().First(); var watch = System.Diagnostics.Stopwatch.StartNew(); var shouldContinue = true; while ((ingestionStatus.Status == Status.Pending) && (shouldContinue)) { // Wait a minute... Thread.Sleep(TimeSpan.FromMinutes(1)); // Try again ingestionStatus = ingestionResult.GetIngestionStatusBySourceId(ingestionStatus.IngestionSourceId); shouldContinue = watch.ElapsedMilliseconds < 360000; } watch.Stop(); if (ingestionStatus.Status == Status.Pending) { // The status of the ingestion did not change. Console.WriteLine( "Ingestion with ID:'{0}' did not complete. Timed out after :'{1}' Milliseconds" .FormatWithInvariantCulture( ingestionStatus.IngestionSourceId, watch.ElapsedMilliseconds)); } else { // The status of the ingestion has changed Console.WriteLine( "Ingestion with ID:'{0}' is complete. Ingestion Status:'{1}'".FormatWithInvariantCulture( ingestionStatus.IngestionSourceId, ingestionStatus.Status)); } } // 4. Show the contents of the table using (var client = Kusto.Data.Net.Client.KustoClientFactory.CreateCslQueryProvider(readStringBuilder)) { while (true) { var query = "test_table | count"; var reader = client.ExecuteQuery(query); Kusto.Cloud.Platform.Data.ExtendedDataReader.WriteAsText(reader, "Data ingested into the table:", tabify: true, firstOnly: true); Console.WriteLine("Press 'r' to retry retrieving data from the table, any other key to quit"); var key = Console.ReadKey(); if (key.KeyChar != 'r' || key.KeyChar != 'R') { break; } } } }