public async Task IngestFromCsvAsync(
string csv, ServicePrincipalOptions servicePrincipal, string cluster, string database, string table, bool isDryRun)
{
KustoConnectionStringBuilder connectionBuilder =
new KustoConnectionStringBuilder($"https://{cluster}.kusto.windows.net")
.WithAadApplicationKeyAuthentication(
servicePrincipal.ClientId,
servicePrincipal.Secret,
servicePrincipal.Tenant);
using (IKustoIngestClient client = KustoIngestFactory.CreateDirectIngestClient(connectionBuilder))
{
KustoIngestionProperties properties =
new(database, table) { Format = DataSourceFormat.csv };
StreamSourceOptions sourceOptions = new() { SourceId = Guid.NewGuid() };
if (!isDryRun)
{
AsyncRetryPolicy retryPolicy = Policy
.Handle <Kusto.Data.Exceptions.KustoException>()
.Or <Kusto.Ingest.Exceptions.KustoException>()
.WaitAndRetryAsync(
Backoff.DecorrelatedJitterBackoffV2(TimeSpan.FromSeconds(10), RetryHelper.MaxRetries),
RetryHelper.GetOnRetryDelegate(RetryHelper.MaxRetries, _loggerService));
IKustoIngestionResult result = await retryPolicy.ExecuteAsync(
() => IngestFromStreamAsync(csv, client, properties, sourceOptions));
IngestionStatus ingestionStatus = result.GetIngestionStatusBySourceId(sourceOptions.SourceId);
for (int i = 0; i < 10 && ingestionStatus.Status == Status.Pending; i++)
{
await Task.Delay(TimeSpan.FromSeconds(30));
ingestionStatus = result.GetIngestionStatusBySourceId(sourceOptions.SourceId);
}
if (ingestionStatus.Status == Status.Pending)
{
throw new InvalidOperationException($"Timeout while ingesting Kusto data.");
}
else if (ingestionStatus.Status != Status.Succeeded)
{
throw new InvalidOperationException(
$"Failed to ingest Kusto data.{Environment.NewLine}{ingestionStatus.Details}");
}
}
}
}
public async Task IngestFromCsvStreamAsync(Stream csv, IngestKustoImageInfoOptions options)
{
KustoConnectionStringBuilder connectionBuilder =
new KustoConnectionStringBuilder($"https://{options.Cluster}.kusto.windows.net")
.WithAadApplicationKeyAuthentication(
options.ServicePrincipal.ClientId,
options.ServicePrincipal.Secret,
options.ServicePrincipal.Tenant);
using (IKustoIngestClient client = KustoIngestFactory.CreateDirectIngestClient(connectionBuilder))
{
KustoIngestionProperties properties =
new KustoIngestionProperties(options.Database, options.Table)
{
Format = DataSourceFormat.csv
};
StreamSourceOptions sourceOptions = new StreamSourceOptions {
SourceId = Guid.NewGuid()
};
if (!options.IsDryRun)
{
IKustoIngestionResult result = await client.IngestFromStreamAsync(csv, properties, sourceOptions);
IngestionStatus ingestionStatus = result.GetIngestionStatusBySourceId(sourceOptions.SourceId);
for (int i = 0; i < 10 && ingestionStatus.Status == Status.Pending; i++)
{
await Task.Delay(TimeSpan.FromSeconds(30));
ingestionStatus = result.GetIngestionStatusBySourceId(sourceOptions.SourceId);
}
if (ingestionStatus.Status != Status.Succeeded)
{
throw new InvalidOperationException(
$"Failed to ingest Kusto data.{Environment.NewLine}{ingestionStatus.Details}");
}
else if (ingestionStatus.Status == Status.Pending)
{
throw new InvalidOperationException($"Timeout while ingesting Kusto data.");
}
}
}
}
public static void Main(string[] args)
{
KustoConnectionStringBuilder readStringBuilder;
KustoConnectionStringBuilder ingestStringBuilder;
readStringBuilder = new KustoConnectionStringBuilder(@"https://kuskusops.kustomfa.windows.net")
{
FederatedSecurity = true,
InitialCatalog = "testdb",
Authority = "72f988bf-86f1-41af-91ab-2d7cd011db47"
};
ingestStringBuilder = new KustoConnectionStringBuilder($"https://ingest-kuskusops.kustomfa.windows.net")
{
FederatedSecurity = true,
InitialCatalog = "testdb",
Authority = "72f988bf-86f1-41af-91ab-2d7cd011db47"
};
using (IKustoIngestClient kustoIngestClient = KustoIngestFactory.CreateQueuedIngestClient(ingestStringBuilder))
{
var kustoIngestionProperties = new KustoQueuedIngestionProperties("testdb", "test_table");
var dataReader = GetDataAsIDataReader();
IKustoIngestionResult ingestionResult =
kustoIngestClient.IngestFromDataReader(dataReader, kustoIngestionProperties);
var ingestionStatus = ingestionResult.GetIngestionStatusCollection().First();
var watch = System.Diagnostics.Stopwatch.StartNew();
var shouldContinue = true;
while ((ingestionStatus.Status == Status.Pending) && (shouldContinue))
{
// Wait a minute...
Thread.Sleep(TimeSpan.FromMinutes(1));
// Try again
ingestionStatus = ingestionResult.GetIngestionStatusBySourceId(ingestionStatus.IngestionSourceId);
shouldContinue = watch.ElapsedMilliseconds < 360000;
}
watch.Stop();
if (ingestionStatus.Status == Status.Pending)
{
// The status of the ingestion did not change.
Console.WriteLine(
"Ingestion with ID:'{0}' did not complete. Timed out after :'{1}' Milliseconds"
.FormatWithInvariantCulture(
ingestionStatus.IngestionSourceId, watch.ElapsedMilliseconds));
}
else
{
// The status of the ingestion has changed
Console.WriteLine(
"Ingestion with ID:'{0}' is complete. Ingestion Status:'{1}'".FormatWithInvariantCulture(
ingestionStatus.IngestionSourceId, ingestionStatus.Status));
}
}
// 4. Show the contents of the table
using (var client = Kusto.Data.Net.Client.KustoClientFactory.CreateCslQueryProvider(readStringBuilder))
{
while (true)
{
var query = "test_table | count";
var reader = client.ExecuteQuery(query);
Kusto.Cloud.Platform.Data.ExtendedDataReader.WriteAsText(reader, "Data ingested into the table:", tabify: true, firstOnly: true);
Console.WriteLine("Press 'r' to retry retrieving data from the table, any other key to quit");
var key = Console.ReadKey();
if (key.KeyChar != 'r' || key.KeyChar != 'R')
{
break;
}
}
}
}
