public async Task IngestFromCsvAsync(
string csv, ServicePrincipalOptions servicePrincipal, string cluster, string database, string table, bool isDryRun)
{
KustoConnectionStringBuilder connectionBuilder =
new KustoConnectionStringBuilder($"https://{cluster}.kusto.windows.net")
.WithAadApplicationKeyAuthentication(
servicePrincipal.ClientId,
servicePrincipal.Secret,
servicePrincipal.Tenant);
using (IKustoIngestClient client = KustoIngestFactory.CreateDirectIngestClient(connectionBuilder))
{
KustoIngestionProperties properties =
new(database, table) { Format = DataSourceFormat.csv };
StreamSourceOptions sourceOptions = new() { SourceId = Guid.NewGuid() };
if (!isDryRun)
{
AsyncRetryPolicy retryPolicy = Policy
.Handle <Kusto.Data.Exceptions.KustoException>()
.Or <Kusto.Ingest.Exceptions.KustoException>()
.WaitAndRetryAsync(
Backoff.DecorrelatedJitterBackoffV2(TimeSpan.FromSeconds(10), RetryHelper.MaxRetries),
RetryHelper.GetOnRetryDelegate(RetryHelper.MaxRetries, _loggerService));
IKustoIngestionResult result = await retryPolicy.ExecuteAsync(
() => IngestFromStreamAsync(csv, client, properties, sourceOptions));
IngestionStatus ingestionStatus = result.GetIngestionStatusBySourceId(sourceOptions.SourceId);
for (int i = 0; i < 10 && ingestionStatus.Status == Status.Pending; i++)
{
await Task.Delay(TimeSpan.FromSeconds(30));
ingestionStatus = result.GetIngestionStatusBySourceId(sourceOptions.SourceId);
}
if (ingestionStatus.Status == Status.Pending)
{
throw new InvalidOperationException($"Timeout while ingesting Kusto data.");
}
else if (ingestionStatus.Status != Status.Succeeded)
{
throw new InvalidOperationException(
$"Failed to ingest Kusto data.{Environment.NewLine}{ingestionStatus.Details}");
}
}
}
}
