static void Main(string[] args)
{
// Ingest From Local Files using KustoQueuedIngestClient and Ingestion Validation
// Create Kusto connection string with App Authentication
var kustoConnectionStringBuilderDM =
new KustoConnectionStringBuilder(@"https://ingest-{clusterNameAndRegion}.kusto.windows.net").WithAadApplicationKeyAuthentication(
applicationClientId: "{Application Client ID}",
applicationKey: "{Application Key (secret)}",
authority: "{AAD TenantID or name}");
// Create a disposable client that will execute the ingestion
IKustoQueuedIngestClient client = KustoIngestFactory.CreateQueuedIngestClient(kustoConnectionStringBuilderDM);
// Ingest from files according to the required properties
var kustoIngestionProperties = new KustoIngestionProperties(databaseName: "myDB", tableName: "myTable");
client.IngestFromStorageAsync(@"ValidTestFile.csv", kustoIngestionProperties);
client.IngestFromStorageAsync(@"InvalidTestFile.csv", kustoIngestionProperties);
// Waiting for the aggregation
Thread.Sleep(TimeSpan.FromMinutes(8));
// Retrieve and validate failures
var ingestionFailures = client.PeekTopIngestionFailuresAsync().GetAwaiter().GetResult();
Ensure.IsTrue((ingestionFailures.Count() > 0), "Failures expected");
// Retrieve, delete and validate failures
ingestionFailures = client.GetAndDiscardTopIngestionFailuresAsync().GetAwaiter().GetResult();
Ensure.IsTrue((ingestionFailures.Count() > 0), "Failures expected");
// Dispose of the client
client.Dispose();
}
static void Main(string[] args)
{
// Ingest From a Local Files using KustoQueuedIngestClient and report status to a queue
// Create Kusto connection string with App Authentication
var kustoConnectionStringBuilderDM =
new KustoConnectionStringBuilder(@"https://ingest-{clusterNameAndRegion}.kusto.windows.net").WithAadApplicationKeyAuthentication(
applicationClientId: "{Application Client ID}",
applicationKey: "{Application Key (secret)}",
authority: "{AAD TenantID or name}");
// Create a disposable client that will execute the ingestion
IKustoQueuedIngestClient client = KustoIngestFactory.CreateQueuedIngestClient(kustoConnectionStringBuilderDM);
// Ingest from a file according to the required properties
var kustoIngestionProperties = new KustoQueuedIngestionProperties(databaseName: "myDB", tableName: "myTable")
{
// Setting the report level to FailuresAndSuccesses will cause both successful and failed ingestions to be reported
// (Rather than the default "FailuresOnly" level - which is demonstrated in the
// 'Ingest From Local File(s) using KustoQueuedIngestClient and Ingestion Validation' section)
ReportLevel = IngestionReportLevel.FailuresAndSuccesses,
// Choose the report method of choice. 'Queue' is the default method.
// For the sake of the example, we will choose it anyway.
ReportMethod = IngestionReportMethod.Queue
};
client.IngestFromStorageAsync("ValidTestFile.csv", kustoIngestionProperties);
client.IngestFromStorageAsync("InvalidTestFile.csv", kustoIngestionProperties);
// Waiting for the aggregation
Thread.Sleep(TimeSpan.FromMinutes(8));
// Retrieve and validate failures
var ingestionFailures = client.PeekTopIngestionFailuresAsync().GetAwaiter().GetResult();
Ensure.IsTrue((ingestionFailures.Count() > 0), "The failed ingestion should have been reported to the failed ingestions queue");
// Retrieve, delete and validate failures
ingestionFailures = client.GetAndDiscardTopIngestionFailuresAsync().GetAwaiter().GetResult();
Ensure.IsTrue((ingestionFailures.Count() > 0), "The failed ingestion should have been reported to the failed ingestions queue");
// Verify the success has also been reported to the queue
var ingestionSuccesses = client.GetAndDiscardTopIngestionSuccessesAsync().GetAwaiter().GetResult();
Ensure.ConditionIsMet((ingestionSuccesses.Count() > 0),
"The successful ingestion should have been reported to the successful ingestions queue");
// Dispose of the client
client.Dispose();
}
public void IngestData(string table, string mappingName, Stream memStream)
{
var ingestProps =
new KustoQueuedIngestionProperties(DatabaseName, table)
{
ReportLevel = IngestionReportLevel.FailuresAndSuccesses,
ReportMethod = IngestionReportMethod.Queue,
JSONMappingReference = mappingName,
Format = DataSourceFormat.json
};
_ingestionClient.IngestFromStream(memStream, ingestProps, leaveOpen: true);
// Wait and retrieve all notifications
Thread.Sleep(10000);
var errors = _ingestionClient.GetAndDiscardTopIngestionFailuresAsync().GetAwaiter().GetResult();
var successes = _ingestionClient.GetAndDiscardTopIngestionSuccessesAsync().GetAwaiter().GetResult();
errors.ForEach((f) => { Logger.Error($"Ingestion error: {f.Info.Details}."); });
successes.ForEach((s) => { Logger.Info($"Ingested : {s.Info.IngestionSourcePath}"); });
}