static void Main(string[] args) { // Ingest From Local Files using KustoQueuedIngestClient and Ingestion Validation // Create Kusto connection string with App Authentication var kustoConnectionStringBuilderDM = new KustoConnectionStringBuilder(@"https://ingest-{clusterNameAndRegion}.kusto.windows.net").WithAadApplicationKeyAuthentication( applicationClientId: "{Application Client ID}", applicationKey: "{Application Key (secret)}", authority: "{AAD TenantID or name}"); // Create a disposable client that will execute the ingestion IKustoQueuedIngestClient client = KustoIngestFactory.CreateQueuedIngestClient(kustoConnectionStringBuilderDM); // Ingest from files according to the required properties var kustoIngestionProperties = new KustoIngestionProperties(databaseName: "myDB", tableName: "myTable"); client.IngestFromStorageAsync(@"ValidTestFile.csv", kustoIngestionProperties); client.IngestFromStorageAsync(@"InvalidTestFile.csv", kustoIngestionProperties); // Waiting for the aggregation Thread.Sleep(TimeSpan.FromMinutes(8)); // Retrieve and validate failures var ingestionFailures = client.PeekTopIngestionFailuresAsync().GetAwaiter().GetResult(); Ensure.IsTrue((ingestionFailures.Count() > 0), "Failures expected"); // Retrieve, delete and validate failures ingestionFailures = client.GetAndDiscardTopIngestionFailuresAsync().GetAwaiter().GetResult(); Ensure.IsTrue((ingestionFailures.Count() > 0), "Failures expected"); // Dispose of the client client.Dispose(); }
static void Main(string[] args) { // Ingest From a Local Files using KustoQueuedIngestClient and report status to a queue // Create Kusto connection string with App Authentication var kustoConnectionStringBuilderDM = new KustoConnectionStringBuilder(@"https://ingest-{clusterNameAndRegion}.kusto.windows.net").WithAadApplicationKeyAuthentication( applicationClientId: "{Application Client ID}", applicationKey: "{Application Key (secret)}", authority: "{AAD TenantID or name}"); // Create a disposable client that will execute the ingestion IKustoQueuedIngestClient client = KustoIngestFactory.CreateQueuedIngestClient(kustoConnectionStringBuilderDM); // Ingest from a file according to the required properties var kustoIngestionProperties = new KustoQueuedIngestionProperties(databaseName: "myDB", tableName: "myTable") { // Setting the report level to FailuresAndSuccesses will cause both successful and failed ingestions to be reported // (Rather than the default "FailuresOnly" level - which is demonstrated in the // 'Ingest From Local File(s) using KustoQueuedIngestClient and Ingestion Validation' section) ReportLevel = IngestionReportLevel.FailuresAndSuccesses, // Choose the report method of choice. 'Queue' is the default method. // For the sake of the example, we will choose it anyway. ReportMethod = IngestionReportMethod.Queue }; client.IngestFromStorageAsync("ValidTestFile.csv", kustoIngestionProperties); client.IngestFromStorageAsync("InvalidTestFile.csv", kustoIngestionProperties); // Waiting for the aggregation Thread.Sleep(TimeSpan.FromMinutes(8)); // Retrieve and validate failures var ingestionFailures = client.PeekTopIngestionFailuresAsync().GetAwaiter().GetResult(); Ensure.IsTrue((ingestionFailures.Count() > 0), "The failed ingestion should have been reported to the failed ingestions queue"); // Retrieve, delete and validate failures ingestionFailures = client.GetAndDiscardTopIngestionFailuresAsync().GetAwaiter().GetResult(); Ensure.IsTrue((ingestionFailures.Count() > 0), "The failed ingestion should have been reported to the failed ingestions queue"); // Verify the success has also been reported to the queue var ingestionSuccesses = client.GetAndDiscardTopIngestionSuccessesAsync().GetAwaiter().GetResult(); Ensure.ConditionIsMet((ingestionSuccesses.Count() > 0), "The successful ingestion should have been reported to the successful ingestions queue"); // Dispose of the client client.Dispose(); }
public void IngestData(string table, string mappingName, Stream memStream) { var ingestProps = new KustoQueuedIngestionProperties(DatabaseName, table) { ReportLevel = IngestionReportLevel.FailuresAndSuccesses, ReportMethod = IngestionReportMethod.Queue, JSONMappingReference = mappingName, Format = DataSourceFormat.json }; _ingestionClient.IngestFromStream(memStream, ingestProps, leaveOpen: true); // Wait and retrieve all notifications Thread.Sleep(10000); var errors = _ingestionClient.GetAndDiscardTopIngestionFailuresAsync().GetAwaiter().GetResult(); var successes = _ingestionClient.GetAndDiscardTopIngestionSuccessesAsync().GetAwaiter().GetResult(); errors.ForEach((f) => { Logger.Error($"Ingestion error: {f.Info.Details}."); }); successes.ForEach((s) => { Logger.Info($"Ingested : {s.Info.IngestionSourcePath}"); }); }