Exemplo n.º 1
0
        static void Main(string[] args)
        {
            // Ingest From a Local Files using KustoQueuedIngestClient and report status to a queue

            // Create Kusto connection string with App Authentication
            var kustoConnectionStringBuilderDM =
                new KustoConnectionStringBuilder(@"https://ingest-{clusterNameAndRegion}.kusto.windows.net").WithAadApplicationKeyAuthentication(
                    applicationClientId: "{Application Client ID}",
                    applicationKey: "{Application Key (secret)}",
                    authority: "{AAD TenantID or name}");

            // Create a disposable client that will execute the ingestion
            IKustoQueuedIngestClient client = KustoIngestFactory.CreateQueuedIngestClient(kustoConnectionStringBuilderDM);

            // Ingest from a file according to the required properties
            var kustoIngestionProperties = new KustoQueuedIngestionProperties(databaseName: "myDB", tableName: "myTable")
            {
                // Setting the report level to FailuresAndSuccesses will cause both successful and failed ingestions to be reported
                // (Rather than the default "FailuresOnly" level - which is demonstrated in the
                // 'Ingest From Local File(s) using KustoQueuedIngestClient and Ingestion Validation' section)
                ReportLevel = IngestionReportLevel.FailuresAndSuccesses,
                // Choose the report method of choice. 'Queue' is the default method.
                // For the sake of the example, we will choose it anyway.
                ReportMethod = IngestionReportMethod.Queue
            };

            client.IngestFromStorageAsync("ValidTestFile.csv", kustoIngestionProperties);
            client.IngestFromStorageAsync("InvalidTestFile.csv", kustoIngestionProperties);

            // Waiting for the aggregation
            Thread.Sleep(TimeSpan.FromMinutes(8));

            // Retrieve and validate failures
            var ingestionFailures = client.PeekTopIngestionFailuresAsync().GetAwaiter().GetResult();

            Ensure.IsTrue((ingestionFailures.Count() > 0), "The failed ingestion should have been reported to the failed ingestions queue");
            // Retrieve, delete and validate failures
            ingestionFailures = client.GetAndDiscardTopIngestionFailuresAsync().GetAwaiter().GetResult();
            Ensure.IsTrue((ingestionFailures.Count() > 0), "The failed ingestion should have been reported to the failed ingestions queue");

            // Verify the success has also been reported to the queue
            var ingestionSuccesses = client.GetAndDiscardTopIngestionSuccessesAsync().GetAwaiter().GetResult();

            Ensure.ConditionIsMet((ingestionSuccesses.Count() > 0),
                                  "The successful ingestion should have been reported to the successful ingestions queue");

            // Dispose of the client
            client.Dispose();
        }
Exemplo n.º 2
0
        public void IngestData(string table, string mappingName, Stream memStream)
        {
            var ingestProps =
                new KustoQueuedIngestionProperties(DatabaseName, table)
            {
                ReportLevel          = IngestionReportLevel.FailuresAndSuccesses,
                ReportMethod         = IngestionReportMethod.Queue,
                JSONMappingReference = mappingName,
                Format = DataSourceFormat.json
            };

            _ingestionClient.IngestFromStream(memStream, ingestProps, leaveOpen: true);

            // Wait and retrieve all notifications
            Thread.Sleep(10000);
            var errors    = _ingestionClient.GetAndDiscardTopIngestionFailuresAsync().GetAwaiter().GetResult();
            var successes = _ingestionClient.GetAndDiscardTopIngestionSuccessesAsync().GetAwaiter().GetResult();

            errors.ForEach((f) => { Logger.Error($"Ingestion error: {f.Info.Details}."); });
            successes.ForEach((s) => { Logger.Info($"Ingested : {s.Info.IngestionSourcePath}"); });
        }