Example #1
0
        public KustoClientLogger(string databaseName, string tableName, List <KustoColumnMapping> _schema, string clientId, string clientKey)
        {
            var csvColumnMappings = new List <CsvColumnMapping>();

            foreach (var mapping in _schema)
            {
                csvColumnMappings.Add(new CsvColumnMapping {
                    ColumnName = mapping.ColumnName, CslDataType = mapping.DataType, Ordinal = mapping.ColumnNumber
                });
            }

            _kustoProperties = new KustoIngestionProperties(databaseName, tableName)
            {
                Format     = DataSourceFormat.csv,
                CSVMapping = csvColumnMappings
            };

            var kustoConnectionStringBuilderDM = new KustoConnectionStringBuilder(@"https://wawstest.kusto.windows.net:443")
            {
                FederatedSecurity   = true,
                InitialCatalog      = databaseName,
                ApplicationKey      = clientKey,
                ApplicationClientId = clientId
            };

            _kustoClient = KustoIngestFactory.CreateDirectIngestClient(kustoConnectionStringBuilderDM);
        }
Example #2
0
 /// <summary>
 ///     Constructor.  Initializes this object and does nothing else.
 /// </summary>
 /// <param name="connectionString">Kusto connection string.</param>
 /// <param name="database">Database into which to ingest.</param>
 /// <param name="table">Table into which to ingest.</param>
 /// <param name="csvMapping">Csv mapping that applies to all CSV files to be uploaded via <see cref="UploadSingleCsvFile"/></param>
 /// <param name="deleteFilesOnSuccess">Whether to delete files upon successful upload.</param>
 /// <param name="checkForIngestionErrors">
 ///     Whether to check for ingestion errors before disposing this object.
 ///     Note that at this time not all uploaded files have necessarily been ingested; this class
 ///     does not wait for ingestions to complete, it only checks for failures of those that have completed.
 /// </param>
 /// <param name="log">Optional log to which to write some debug information.</param>
 public KustoUploader
 (
     string connectionString,
     string database,
     string table,
     IEnumerable <CsvColumnMapping> csvMapping,
     bool deleteFilesOnSuccess,
     bool checkForIngestionErrors,
     ILog log = null
 )
 {
     _log = log;
     _deleteFilesOnSuccess    = deleteFilesOnSuccess;
     _checkForIngestionErrors = checkForIngestionErrors;
     _client              = KustoIngestFactory.CreateQueuedIngestClient(connectionString);
     _hasUploadErrors     = false;
     _ingestionProperties = new KustoQueuedIngestionProperties(database, table)
     {
         CSVMapping   = csvMapping,
         ReportLevel  = IngestionReportLevel.FailuresOnly,
         ReportMethod = IngestionReportMethod.Queue,
     };
     _block = new ActionBlock <FileDescription>
              (
         UploadSingleCsvFile,
         new ExecutionDataflowBlockOptions()
     {
         MaxDegreeOfParallelism = 1,
         BoundedCapacity        = DataflowBlockOptions.Unbounded
     }
              );
 }
        static async Task Main(string[] args)
        {
            // Async Ingestion From a Single Azure Blob using KustoQueuedIngestClient with (optional) RetryPolicy:

            //Create Kusto connection string with App Authentication
            var kustoConnectionStringBuilderDM =
                new KustoConnectionStringBuilder(@"https://ingest-{clusterNameAndRegion}.kusto.windows.net").WithAadApplicationKeyAuthentication(
                    applicationClientId: "{Application Client ID}",
                    applicationKey: "{Application Key (secret)}",
                    authority: "{AAD TenantID or name}");

            // Create an ingest client
            // Note, that creating a separate instance per ingestion operation is an anti-pattern.
            // IngestClient classes are thread-safe and intended for reuse
            IKustoIngestClient client = KustoIngestFactory.CreateQueuedIngestClient(kustoConnectionStringBuilderDM);

            // Ingest from blobs according to the required properties
            var kustoIngestionProperties = new KustoIngestionProperties(databaseName: "myDB", tableName: "myTable");

            var sourceOptions = new StorageSourceOptions()
            {
                DeleteSourceOnSuccess = true
            };

            //// Create your custom implementation of IRetryPolicy, which will affect how the ingest client handles retrying on transient failures
            IRetryPolicy retryPolicy = new NoRetry();

            //// This line sets the retry policy on the ingest client that will be enforced on every ingest call from here on
            ((IKustoQueuedIngestClient)client).QueueRetryPolicy = retryPolicy;

            await client.IngestFromStorageAsync(uri : @"BLOB-URI-WITH-SAS-KEY", ingestionProperties : kustoIngestionProperties, sourceOptions);

            client.Dispose();
        }
Example #4
0
        public BlockingKustoUploader(
            KustoConnectionStringBuilder kscb,
            string tableName,
            int batchSize,
            TimeSpan flushDuration)
        {
            csb             = kscb;
            TableName       = TableName;
            BatchSize       = batchSize;
            _flushDuration  = flushDuration;
            _lastUploadTime = DateTime.Now;
            Completed       = new AutoResetEvent(false);

            _ingestionProperties = new KustoIngestionProperties(csb.InitialCatalog, tableName);
            _fields = typeof(EtwEvent).GetFields().Select(f => f.Name).ToArray();

            if (csb.DataSource.StartsWith("https://ingest-"))
            {
                _ingestClient = KustoIngestFactory.CreateQueuedIngestClient(csb);
            }
            else
            {
                _ingestClient = KustoIngestFactory.CreateDirectIngestClient(csb);
            }

            _nextBatch = new List <T>();
        }
Example #5
0
        public Monitor(Configuration configuration)
        {
            Contract.RequiresNotNull(configuration);
            _configuration = configuration;

            _logger = CreateLogger();

            Contract.RequiresNotNullOrEmpty(_configuration.KustoIngestionClusterUrl);
            Contract.RequiresNotNullOrEmpty(_configuration.ApplicationClientId);
            Contract.RequiresNotNullOrEmpty(_configuration.ApplicationKey);
            Contract.RequiresNotNullOrEmpty(_configuration.Authority);
            var kustoIngestConnectionString = new KustoConnectionStringBuilder(_configuration.KustoIngestionClusterUrl)
                                              .WithAadApplicationKeyAuthentication(_configuration.ApplicationClientId, _configuration.ApplicationKey, _configuration.Authority);

            _kustoIngestClient = KustoIngestFactory.CreateDirectIngestClient(kustoIngestConnectionString);

            var kustoConnectionString = new KustoConnectionStringBuilder(_configuration.KustoClusterUrl)
                                        .WithAadApplicationKeyAuthentication(_configuration.ApplicationClientId, _configuration.ApplicationKey, _configuration.Authority);

            _cslQueryProvider = KustoClientFactory.CreateCslQueryProvider(kustoConnectionString);

            Contract.RequiresNotNull(_configuration.KustoNotifier);
            _alertNotifier = new KustoWriter <Notification>(_configuration.KustoNotifier, _logger, _kustoIngestClient);

            Contract.RequiresNotNull(_configuration.SchedulerKustoNotifier);
            _schedulerLogWriter = new KustoWriter <Scheduler.LogEntry>(_configuration.SchedulerKustoNotifier, _logger, _kustoIngestClient);

            _scheduler = new Scheduler(_configuration.Scheduler, _logger, _clock, _schedulerLogWriter);
        }
Example #6
0
        /// <inheritdoc />
        public IBigBrother UseKusto(string kustoEngineName, string kustoEngineLocation, string kustoDb, string tenantId)
        {
            KustoDbName = kustoDb;
            var kustoUri       = $"https://{kustoEngineName}.{kustoEngineLocation}.kusto.windows.net";
            var kustoIngestUri = $"https://ingest-{kustoEngineName}.{kustoEngineLocation}.kusto.windows.net";
            var token          = new AzureServiceTokenProvider().GetAccessTokenAsync(kustoUri, string.Empty).Result;

            KustoAdminClient = KustoClientFactory.CreateCslAdminProvider(
                new KustoConnectionStringBuilder(kustoUri)
            {
                FederatedSecurity = true,
                InitialCatalog    = KustoDbName,
                AuthorityId       = tenantId,
                ApplicationToken  = token
            });

            KustoIngestClient = KustoIngestFactory.CreateQueuedIngestClient(
                new KustoConnectionStringBuilder(kustoIngestUri)
            {
                FederatedSecurity = true,
                InitialCatalog    = KustoDbName,
                AuthorityId       = tenantId,
                ApplicationToken  = token
            });

            SetupKustoSubscription();
            return(this);
        }
        static void Main(string[] args)
        {
            // Ingest From Local Files using KustoQueuedIngestClient and Ingestion Validation

            // Create Kusto connection string with App Authentication
            var kustoConnectionStringBuilderDM =
                new KustoConnectionStringBuilder(@"https://ingest-{clusterNameAndRegion}.kusto.windows.net").WithAadApplicationKeyAuthentication(
                    applicationClientId: "{Application Client ID}",
                    applicationKey: "{Application Key (secret)}",
                    authority: "{AAD TenantID or name}");

            // Create a disposable client that will execute the ingestion
            IKustoQueuedIngestClient client = KustoIngestFactory.CreateQueuedIngestClient(kustoConnectionStringBuilderDM);

            // Ingest from files according to the required properties
            var kustoIngestionProperties = new KustoIngestionProperties(databaseName: "myDB", tableName: "myTable");

            client.IngestFromStorageAsync(@"ValidTestFile.csv", kustoIngestionProperties);
            client.IngestFromStorageAsync(@"InvalidTestFile.csv", kustoIngestionProperties);

            // Waiting for the aggregation
            Thread.Sleep(TimeSpan.FromMinutes(8));

            // Retrieve and validate failures
            var ingestionFailures = client.PeekTopIngestionFailures().GetAwaiter().GetResult();

            Ensure.IsTrue((ingestionFailures.Count() > 0), "Failures expected");
            // Retrieve, delete and validate failures
            ingestionFailures = client.GetAndDiscardTopIngestionFailures().GetAwaiter().GetResult();
            Ensure.IsTrue((ingestionFailures.Count() > 0), "Failures expected");

            // Dispose of the client
            client.Dispose();
        }
Example #8
0
        public Monitor(Configuration configuration)
        {
            Contract.RequiresNotNull(configuration);
            _configuration = configuration;

            _logger = CreateLogger();

            // TODO(jubayard): use streaming ingestion instead of direct ingestion. There seems to be some assembly
            // issues when attempting to do that
            Contract.RequiresNotNullOrEmpty(_configuration.KustoIngestionClusterUrl);
            Contract.RequiresNotNullOrEmpty(_configuration.ApplicationClientId);
            Contract.RequiresNotNullOrEmpty(_configuration.ApplicationKey);
            Contract.RequiresNotNullOrEmpty(_configuration.Authority);
            var kustoIngestConnectionString = new KustoConnectionStringBuilder(_configuration.KustoIngestionClusterUrl)
                                              .WithAadApplicationKeyAuthentication(_configuration.ApplicationClientId, _configuration.ApplicationKey, _configuration.Authority);

            _kustoIngestClient = KustoIngestFactory.CreateDirectIngestClient(kustoIngestConnectionString);

            var kustoConnectionString = new KustoConnectionStringBuilder(_configuration.KustoClusterUrl)
                                        .WithAadApplicationKeyAuthentication(_configuration.ApplicationClientId, _configuration.ApplicationKey, _configuration.Authority);

            _cslQueryProvider = KustoClientFactory.CreateCslQueryProvider(kustoConnectionString);

            Contract.RequiresNotNull(_configuration.KustoNotifier);
            _alertNotifier = new KustoWriter <Notification>(_configuration.KustoNotifier, _logger, _kustoIngestClient);

            Contract.RequiresNotNull(_configuration.SchedulerKustoNotifier);
            _schedulerLogWriter = new KustoWriter <Scheduler.LogEntry>(_configuration.SchedulerKustoNotifier, _logger, _kustoIngestClient);

            _scheduler = new Scheduler(_configuration.Scheduler, _logger, _clock, _schedulerLogWriter);
        }
 public static IServiceCollection AddKustoIngest(this IServiceCollection services, Action <IOptions <KustoOptions> > configure)
 {
     services.Configure(configure);
     services.AddSingleton <IKustoIngestClient>(provider =>
                                                KustoIngestFactory.CreateQueuedIngestClient(
                                                    provider.GetRequiredService <IOptions <KustoOptions> >().Value.IngestConnectionString));
     return(services);
 }
Example #10
0
        public AdxOutput(
            BaseLogger logger,
            string authority,
            string appclientId,
            string appKey,
            string cluster,
            string database,
            string table,
            bool createOrResetTable = false,
            bool directIngest       = false)
        {
            _logger = logger;

            _table = table;
            _createOrResetTable = createOrResetTable;

            _batchSize       = 10000;
            _flushDuration   = TimeSpan.FromMilliseconds(5);
            _lastUploadTime  = DateTime.UtcNow;
            _initializeTable = false;
            _nextBatch       = new List <IDictionary <string, object> >();

            Completed = new AutoResetEvent(false);

            // Setting up kusto connection
            if (!string.IsNullOrEmpty(authority))
            {
                if (!string.IsNullOrEmpty(appclientId) && !string.IsNullOrEmpty(appKey))
                {
                    kscbIngest = new KustoConnectionStringBuilder($"https://ingest-{cluster}", database).WithAadApplicationKeyAuthentication(appclientId, appKey, authority);
                    kscbAdmin  = new KustoConnectionStringBuilder($"https://{cluster}", database).WithAadApplicationKeyAuthentication(appclientId, appKey, authority);
                }
                else
                {
                    kscbIngest = new KustoConnectionStringBuilder($"https://ingest-{cluster}", database).WithAadUserPromptAuthentication(authority);
                    kscbAdmin  = new KustoConnectionStringBuilder($"https://{cluster}", database).WithAadUserPromptAuthentication(authority);
                }
            }

            if (kscbAdmin != null)
            {
                _ingestionProperties = new KustoIngestionProperties(kscbIngest.InitialCatalog, table);

                if (directIngest)
                {
                    _ingestClient = KustoIngestFactory.CreateDirectIngestClient(this.kscbAdmin);
                }
                else
                {
                    _ingestClient = KustoIngestFactory.CreateQueuedIngestClient(kscbIngest);
                }
            }
            else
            {
                _logger.Log(LogLevel.ERROR, "ERROR getting ADX connection strings. Please double check the information provided.");
                _error = true;
            }
        }
        public static void Run([CosmosDBTrigger(

                                    databaseName: "data",
                                    collectionName: "device",
                                    ConnectionStringSetting = "rbdemocosmosdb_DOCUMENTDB",
                                    LeaseCollectionName = "leases", CreateLeaseCollectionIfNotExists = true)] IReadOnlyList <Document> input, ILogger log)
        {
            if (input != null && input.Count > 0)
            {
                log.LogInformation("Documents modified " + input.Count);
                log.LogInformation("First document Id " + input[0].Id);
                //input[0].ToString() is the data that needs to be ingested.
                var tenantId             = System.Environment.GetEnvironmentVariable($"tenantId");
                var applicationClientId  = System.Environment.GetEnvironmentVariable($"applicationClientId");
                var applicationKey       = System.Environment.GetEnvironmentVariable($"applicationKey");
                var serviceNameAndRegion = System.Environment.GetEnvironmentVariable($"serviceNameAndRegion");
                var kustoDatabase        = "device";
                var kustoTableName       = "Staging";
                var mappingName          = "deviceMapping";

                var kustoIngestConnectionStringBuilder =
                    new KustoConnectionStringBuilder($"https://ingest-{serviceNameAndRegion}.kusto.windows.net")
                {
                    FederatedSecurity   = true,
                    InitialCatalog      = kustoDatabase,
                    ApplicationClientId = applicationClientId,
                    ApplicationKey      = applicationKey,
                    Authority           = tenantId
                };
                using (var ingestClient = KustoIngestFactory.CreateQueuedIngestClient(kustoIngestConnectionStringBuilder))
                {
                    var properties = new KustoQueuedIngestionProperties(kustoDatabase, kustoTableName)
                    {
                        Format = DataSourceFormat.json,
                        JSONMappingReference = mappingName,
                        ReportLevel          = IngestionReportLevel.FailuresAndSuccesses,
                        ReportMethod         = IngestionReportMethod.Queue
                    };
                    string inputValue = input[0].ToString().Replace("\r\n", "");
                    //string inputValue = "{ \"Id\":\"1\", \"Timestamp\":\"2\", \"Message\":\"This is a test message\" }";
                    //object val = Newtonsoft.Json.JsonConvert.DeserializeObject(inputValue);
                    var stream = new System.IO.MemoryStream(System.Text.Encoding.UTF8.GetBytes(inputValue));
                    stream.Seek(0, SeekOrigin.Begin);
                    // Post ingestion message
                    ingestClient.IngestFromStream(stream, properties, leaveOpen: true);

                    // Wait and retrieve all notifications
                    //  - Actual duration should be decided based on the effective Ingestion Batching Policy set on the table/database
                    Thread.Sleep(10000);
                    var errors    = ingestClient.GetAndDiscardTopIngestionFailures().GetAwaiter().GetResult();
                    var successes = ingestClient.GetAndDiscardTopIngestionSuccesses().GetAwaiter().GetResult();

                    errors.ForEach((f) => { Console.WriteLine($"Ingestion error: {f.Info.Details}"); });
                    successes.ForEach((s) => { Console.WriteLine($"Ingested: {s.Info.IngestionSourcePath}"); });
                }
            }
        }
Example #12
0
        static async Task Main(string[] args)
        {
            // Ingest From a Local File using KustoQueuedIngestClient and report status to a table

            // Create Kusto connection string with App Authentication
            var kustoConnectionStringBuilderDM =
                new KustoConnectionStringBuilder(@"https://ingest-{clusterNameAndRegion}.kusto.windows.net").WithAadApplicationKeyAuthentication(
                    applicationClientId: "{Application Client ID}",
                    applicationKey: "{Application Key (secret)}",
                    authority: "{AAD TenantID or name}");

            // Create a disposable client that will execute the ingestion
            IKustoQueuedIngestClient client = KustoIngestFactory.CreateQueuedIngestClient(kustoConnectionStringBuilderDM);

            // Ingest from a file according to the required properties
            var kustoIngestionProperties = new KustoQueuedIngestionProperties(databaseName: "myDB", tableName: "myDB")
            {
                // Setting the report level to FailuresAndSuccesses will cause both successful and failed ingestions to be reported
                // (Rather than the default "FailuresOnly" level)
                ReportLevel = IngestionReportLevel.FailuresAndSuccesses,
                // Choose the report method of choice
                ReportMethod = IngestionReportMethod.Table
            };

            var filePath        = @"< Path to file >";
            var fileIdentifier  = Guid.NewGuid();
            var fileDescription = new FileDescription()
            {
                FilePath = filePath, SourceId = fileIdentifier
            };
            var sourceOptions = new StorageSourceOptions()
            {
                SourceId = fileDescription.SourceId.Value
            };

            // Execute the ingest operation and save the result.
            var clientResult = await client.IngestFromStorageAsync(fileDescription.FilePath,
                                                                   ingestionProperties : kustoIngestionProperties, sourceOptions);

            // Use the fileIdentifier you supplied to get the status of your ingestion
            var ingestionStatus = clientResult.GetIngestionStatusBySourceId(fileIdentifier);

            while (ingestionStatus.Status == Status.Pending)
            {
                // Wait a minute...
                Thread.Sleep(TimeSpan.FromMinutes(1));
                // Try again
                ingestionStatus = clientResult.GetIngestionStatusBySourceId(fileIdentifier);
            }

            // Verify the results of the ingestion
            Ensure.ConditionIsMet(ingestionStatus.Status == Status.Succeeded,
                                  "The file should have been ingested successfully");

            // Dispose of the client
            client.Dispose();
        }
Example #13
0
        protected FI(ILogger iLog, Microsoft.Azure.Management.Fluent.IAzure iAzure, string iRGName, string kustoConn, string kustoDBName, string kustoTableName)
        {
            curRGName  = iRGName;
            log        = iLog;
            curSubName = iAzure.SubscriptionId;

            ingestClient       = KustoIngestFactory.CreateQueuedIngestClient(kustoConn);
            ingestProps        = new KustoIngestionProperties(kustoDBName, kustoTableName);
            ingestProps.Format = Kusto.Data.Common.DataSourceFormat.csv;
        }
Example #14
0
        public IKustoIngestClient GetClient()
        {
            string ingestConnectionString = _kustoOptions.Value.IngestConnectionString;

            if (string.IsNullOrWhiteSpace(ingestConnectionString))
            {
                throw new InvalidCastException($"Kusto {nameof(_kustoOptions.Value.IngestConnectionString)} is not configured in settings or related KeyVault");
            }

            return(_clients.GetOrAdd(ingestConnectionString, _ => KustoIngestFactory.CreateQueuedIngestClient(ingestConnectionString)));
        }
Example #15
0
        public static Result <IKustoIngestClient> CreateKustoIngestClient(string kustoIngestionClusterUrl, string azureTenantId, string azureAppId, string azureAppKey)
        {
            Contract.RequiresNotNullOrEmpty(kustoIngestionClusterUrl);
            Contract.RequiresNotNullOrEmpty(azureTenantId);
            Contract.RequiresNotNullOrEmpty(azureAppId);
            Contract.RequiresNotNullOrEmpty(azureAppKey);

            var kustoIngestConnectionString = new KustoConnectionStringBuilder(kustoIngestionClusterUrl)
                                              .WithAadApplicationKeyAuthentication(azureAppId, azureAppKey, azureTenantId);

            return(new Result <IKustoIngestClient>(KustoIngestFactory.CreateDirectIngestClient(kustoIngestConnectionString)));
        }
Example #16
0
        /// <summary>
        /// Performs a write to the indicated table, sending the appropriately filtered test report to Kusto
        /// Returns the number of lines written
        /// Skips the final write step when testing
        /// </summary>
        public bool WriteDataForTable(bool testing, List <AdoBuildResultRow> reports)
        {
            try
            {
                // Create Kusto connection string with App Authentication
                var kustoConnectionStringBuilderEngine =
                    new KustoConnectionStringBuilder(config.clusterUri).WithAadApplicationKeyAuthentication(
                        applicationClientId: config.clientId,
                        applicationKey: config.secretKey,
                        authority: config.authority);

                using (var ingestClient = KustoIngestFactory.CreateDirectIngestClient(kustoConnectionStringBuilderEngine))
                {
                    var ingestProps = new KustoQueuedIngestionProperties(config.dbName, config.tableName);
                    // For the sake of getting both failure and success notifications we set this to IngestionReportLevel.FailuresAndSuccesses
                    // Usually the recommended level is IngestionReportLevel.FailuresOnly
                    ingestProps.ReportLevel      = IngestionReportLevel.FailuresAndSuccesses;
                    ingestProps.ReportMethod     = IngestionReportMethod.Queue;
                    ingestProps.IngestionMapping = new IngestionMapping {
                        IngestionMappingReference = config.mappingName
                    };
                    ingestProps.Format = DataSourceFormat.json;

                    // Prepare data for ingestion
                    using (var memStream = new MemoryStream())
                        using (var writer = new StreamWriter(memStream))
                        {
                            foreach (AdoBuildResultRow eachReport in reports)
                            {
                                writer.WriteLine(JsonConvert.SerializeObject(eachReport));
                                Console.WriteLine($"Writing: {eachReport.jobname}, ({eachReport.passed}/{eachReport.failed}/{eachReport.other}/{eachReport.total}) for {eachReport.timestamp}");
                            }

                            writer.Flush();
                            memStream.Seek(0, SeekOrigin.Begin);

                            // Post ingestion message
                            if (!testing)
                            {
                                ingestClient.IngestFromStream(memStream, ingestProps, leaveOpen: true);
                            }
                        }
                }
                return(true);
            }
            catch (Exception e)
            {
                Console.WriteLine($"ERROR: Could not write to: {config.clusterUri}.{config.dbName}.{config.tableName}, using Mapping: {config.mappingName}");
                Console.WriteLine(e.ToString());
                return(false);
            }
        }
Example #17
0
        public async Task InsertRowAsync(T row, CancellationToken cancellationToken = default)
        {
            using IKustoQueuedIngestClient ingestClient = KustoIngestFactory.CreateQueuedIngestClient(_connectionStringBuilder);

            string serializedData = JsonConvert.SerializeObject(row);

            byte[] serializedBytes = Encoding.UTF8.GetBytes(serializedData);

            using MemoryStream dataStream = new MemoryStream(serializedBytes);

            // IKustoQueuedIngestClient doesn't support cancellation at the moment. Update the line below if it does in the future.
            await ingestClient.IngestFromStreamAsync(dataStream, _ingestionProperties, leaveOpen : true);
        }
Example #18
0
        private static bool TryInitialize(ILogger log)
        {
            lock (_lock)
            {
                if (!_initialized)
                {
                    string kustoIngestUrl = GetEnvVariable("KustoIngestUrl");
                    string clientId       = GetEnvVariable("ClientId");
                    string clientSecret   = GetEnvVariable("ClientSecret");
                    string tenantId       = GetEnvVariable("TenantId");

                    if (String.IsNullOrWhiteSpace(kustoIngestUrl) ||
                        String.IsNullOrWhiteSpace(clientId) ||
                        String.IsNullOrWhiteSpace(clientSecret) ||
                        String.IsNullOrWhiteSpace(tenantId))
                    {
                        log.LogError($"Could not initialize the Kusto client because the connection parameters are wrong (url: {kustoIngestUrl} clientId: {clientId}, tenant: {tenantId}");

                        return(false);
                    }

                    //Initialize adx
                    connection =
                        new KustoConnectionStringBuilder(GetEnvVariable("KustoIngestUrl")).WithAadApplicationKeyAuthentication(
                            applicationClientId: GetEnvVariable("ClientId"),
                            applicationKey: GetEnvVariable("ClientSecret"),
                            authority: GetEnvVariable("TenantId"));

                    adx = KustoIngestFactory.CreateQueuedIngestClient(connection);

                    if (adx != null)
                    {
                        _initialized = true;
                        log.LogInformation("Function successfully initialized");

                        return(true);
                    }
                    else
                    {
                        log.LogWarning("Function not successfully initialized");
                    }
                }
                else
                {
                    return(true);
                }
            }

            return(false);
        }
Example #19
0
        internal IBigBrother UseKustoInternal(string kustoEngineName, string kustoEngineLocation, string kustoDb, string tenantId)
        {
            try
            {
                KustoDbName = kustoDb;
                var kustoUri = $"https://{kustoEngineName}.{kustoEngineLocation}.kusto.windows.net";
                var token    = new AzureServiceTokenProvider().GetAccessTokenAsync(kustoUri, string.Empty).Result;

                KustoAdminClient = KustoClientFactory.CreateCslAdminProvider(
                    new KustoConnectionStringBuilder(kustoUri)
                {
                    FederatedSecurity = true,
                    InitialCatalog    = KustoDbName,
                    Authority         = tenantId,
                    ApplicationToken  = token
                });

                var kustoQueuedIngestUri = $"https://ingest-{kustoEngineName}.{kustoEngineLocation}.kusto.windows.net";
                var kustoIngestUri       = $"https://{kustoEngineName}.{kustoEngineLocation}.kusto.windows.net";

                KustoQueuedIngestClient = KustoIngestFactory.CreateQueuedIngestClient(
                    new KustoConnectionStringBuilder(kustoQueuedIngestUri)
                {
                    FederatedSecurity = true,
                    InitialCatalog    = KustoDbName,
                    Authority         = tenantId,
                    ApplicationToken  = token
                });

                KustoDirectIngestClient = KustoIngestFactory.CreateDirectIngestClient(
                    new KustoConnectionStringBuilder(kustoIngestUri)
                {
                    FederatedSecurity = true,
                    InitialCatalog    = KustoDbName,
                    Authority         = tenantId,
                    ApplicationToken  = token
                });

                SetupKustoSubscription();

                KustoIngestionTimeMetric = InternalClient.GetMetric(new MetricIdentifier("", "KustoIngestionTime"));
            }
            catch (Exception e)
            {
                InternalStream.OnNext(e.ToExceptionEvent());
            }

            return(this);
        }
        static void Main(string[] args)
        {
            // Ingest From a Local Files using KustoQueuedIngestClient and report status to a queue

            // Create Kusto connection string with App Authentication
            var kustoConnectionStringBuilderDM =
                new KustoConnectionStringBuilder(@"https://ingest-{clusterNameAndRegion}.kusto.windows.net").WithAadApplicationKeyAuthentication(
                    applicationClientId: "{Application Client ID}",
                    applicationKey: "{Application Key (secret)}",
                    authority: "{AAD TenantID or name}");

            // Create a disposable client that will execute the ingestion
            IKustoQueuedIngestClient client = KustoIngestFactory.CreateQueuedIngestClient(kustoConnectionStringBuilderDM);

            // Ingest from a file according to the required properties
            var kustoIngestionProperties = new KustoQueuedIngestionProperties(databaseName: "myDB", tableName: "myTable")
            {
                // Setting the report level to FailuresAndSuccesses will cause both successful and failed ingestions to be reported
                // (Rather than the default "FailuresOnly" level - which is demonstrated in the
                // 'Ingest From Local File(s) using KustoQueuedIngestClient and Ingestion Validation' section)
                ReportLevel = IngestionReportLevel.FailuresAndSuccesses,
                // Choose the report method of choice. 'Queue' is the default method.
                // For the sake of the example, we will choose it anyway.
                ReportMethod = IngestionReportMethod.Queue
            };

            client.IngestFromStorageAsync("ValidTestFile.csv", kustoIngestionProperties);
            client.IngestFromStorageAsync("InvalidTestFile.csv", kustoIngestionProperties);

            // Waiting for the aggregation
            Thread.Sleep(TimeSpan.FromMinutes(8));

            // Retrieve and validate failures
            var ingestionFailures = client.PeekTopIngestionFailures().GetAwaiter().GetResult();

            Ensure.IsTrue((ingestionFailures.Count() > 0), "The failed ingestion should have been reported to the failed ingestions queue");
            // Retrieve, delete and validate failures
            ingestionFailures = client.GetAndDiscardTopIngestionFailures().GetAwaiter().GetResult();
            Ensure.IsTrue((ingestionFailures.Count() > 0), "The failed ingestion should have been reported to the failed ingestions queue");

            // Verify the success has also been reported to the queue
            var ingestionSuccesses = client.GetAndDiscardTopIngestionSuccesses().GetAwaiter().GetResult();

            Ensure.ConditionIsMet((ingestionSuccesses.Count() > 0),
                                  "The successful ingestion should have been reported to the successful ingestions queue");

            // Dispose of the client
            client.Dispose();
        }
Example #21
0
        public async Task IngestFromCsvAsync(
            string csv, ServicePrincipalOptions servicePrincipal, string cluster, string database, string table, bool isDryRun)
        {
            KustoConnectionStringBuilder connectionBuilder =
                new KustoConnectionStringBuilder($"https://{cluster}.kusto.windows.net")
                .WithAadApplicationKeyAuthentication(
                    servicePrincipal.ClientId,
                    servicePrincipal.Secret,
                    servicePrincipal.Tenant);

            using (IKustoIngestClient client = KustoIngestFactory.CreateDirectIngestClient(connectionBuilder))
            {
                KustoIngestionProperties properties =
                    new(database, table) { Format = DataSourceFormat.csv };
                StreamSourceOptions sourceOptions = new() { SourceId = Guid.NewGuid() };

                if (!isDryRun)
                {
                    AsyncRetryPolicy retryPolicy = Policy
                                                   .Handle <Kusto.Data.Exceptions.KustoException>()
                                                   .Or <Kusto.Ingest.Exceptions.KustoException>()
                                                   .WaitAndRetryAsync(
                        Backoff.DecorrelatedJitterBackoffV2(TimeSpan.FromSeconds(10), RetryHelper.MaxRetries),
                        RetryHelper.GetOnRetryDelegate(RetryHelper.MaxRetries, _loggerService));

                    IKustoIngestionResult result = await retryPolicy.ExecuteAsync(
                        () => IngestFromStreamAsync(csv, client, properties, sourceOptions));

                    IngestionStatus ingestionStatus = result.GetIngestionStatusBySourceId(sourceOptions.SourceId);
                    for (int i = 0; i < 10 && ingestionStatus.Status == Status.Pending; i++)
                    {
                        await Task.Delay(TimeSpan.FromSeconds(30));

                        ingestionStatus = result.GetIngestionStatusBySourceId(sourceOptions.SourceId);
                    }

                    if (ingestionStatus.Status == Status.Pending)
                    {
                        throw new InvalidOperationException($"Timeout while ingesting Kusto data.");
                    }
                    else if (ingestionStatus.Status != Status.Succeeded)
                    {
                        throw new InvalidOperationException(
                                  $"Failed to ingest Kusto data.{Environment.NewLine}{ingestionStatus.Details}");
                    }
                }
            }
        }
Example #22
0
        /// <summary>
        /// Ingest data into Kusto.
        /// </summary>
        /// <param name="table">Name of table to ingest into.</param>
        /// <param name="mappingName">Name of table mapping to ingest with.</param>
        /// <param name="stream">input JSON data stream.</param>
        /// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
        private static async Task <IKustoIngestionResult> KustoIngest(KustoConnectionStringBuilder kusto, string db, string table, string mappingName, Stream stream)
        {
            // Create a disposable client that will execute the ingestion
            using IKustoIngestClient client = KustoIngestFactory.CreateDirectIngestClient(kusto);
            var ingestProps = new KustoIngestionProperties(db, table)
            {
                JSONMappingReference = mappingName,
                Format = DataSourceFormat.json,
            };
            var ssOptions = new StreamSourceOptions
            {
                CompressionType = DataSourceCompressionType.GZip,
            };

            return(await client.IngestFromStreamAsync(stream, ingestProps, ssOptions));
        }
Example #23
0
        public BlockingKustoUploader(
            string blobConnectionString,
            string blobContainerName,
            KustoConnectionStringBuilder adminCsb,
            KustoConnectionStringBuilder kscb,
            bool demoMode,
            string tableName,
            int batchSize,
            TimeSpan flushDuration,
            bool resetTable = false)
        {
            Csb              = kscb;
            AdminCsb         = adminCsb;
            TableName        = tableName;
            BatchSize        = batchSize;
            _flushDuration   = flushDuration;
            _lastUploadTime  = DateTime.UtcNow;
            _resetTable      = resetTable;
            _initializeTable = false;

            Completed = new AutoResetEvent(false);

            if (Csb != null)
            {
                _ingestionProperties = new KustoIngestionProperties(Csb.InitialCatalog, tableName);

                if (demoMode)
                {
                    _ingestClient = KustoIngestFactory.CreateDirectIngestClient(this.AdminCsb);
                }
                else
                {
                    _ingestClient = KustoIngestFactory.CreateQueuedIngestClient(Csb);
                }
            }

            if (!string.IsNullOrEmpty(blobConnectionString) &&
                !string.IsNullOrEmpty(blobContainerName))
            {
                blobServiceClient   = new BlobServiceClient(blobConnectionString);
                blobContainerClient = blobServiceClient.GetBlobContainerClient(blobContainerName);
                blobContainerClient.CreateIfNotExists();
            }

            _nextBatch = new List <IDictionary <string, object> >();
        }
        public KustoTimelineTelemetryRepository(ILogger <KustoTimelineTelemetryRepository> logger, IOptionsSnapshot <KustoTimelineTelemetryOptions> options)
        {
            _logger = logger;

            // removing the IngestConnectionString was a default setup in local debugging
            if (string.IsNullOrEmpty(options.Value.IngestConnectionString))
            {
                _logger.LogDebug("No ingest connection string provided; will ignore ingest operations");
                _ingest = new NullKustoIngestClient();
            }
            else
            {
                _ingest = KustoIngestFactory.CreateQueuedIngestClient(options.Value.IngestConnectionString);
            }
            _query    = KustoClientFactory.CreateCslQueryProvider(options.Value.QueryConnectionString);
            _database = options.Value.Database;
        }
Example #25
0
        public async Task PublishManualReviews(IEnumerable <SloManualReview> results)
        {
            var dt = results.ToDataTable();

            var kustoConnectionStringBuilderEngine = new KustoConnectionStringBuilder(kustoUrl_).WithAadUserPromptAuthentication();

            using (var client = KustoIngestFactory.CreateDirectIngestClient(kustoConnectionStringBuilderEngine))
            {
                //Ingest from blobs according to the required properties
                var kustoIngestionProperties = new KustoIngestionProperties(
                    databaseName: kustoDb_,
                    tableName: kustoManualReviewTable_
                    );

                var reader = dt.CreateDataReader();
                var result = await client.IngestFromDataReaderAsync(reader, kustoIngestionProperties);
            }
        }
        public BlockingKustoUploader(
            string outputFileName,
            KustoConnectionStringBuilder adminCsb,
            KustoConnectionStringBuilder kscb,
            bool demoMode,
            string tableName,
            int batchSize,
            TimeSpan flushDuration,
            bool resetTable = false)
        {
            OutputFileName   = outputFileName;
            Csb              = kscb;
            AdminCsb         = adminCsb;
            TableName        = tableName;
            BatchSize        = batchSize;
            _flushDuration   = flushDuration;
            _lastUploadTime  = DateTime.UtcNow;
            _resetTable      = resetTable;
            _initializeTable = false;

            Completed = new AutoResetEvent(false);

            if (Csb != null)
            {
                _ingestionProperties = new KustoIngestionProperties(Csb.InitialCatalog, tableName);

                if (demoMode)
                {
                    _ingestClient = KustoIngestFactory.CreateDirectIngestClient(this.AdminCsb);
                }
                else
                {
                    _ingestClient = KustoIngestFactory.CreateQueuedIngestClient(Csb);
                }
            }

            if (!string.IsNullOrEmpty(OutputFileName))
            {
                outputFile = new StreamWriter(this.OutputFileName);
                outputFile.Write($"[{Environment.NewLine}");
            }

            _nextBatch = new List <IDictionary <string, object> >();
        }
Example #27
0
        public async Task IngestFromCsvStreamAsync(Stream csv, IngestKustoImageInfoOptions options)
        {
            KustoConnectionStringBuilder connectionBuilder =
                new KustoConnectionStringBuilder($"https://{options.Cluster}.kusto.windows.net")
                .WithAadApplicationKeyAuthentication(
                    options.ServicePrincipal.ClientId,
                    options.ServicePrincipal.Secret,
                    options.ServicePrincipal.Tenant);

            using (IKustoIngestClient client = KustoIngestFactory.CreateDirectIngestClient(connectionBuilder))
            {
                KustoIngestionProperties properties =
                    new KustoIngestionProperties(options.Database, options.Table)
                {
                    Format = DataSourceFormat.csv
                };
                StreamSourceOptions sourceOptions = new StreamSourceOptions {
                    SourceId = Guid.NewGuid()
                };

                if (!options.IsDryRun)
                {
                    IKustoIngestionResult result = await client.IngestFromStreamAsync(csv, properties, sourceOptions);

                    IngestionStatus ingestionStatus = result.GetIngestionStatusBySourceId(sourceOptions.SourceId);
                    for (int i = 0; i < 10 && ingestionStatus.Status == Status.Pending; i++)
                    {
                        await Task.Delay(TimeSpan.FromSeconds(30));

                        ingestionStatus = result.GetIngestionStatusBySourceId(sourceOptions.SourceId);
                    }

                    if (ingestionStatus.Status != Status.Succeeded)
                    {
                        throw new InvalidOperationException(
                                  $"Failed to ingest Kusto data.{Environment.NewLine}{ingestionStatus.Details}");
                    }
                    else if (ingestionStatus.Status == Status.Pending)
                    {
                        throw new InvalidOperationException($"Timeout while ingesting Kusto data.");
                    }
                }
            }
        }
        static void Main(string[] args)
        {
            // Ingest From Local File using KustoDirectIngestClient (only for test purposes):

            // Create Kusto connection string with App Authentication
            var kustoConnectionStringBuilderEngine =
                new KustoConnectionStringBuilder(@"https://{clusterNameAndRegion}.kusto.windows.net").WithAadApplicationKeyAuthentication(
                    applicationClientId: "{Application Client ID}",
                    applicationKey: "{Application Key (secret)}",
                    authority: "{AAD TenantID or name}");

            // Create a disposable client that will execute the ingestion
            using (IKustoIngestClient client = KustoIngestFactory.CreateDirectIngestClient(kustoConnectionStringBuilderEngine))
            {
                //Ingest from blobs according to the required properties
                var kustoIngestionProperties = new KustoIngestionProperties(databaseName: "myDB", tableName: "myTable");

                client.IngestFromStorageAsync(@"< Path to local file >", ingestionProperties: kustoIngestionProperties).GetAwaiter().GetResult();
            }
        }
 private static async Task IngestTriageItemsIntoKusto(TriageItem[] triageItems, ILogger log)
 {
     log.LogInformation("Entering IngestTriageItemIntoKusto");
     string             kustoIngestConnectionString = System.Environment.GetEnvironmentVariable("KustoIngestConnectionString");
     string             databaseName = System.Environment.GetEnvironmentVariable("KustoDatabaseName");
     IKustoIngestClient ingestClient = KustoIngestFactory.CreateQueuedIngestClient(kustoIngestConnectionString);
     await KustoHelpers.WriteDataToKustoInMemoryAsync(
         ingestClient,
         databaseName,
         "TimelineIssuesTriage",
         log,
         triageItems,
         b => new[]
     {
         new KustoValue("ModifiedDateTime", b.ModifiedDateTime.ToString(), KustoDataTypes.DateTime),
         new KustoValue("BuildId", b.BuildId.ToString(), KustoDataTypes.Int),
         new KustoValue("RecordId", b.RecordId.ToString(), KustoDataTypes.Guid),
         new KustoValue("Index", b.Index.ToString(), KustoDataTypes.Int),
         new KustoValue("UpdatedCategory", b?.UpdatedCategory, KustoDataTypes.String),
         new KustoValue("Url", b?.Url, KustoDataTypes.String)
     });
 }
Example #30
0
        private IKustoIngestClient GetKustoIngestClient(string appId, string appKey, EventSinkInfo eventSinkInfo)
        {
            IKustoIngestClient _client;

            try
            {
                var kcsb = new KustoConnectionStringBuilder(eventSinkInfo.Settings.EndpointUrl)
                {
                    InitialCatalog    = eventSinkInfo.Settings.Database,
                    FederatedSecurity = true
                };

                kcsb    = kcsb.WithAadApplicationKeyAuthentication(appId, appKey, eventSinkInfo.Settings.Authority);
                _client = KustoIngestFactory.CreateQueuedIngestClient(kcsb);
            }
            catch (Exception e)
            {
                Console.WriteLine(e);
                throw;
            }

            return(_client);
        }