public KustoClientLogger(string databaseName, string tableName, List <KustoColumnMapping> _schema, string clientId, string clientKey) { var csvColumnMappings = new List <CsvColumnMapping>(); foreach (var mapping in _schema) { csvColumnMappings.Add(new CsvColumnMapping { ColumnName = mapping.ColumnName, CslDataType = mapping.DataType, Ordinal = mapping.ColumnNumber }); } _kustoProperties = new KustoIngestionProperties(databaseName, tableName) { Format = DataSourceFormat.csv, CSVMapping = csvColumnMappings }; var kustoConnectionStringBuilderDM = new KustoConnectionStringBuilder(@"https://wawstest.kusto.windows.net:443") { FederatedSecurity = true, InitialCatalog = databaseName, ApplicationKey = clientKey, ApplicationClientId = clientId }; _kustoClient = KustoIngestFactory.CreateDirectIngestClient(kustoConnectionStringBuilderDM); }
/// <summary> /// Constructor. Initializes this object and does nothing else. /// </summary> /// <param name="connectionString">Kusto connection string.</param> /// <param name="database">Database into which to ingest.</param> /// <param name="table">Table into which to ingest.</param> /// <param name="csvMapping">Csv mapping that applies to all CSV files to be uploaded via <see cref="UploadSingleCsvFile"/></param> /// <param name="deleteFilesOnSuccess">Whether to delete files upon successful upload.</param> /// <param name="checkForIngestionErrors"> /// Whether to check for ingestion errors before disposing this object. /// Note that at this time not all uploaded files have necessarily been ingested; this class /// does not wait for ingestions to complete, it only checks for failures of those that have completed. /// </param> /// <param name="log">Optional log to which to write some debug information.</param> public KustoUploader ( string connectionString, string database, string table, IEnumerable <CsvColumnMapping> csvMapping, bool deleteFilesOnSuccess, bool checkForIngestionErrors, ILog log = null ) { _log = log; _deleteFilesOnSuccess = deleteFilesOnSuccess; _checkForIngestionErrors = checkForIngestionErrors; _client = KustoIngestFactory.CreateQueuedIngestClient(connectionString); _hasUploadErrors = false; _ingestionProperties = new KustoQueuedIngestionProperties(database, table) { CSVMapping = csvMapping, ReportLevel = IngestionReportLevel.FailuresOnly, ReportMethod = IngestionReportMethod.Queue, }; _block = new ActionBlock <FileDescription> ( UploadSingleCsvFile, new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = 1, BoundedCapacity = DataflowBlockOptions.Unbounded } ); }
static async Task Main(string[] args) { // Async Ingestion From a Single Azure Blob using KustoQueuedIngestClient with (optional) RetryPolicy: //Create Kusto connection string with App Authentication var kustoConnectionStringBuilderDM = new KustoConnectionStringBuilder(@"https://ingest-{clusterNameAndRegion}.kusto.windows.net").WithAadApplicationKeyAuthentication( applicationClientId: "{Application Client ID}", applicationKey: "{Application Key (secret)}", authority: "{AAD TenantID or name}"); // Create an ingest client // Note, that creating a separate instance per ingestion operation is an anti-pattern. // IngestClient classes are thread-safe and intended for reuse IKustoIngestClient client = KustoIngestFactory.CreateQueuedIngestClient(kustoConnectionStringBuilderDM); // Ingest from blobs according to the required properties var kustoIngestionProperties = new KustoIngestionProperties(databaseName: "myDB", tableName: "myTable"); var sourceOptions = new StorageSourceOptions() { DeleteSourceOnSuccess = true }; //// Create your custom implementation of IRetryPolicy, which will affect how the ingest client handles retrying on transient failures IRetryPolicy retryPolicy = new NoRetry(); //// This line sets the retry policy on the ingest client that will be enforced on every ingest call from here on ((IKustoQueuedIngestClient)client).QueueRetryPolicy = retryPolicy; await client.IngestFromStorageAsync(uri : @"BLOB-URI-WITH-SAS-KEY", ingestionProperties : kustoIngestionProperties, sourceOptions); client.Dispose(); }
public BlockingKustoUploader( KustoConnectionStringBuilder kscb, string tableName, int batchSize, TimeSpan flushDuration) { csb = kscb; TableName = TableName; BatchSize = batchSize; _flushDuration = flushDuration; _lastUploadTime = DateTime.Now; Completed = new AutoResetEvent(false); _ingestionProperties = new KustoIngestionProperties(csb.InitialCatalog, tableName); _fields = typeof(EtwEvent).GetFields().Select(f => f.Name).ToArray(); if (csb.DataSource.StartsWith("https://ingest-")) { _ingestClient = KustoIngestFactory.CreateQueuedIngestClient(csb); } else { _ingestClient = KustoIngestFactory.CreateDirectIngestClient(csb); } _nextBatch = new List <T>(); }
public Monitor(Configuration configuration) { Contract.RequiresNotNull(configuration); _configuration = configuration; _logger = CreateLogger(); Contract.RequiresNotNullOrEmpty(_configuration.KustoIngestionClusterUrl); Contract.RequiresNotNullOrEmpty(_configuration.ApplicationClientId); Contract.RequiresNotNullOrEmpty(_configuration.ApplicationKey); Contract.RequiresNotNullOrEmpty(_configuration.Authority); var kustoIngestConnectionString = new KustoConnectionStringBuilder(_configuration.KustoIngestionClusterUrl) .WithAadApplicationKeyAuthentication(_configuration.ApplicationClientId, _configuration.ApplicationKey, _configuration.Authority); _kustoIngestClient = KustoIngestFactory.CreateDirectIngestClient(kustoIngestConnectionString); var kustoConnectionString = new KustoConnectionStringBuilder(_configuration.KustoClusterUrl) .WithAadApplicationKeyAuthentication(_configuration.ApplicationClientId, _configuration.ApplicationKey, _configuration.Authority); _cslQueryProvider = KustoClientFactory.CreateCslQueryProvider(kustoConnectionString); Contract.RequiresNotNull(_configuration.KustoNotifier); _alertNotifier = new KustoWriter <Notification>(_configuration.KustoNotifier, _logger, _kustoIngestClient); Contract.RequiresNotNull(_configuration.SchedulerKustoNotifier); _schedulerLogWriter = new KustoWriter <Scheduler.LogEntry>(_configuration.SchedulerKustoNotifier, _logger, _kustoIngestClient); _scheduler = new Scheduler(_configuration.Scheduler, _logger, _clock, _schedulerLogWriter); }
/// <inheritdoc /> public IBigBrother UseKusto(string kustoEngineName, string kustoEngineLocation, string kustoDb, string tenantId) { KustoDbName = kustoDb; var kustoUri = $"https://{kustoEngineName}.{kustoEngineLocation}.kusto.windows.net"; var kustoIngestUri = $"https://ingest-{kustoEngineName}.{kustoEngineLocation}.kusto.windows.net"; var token = new AzureServiceTokenProvider().GetAccessTokenAsync(kustoUri, string.Empty).Result; KustoAdminClient = KustoClientFactory.CreateCslAdminProvider( new KustoConnectionStringBuilder(kustoUri) { FederatedSecurity = true, InitialCatalog = KustoDbName, AuthorityId = tenantId, ApplicationToken = token }); KustoIngestClient = KustoIngestFactory.CreateQueuedIngestClient( new KustoConnectionStringBuilder(kustoIngestUri) { FederatedSecurity = true, InitialCatalog = KustoDbName, AuthorityId = tenantId, ApplicationToken = token }); SetupKustoSubscription(); return(this); }
static void Main(string[] args) { // Ingest From Local Files using KustoQueuedIngestClient and Ingestion Validation // Create Kusto connection string with App Authentication var kustoConnectionStringBuilderDM = new KustoConnectionStringBuilder(@"https://ingest-{clusterNameAndRegion}.kusto.windows.net").WithAadApplicationKeyAuthentication( applicationClientId: "{Application Client ID}", applicationKey: "{Application Key (secret)}", authority: "{AAD TenantID or name}"); // Create a disposable client that will execute the ingestion IKustoQueuedIngestClient client = KustoIngestFactory.CreateQueuedIngestClient(kustoConnectionStringBuilderDM); // Ingest from files according to the required properties var kustoIngestionProperties = new KustoIngestionProperties(databaseName: "myDB", tableName: "myTable"); client.IngestFromStorageAsync(@"ValidTestFile.csv", kustoIngestionProperties); client.IngestFromStorageAsync(@"InvalidTestFile.csv", kustoIngestionProperties); // Waiting for the aggregation Thread.Sleep(TimeSpan.FromMinutes(8)); // Retrieve and validate failures var ingestionFailures = client.PeekTopIngestionFailures().GetAwaiter().GetResult(); Ensure.IsTrue((ingestionFailures.Count() > 0), "Failures expected"); // Retrieve, delete and validate failures ingestionFailures = client.GetAndDiscardTopIngestionFailures().GetAwaiter().GetResult(); Ensure.IsTrue((ingestionFailures.Count() > 0), "Failures expected"); // Dispose of the client client.Dispose(); }
public Monitor(Configuration configuration) { Contract.RequiresNotNull(configuration); _configuration = configuration; _logger = CreateLogger(); // TODO(jubayard): use streaming ingestion instead of direct ingestion. There seems to be some assembly // issues when attempting to do that Contract.RequiresNotNullOrEmpty(_configuration.KustoIngestionClusterUrl); Contract.RequiresNotNullOrEmpty(_configuration.ApplicationClientId); Contract.RequiresNotNullOrEmpty(_configuration.ApplicationKey); Contract.RequiresNotNullOrEmpty(_configuration.Authority); var kustoIngestConnectionString = new KustoConnectionStringBuilder(_configuration.KustoIngestionClusterUrl) .WithAadApplicationKeyAuthentication(_configuration.ApplicationClientId, _configuration.ApplicationKey, _configuration.Authority); _kustoIngestClient = KustoIngestFactory.CreateDirectIngestClient(kustoIngestConnectionString); var kustoConnectionString = new KustoConnectionStringBuilder(_configuration.KustoClusterUrl) .WithAadApplicationKeyAuthentication(_configuration.ApplicationClientId, _configuration.ApplicationKey, _configuration.Authority); _cslQueryProvider = KustoClientFactory.CreateCslQueryProvider(kustoConnectionString); Contract.RequiresNotNull(_configuration.KustoNotifier); _alertNotifier = new KustoWriter <Notification>(_configuration.KustoNotifier, _logger, _kustoIngestClient); Contract.RequiresNotNull(_configuration.SchedulerKustoNotifier); _schedulerLogWriter = new KustoWriter <Scheduler.LogEntry>(_configuration.SchedulerKustoNotifier, _logger, _kustoIngestClient); _scheduler = new Scheduler(_configuration.Scheduler, _logger, _clock, _schedulerLogWriter); }
public static IServiceCollection AddKustoIngest(this IServiceCollection services, Action <IOptions <KustoOptions> > configure) { services.Configure(configure); services.AddSingleton <IKustoIngestClient>(provider => KustoIngestFactory.CreateQueuedIngestClient( provider.GetRequiredService <IOptions <KustoOptions> >().Value.IngestConnectionString)); return(services); }
public AdxOutput( BaseLogger logger, string authority, string appclientId, string appKey, string cluster, string database, string table, bool createOrResetTable = false, bool directIngest = false) { _logger = logger; _table = table; _createOrResetTable = createOrResetTable; _batchSize = 10000; _flushDuration = TimeSpan.FromMilliseconds(5); _lastUploadTime = DateTime.UtcNow; _initializeTable = false; _nextBatch = new List <IDictionary <string, object> >(); Completed = new AutoResetEvent(false); // Setting up kusto connection if (!string.IsNullOrEmpty(authority)) { if (!string.IsNullOrEmpty(appclientId) && !string.IsNullOrEmpty(appKey)) { kscbIngest = new KustoConnectionStringBuilder($"https://ingest-{cluster}", database).WithAadApplicationKeyAuthentication(appclientId, appKey, authority); kscbAdmin = new KustoConnectionStringBuilder($"https://{cluster}", database).WithAadApplicationKeyAuthentication(appclientId, appKey, authority); } else { kscbIngest = new KustoConnectionStringBuilder($"https://ingest-{cluster}", database).WithAadUserPromptAuthentication(authority); kscbAdmin = new KustoConnectionStringBuilder($"https://{cluster}", database).WithAadUserPromptAuthentication(authority); } } if (kscbAdmin != null) { _ingestionProperties = new KustoIngestionProperties(kscbIngest.InitialCatalog, table); if (directIngest) { _ingestClient = KustoIngestFactory.CreateDirectIngestClient(this.kscbAdmin); } else { _ingestClient = KustoIngestFactory.CreateQueuedIngestClient(kscbIngest); } } else { _logger.Log(LogLevel.ERROR, "ERROR getting ADX connection strings. Please double check the information provided."); _error = true; } }
public static void Run([CosmosDBTrigger( databaseName: "data", collectionName: "device", ConnectionStringSetting = "rbdemocosmosdb_DOCUMENTDB", LeaseCollectionName = "leases", CreateLeaseCollectionIfNotExists = true)] IReadOnlyList <Document> input, ILogger log) { if (input != null && input.Count > 0) { log.LogInformation("Documents modified " + input.Count); log.LogInformation("First document Id " + input[0].Id); //input[0].ToString() is the data that needs to be ingested. var tenantId = System.Environment.GetEnvironmentVariable($"tenantId"); var applicationClientId = System.Environment.GetEnvironmentVariable($"applicationClientId"); var applicationKey = System.Environment.GetEnvironmentVariable($"applicationKey"); var serviceNameAndRegion = System.Environment.GetEnvironmentVariable($"serviceNameAndRegion"); var kustoDatabase = "device"; var kustoTableName = "Staging"; var mappingName = "deviceMapping"; var kustoIngestConnectionStringBuilder = new KustoConnectionStringBuilder($"https://ingest-{serviceNameAndRegion}.kusto.windows.net") { FederatedSecurity = true, InitialCatalog = kustoDatabase, ApplicationClientId = applicationClientId, ApplicationKey = applicationKey, Authority = tenantId }; using (var ingestClient = KustoIngestFactory.CreateQueuedIngestClient(kustoIngestConnectionStringBuilder)) { var properties = new KustoQueuedIngestionProperties(kustoDatabase, kustoTableName) { Format = DataSourceFormat.json, JSONMappingReference = mappingName, ReportLevel = IngestionReportLevel.FailuresAndSuccesses, ReportMethod = IngestionReportMethod.Queue }; string inputValue = input[0].ToString().Replace("\r\n", ""); //string inputValue = "{ \"Id\":\"1\", \"Timestamp\":\"2\", \"Message\":\"This is a test message\" }"; //object val = Newtonsoft.Json.JsonConvert.DeserializeObject(inputValue); var stream = new System.IO.MemoryStream(System.Text.Encoding.UTF8.GetBytes(inputValue)); stream.Seek(0, SeekOrigin.Begin); // Post ingestion message ingestClient.IngestFromStream(stream, properties, leaveOpen: true); // Wait and retrieve all notifications // - Actual duration should be decided based on the effective Ingestion Batching Policy set on the table/database Thread.Sleep(10000); var errors = ingestClient.GetAndDiscardTopIngestionFailures().GetAwaiter().GetResult(); var successes = ingestClient.GetAndDiscardTopIngestionSuccesses().GetAwaiter().GetResult(); errors.ForEach((f) => { Console.WriteLine($"Ingestion error: {f.Info.Details}"); }); successes.ForEach((s) => { Console.WriteLine($"Ingested: {s.Info.IngestionSourcePath}"); }); } } }
static async Task Main(string[] args) { // Ingest From a Local File using KustoQueuedIngestClient and report status to a table // Create Kusto connection string with App Authentication var kustoConnectionStringBuilderDM = new KustoConnectionStringBuilder(@"https://ingest-{clusterNameAndRegion}.kusto.windows.net").WithAadApplicationKeyAuthentication( applicationClientId: "{Application Client ID}", applicationKey: "{Application Key (secret)}", authority: "{AAD TenantID or name}"); // Create a disposable client that will execute the ingestion IKustoQueuedIngestClient client = KustoIngestFactory.CreateQueuedIngestClient(kustoConnectionStringBuilderDM); // Ingest from a file according to the required properties var kustoIngestionProperties = new KustoQueuedIngestionProperties(databaseName: "myDB", tableName: "myDB") { // Setting the report level to FailuresAndSuccesses will cause both successful and failed ingestions to be reported // (Rather than the default "FailuresOnly" level) ReportLevel = IngestionReportLevel.FailuresAndSuccesses, // Choose the report method of choice ReportMethod = IngestionReportMethod.Table }; var filePath = @"< Path to file >"; var fileIdentifier = Guid.NewGuid(); var fileDescription = new FileDescription() { FilePath = filePath, SourceId = fileIdentifier }; var sourceOptions = new StorageSourceOptions() { SourceId = fileDescription.SourceId.Value }; // Execute the ingest operation and save the result. var clientResult = await client.IngestFromStorageAsync(fileDescription.FilePath, ingestionProperties : kustoIngestionProperties, sourceOptions); // Use the fileIdentifier you supplied to get the status of your ingestion var ingestionStatus = clientResult.GetIngestionStatusBySourceId(fileIdentifier); while (ingestionStatus.Status == Status.Pending) { // Wait a minute... Thread.Sleep(TimeSpan.FromMinutes(1)); // Try again ingestionStatus = clientResult.GetIngestionStatusBySourceId(fileIdentifier); } // Verify the results of the ingestion Ensure.ConditionIsMet(ingestionStatus.Status == Status.Succeeded, "The file should have been ingested successfully"); // Dispose of the client client.Dispose(); }
protected FI(ILogger iLog, Microsoft.Azure.Management.Fluent.IAzure iAzure, string iRGName, string kustoConn, string kustoDBName, string kustoTableName) { curRGName = iRGName; log = iLog; curSubName = iAzure.SubscriptionId; ingestClient = KustoIngestFactory.CreateQueuedIngestClient(kustoConn); ingestProps = new KustoIngestionProperties(kustoDBName, kustoTableName); ingestProps.Format = Kusto.Data.Common.DataSourceFormat.csv; }
public IKustoIngestClient GetClient() { string ingestConnectionString = _kustoOptions.Value.IngestConnectionString; if (string.IsNullOrWhiteSpace(ingestConnectionString)) { throw new InvalidCastException($"Kusto {nameof(_kustoOptions.Value.IngestConnectionString)} is not configured in settings or related KeyVault"); } return(_clients.GetOrAdd(ingestConnectionString, _ => KustoIngestFactory.CreateQueuedIngestClient(ingestConnectionString))); }
public static Result <IKustoIngestClient> CreateKustoIngestClient(string kustoIngestionClusterUrl, string azureTenantId, string azureAppId, string azureAppKey) { Contract.RequiresNotNullOrEmpty(kustoIngestionClusterUrl); Contract.RequiresNotNullOrEmpty(azureTenantId); Contract.RequiresNotNullOrEmpty(azureAppId); Contract.RequiresNotNullOrEmpty(azureAppKey); var kustoIngestConnectionString = new KustoConnectionStringBuilder(kustoIngestionClusterUrl) .WithAadApplicationKeyAuthentication(azureAppId, azureAppKey, azureTenantId); return(new Result <IKustoIngestClient>(KustoIngestFactory.CreateDirectIngestClient(kustoIngestConnectionString))); }
/// <summary> /// Performs a write to the indicated table, sending the appropriately filtered test report to Kusto /// Returns the number of lines written /// Skips the final write step when testing /// </summary> public bool WriteDataForTable(bool testing, List <AdoBuildResultRow> reports) { try { // Create Kusto connection string with App Authentication var kustoConnectionStringBuilderEngine = new KustoConnectionStringBuilder(config.clusterUri).WithAadApplicationKeyAuthentication( applicationClientId: config.clientId, applicationKey: config.secretKey, authority: config.authority); using (var ingestClient = KustoIngestFactory.CreateDirectIngestClient(kustoConnectionStringBuilderEngine)) { var ingestProps = new KustoQueuedIngestionProperties(config.dbName, config.tableName); // For the sake of getting both failure and success notifications we set this to IngestionReportLevel.FailuresAndSuccesses // Usually the recommended level is IngestionReportLevel.FailuresOnly ingestProps.ReportLevel = IngestionReportLevel.FailuresAndSuccesses; ingestProps.ReportMethod = IngestionReportMethod.Queue; ingestProps.IngestionMapping = new IngestionMapping { IngestionMappingReference = config.mappingName }; ingestProps.Format = DataSourceFormat.json; // Prepare data for ingestion using (var memStream = new MemoryStream()) using (var writer = new StreamWriter(memStream)) { foreach (AdoBuildResultRow eachReport in reports) { writer.WriteLine(JsonConvert.SerializeObject(eachReport)); Console.WriteLine($"Writing: {eachReport.jobname}, ({eachReport.passed}/{eachReport.failed}/{eachReport.other}/{eachReport.total}) for {eachReport.timestamp}"); } writer.Flush(); memStream.Seek(0, SeekOrigin.Begin); // Post ingestion message if (!testing) { ingestClient.IngestFromStream(memStream, ingestProps, leaveOpen: true); } } } return(true); } catch (Exception e) { Console.WriteLine($"ERROR: Could not write to: {config.clusterUri}.{config.dbName}.{config.tableName}, using Mapping: {config.mappingName}"); Console.WriteLine(e.ToString()); return(false); } }
public async Task InsertRowAsync(T row, CancellationToken cancellationToken = default) { using IKustoQueuedIngestClient ingestClient = KustoIngestFactory.CreateQueuedIngestClient(_connectionStringBuilder); string serializedData = JsonConvert.SerializeObject(row); byte[] serializedBytes = Encoding.UTF8.GetBytes(serializedData); using MemoryStream dataStream = new MemoryStream(serializedBytes); // IKustoQueuedIngestClient doesn't support cancellation at the moment. Update the line below if it does in the future. await ingestClient.IngestFromStreamAsync(dataStream, _ingestionProperties, leaveOpen : true); }
private static bool TryInitialize(ILogger log) { lock (_lock) { if (!_initialized) { string kustoIngestUrl = GetEnvVariable("KustoIngestUrl"); string clientId = GetEnvVariable("ClientId"); string clientSecret = GetEnvVariable("ClientSecret"); string tenantId = GetEnvVariable("TenantId"); if (String.IsNullOrWhiteSpace(kustoIngestUrl) || String.IsNullOrWhiteSpace(clientId) || String.IsNullOrWhiteSpace(clientSecret) || String.IsNullOrWhiteSpace(tenantId)) { log.LogError($"Could not initialize the Kusto client because the connection parameters are wrong (url: {kustoIngestUrl} clientId: {clientId}, tenant: {tenantId}"); return(false); } //Initialize adx connection = new KustoConnectionStringBuilder(GetEnvVariable("KustoIngestUrl")).WithAadApplicationKeyAuthentication( applicationClientId: GetEnvVariable("ClientId"), applicationKey: GetEnvVariable("ClientSecret"), authority: GetEnvVariable("TenantId")); adx = KustoIngestFactory.CreateQueuedIngestClient(connection); if (adx != null) { _initialized = true; log.LogInformation("Function successfully initialized"); return(true); } else { log.LogWarning("Function not successfully initialized"); } } else { return(true); } } return(false); }
internal IBigBrother UseKustoInternal(string kustoEngineName, string kustoEngineLocation, string kustoDb, string tenantId) { try { KustoDbName = kustoDb; var kustoUri = $"https://{kustoEngineName}.{kustoEngineLocation}.kusto.windows.net"; var token = new AzureServiceTokenProvider().GetAccessTokenAsync(kustoUri, string.Empty).Result; KustoAdminClient = KustoClientFactory.CreateCslAdminProvider( new KustoConnectionStringBuilder(kustoUri) { FederatedSecurity = true, InitialCatalog = KustoDbName, Authority = tenantId, ApplicationToken = token }); var kustoQueuedIngestUri = $"https://ingest-{kustoEngineName}.{kustoEngineLocation}.kusto.windows.net"; var kustoIngestUri = $"https://{kustoEngineName}.{kustoEngineLocation}.kusto.windows.net"; KustoQueuedIngestClient = KustoIngestFactory.CreateQueuedIngestClient( new KustoConnectionStringBuilder(kustoQueuedIngestUri) { FederatedSecurity = true, InitialCatalog = KustoDbName, Authority = tenantId, ApplicationToken = token }); KustoDirectIngestClient = KustoIngestFactory.CreateDirectIngestClient( new KustoConnectionStringBuilder(kustoIngestUri) { FederatedSecurity = true, InitialCatalog = KustoDbName, Authority = tenantId, ApplicationToken = token }); SetupKustoSubscription(); KustoIngestionTimeMetric = InternalClient.GetMetric(new MetricIdentifier("", "KustoIngestionTime")); } catch (Exception e) { InternalStream.OnNext(e.ToExceptionEvent()); } return(this); }
static void Main(string[] args) { // Ingest From a Local Files using KustoQueuedIngestClient and report status to a queue // Create Kusto connection string with App Authentication var kustoConnectionStringBuilderDM = new KustoConnectionStringBuilder(@"https://ingest-{clusterNameAndRegion}.kusto.windows.net").WithAadApplicationKeyAuthentication( applicationClientId: "{Application Client ID}", applicationKey: "{Application Key (secret)}", authority: "{AAD TenantID or name}"); // Create a disposable client that will execute the ingestion IKustoQueuedIngestClient client = KustoIngestFactory.CreateQueuedIngestClient(kustoConnectionStringBuilderDM); // Ingest from a file according to the required properties var kustoIngestionProperties = new KustoQueuedIngestionProperties(databaseName: "myDB", tableName: "myTable") { // Setting the report level to FailuresAndSuccesses will cause both successful and failed ingestions to be reported // (Rather than the default "FailuresOnly" level - which is demonstrated in the // 'Ingest From Local File(s) using KustoQueuedIngestClient and Ingestion Validation' section) ReportLevel = IngestionReportLevel.FailuresAndSuccesses, // Choose the report method of choice. 'Queue' is the default method. // For the sake of the example, we will choose it anyway. ReportMethod = IngestionReportMethod.Queue }; client.IngestFromStorageAsync("ValidTestFile.csv", kustoIngestionProperties); client.IngestFromStorageAsync("InvalidTestFile.csv", kustoIngestionProperties); // Waiting for the aggregation Thread.Sleep(TimeSpan.FromMinutes(8)); // Retrieve and validate failures var ingestionFailures = client.PeekTopIngestionFailures().GetAwaiter().GetResult(); Ensure.IsTrue((ingestionFailures.Count() > 0), "The failed ingestion should have been reported to the failed ingestions queue"); // Retrieve, delete and validate failures ingestionFailures = client.GetAndDiscardTopIngestionFailures().GetAwaiter().GetResult(); Ensure.IsTrue((ingestionFailures.Count() > 0), "The failed ingestion should have been reported to the failed ingestions queue"); // Verify the success has also been reported to the queue var ingestionSuccesses = client.GetAndDiscardTopIngestionSuccesses().GetAwaiter().GetResult(); Ensure.ConditionIsMet((ingestionSuccesses.Count() > 0), "The successful ingestion should have been reported to the successful ingestions queue"); // Dispose of the client client.Dispose(); }
public async Task IngestFromCsvAsync( string csv, ServicePrincipalOptions servicePrincipal, string cluster, string database, string table, bool isDryRun) { KustoConnectionStringBuilder connectionBuilder = new KustoConnectionStringBuilder($"https://{cluster}.kusto.windows.net") .WithAadApplicationKeyAuthentication( servicePrincipal.ClientId, servicePrincipal.Secret, servicePrincipal.Tenant); using (IKustoIngestClient client = KustoIngestFactory.CreateDirectIngestClient(connectionBuilder)) { KustoIngestionProperties properties = new(database, table) { Format = DataSourceFormat.csv }; StreamSourceOptions sourceOptions = new() { SourceId = Guid.NewGuid() }; if (!isDryRun) { AsyncRetryPolicy retryPolicy = Policy .Handle <Kusto.Data.Exceptions.KustoException>() .Or <Kusto.Ingest.Exceptions.KustoException>() .WaitAndRetryAsync( Backoff.DecorrelatedJitterBackoffV2(TimeSpan.FromSeconds(10), RetryHelper.MaxRetries), RetryHelper.GetOnRetryDelegate(RetryHelper.MaxRetries, _loggerService)); IKustoIngestionResult result = await retryPolicy.ExecuteAsync( () => IngestFromStreamAsync(csv, client, properties, sourceOptions)); IngestionStatus ingestionStatus = result.GetIngestionStatusBySourceId(sourceOptions.SourceId); for (int i = 0; i < 10 && ingestionStatus.Status == Status.Pending; i++) { await Task.Delay(TimeSpan.FromSeconds(30)); ingestionStatus = result.GetIngestionStatusBySourceId(sourceOptions.SourceId); } if (ingestionStatus.Status == Status.Pending) { throw new InvalidOperationException($"Timeout while ingesting Kusto data."); } else if (ingestionStatus.Status != Status.Succeeded) { throw new InvalidOperationException( $"Failed to ingest Kusto data.{Environment.NewLine}{ingestionStatus.Details}"); } } } }
/// <summary> /// Ingest data into Kusto. /// </summary> /// <param name="table">Name of table to ingest into.</param> /// <param name="mappingName">Name of table mapping to ingest with.</param> /// <param name="stream">input JSON data stream.</param> /// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns> private static async Task <IKustoIngestionResult> KustoIngest(KustoConnectionStringBuilder kusto, string db, string table, string mappingName, Stream stream) { // Create a disposable client that will execute the ingestion using IKustoIngestClient client = KustoIngestFactory.CreateDirectIngestClient(kusto); var ingestProps = new KustoIngestionProperties(db, table) { JSONMappingReference = mappingName, Format = DataSourceFormat.json, }; var ssOptions = new StreamSourceOptions { CompressionType = DataSourceCompressionType.GZip, }; return(await client.IngestFromStreamAsync(stream, ingestProps, ssOptions)); }
public BlockingKustoUploader( string blobConnectionString, string blobContainerName, KustoConnectionStringBuilder adminCsb, KustoConnectionStringBuilder kscb, bool demoMode, string tableName, int batchSize, TimeSpan flushDuration, bool resetTable = false) { Csb = kscb; AdminCsb = adminCsb; TableName = tableName; BatchSize = batchSize; _flushDuration = flushDuration; _lastUploadTime = DateTime.UtcNow; _resetTable = resetTable; _initializeTable = false; Completed = new AutoResetEvent(false); if (Csb != null) { _ingestionProperties = new KustoIngestionProperties(Csb.InitialCatalog, tableName); if (demoMode) { _ingestClient = KustoIngestFactory.CreateDirectIngestClient(this.AdminCsb); } else { _ingestClient = KustoIngestFactory.CreateQueuedIngestClient(Csb); } } if (!string.IsNullOrEmpty(blobConnectionString) && !string.IsNullOrEmpty(blobContainerName)) { blobServiceClient = new BlobServiceClient(blobConnectionString); blobContainerClient = blobServiceClient.GetBlobContainerClient(blobContainerName); blobContainerClient.CreateIfNotExists(); } _nextBatch = new List <IDictionary <string, object> >(); }
public KustoTimelineTelemetryRepository(ILogger <KustoTimelineTelemetryRepository> logger, IOptionsSnapshot <KustoTimelineTelemetryOptions> options) { _logger = logger; // removing the IngestConnectionString was a default setup in local debugging if (string.IsNullOrEmpty(options.Value.IngestConnectionString)) { _logger.LogDebug("No ingest connection string provided; will ignore ingest operations"); _ingest = new NullKustoIngestClient(); } else { _ingest = KustoIngestFactory.CreateQueuedIngestClient(options.Value.IngestConnectionString); } _query = KustoClientFactory.CreateCslQueryProvider(options.Value.QueryConnectionString); _database = options.Value.Database; }
public async Task PublishManualReviews(IEnumerable <SloManualReview> results) { var dt = results.ToDataTable(); var kustoConnectionStringBuilderEngine = new KustoConnectionStringBuilder(kustoUrl_).WithAadUserPromptAuthentication(); using (var client = KustoIngestFactory.CreateDirectIngestClient(kustoConnectionStringBuilderEngine)) { //Ingest from blobs according to the required properties var kustoIngestionProperties = new KustoIngestionProperties( databaseName: kustoDb_, tableName: kustoManualReviewTable_ ); var reader = dt.CreateDataReader(); var result = await client.IngestFromDataReaderAsync(reader, kustoIngestionProperties); } }
public BlockingKustoUploader( string outputFileName, KustoConnectionStringBuilder adminCsb, KustoConnectionStringBuilder kscb, bool demoMode, string tableName, int batchSize, TimeSpan flushDuration, bool resetTable = false) { OutputFileName = outputFileName; Csb = kscb; AdminCsb = adminCsb; TableName = tableName; BatchSize = batchSize; _flushDuration = flushDuration; _lastUploadTime = DateTime.UtcNow; _resetTable = resetTable; _initializeTable = false; Completed = new AutoResetEvent(false); if (Csb != null) { _ingestionProperties = new KustoIngestionProperties(Csb.InitialCatalog, tableName); if (demoMode) { _ingestClient = KustoIngestFactory.CreateDirectIngestClient(this.AdminCsb); } else { _ingestClient = KustoIngestFactory.CreateQueuedIngestClient(Csb); } } if (!string.IsNullOrEmpty(OutputFileName)) { outputFile = new StreamWriter(this.OutputFileName); outputFile.Write($"[{Environment.NewLine}"); } _nextBatch = new List <IDictionary <string, object> >(); }
public async Task IngestFromCsvStreamAsync(Stream csv, IngestKustoImageInfoOptions options) { KustoConnectionStringBuilder connectionBuilder = new KustoConnectionStringBuilder($"https://{options.Cluster}.kusto.windows.net") .WithAadApplicationKeyAuthentication( options.ServicePrincipal.ClientId, options.ServicePrincipal.Secret, options.ServicePrincipal.Tenant); using (IKustoIngestClient client = KustoIngestFactory.CreateDirectIngestClient(connectionBuilder)) { KustoIngestionProperties properties = new KustoIngestionProperties(options.Database, options.Table) { Format = DataSourceFormat.csv }; StreamSourceOptions sourceOptions = new StreamSourceOptions { SourceId = Guid.NewGuid() }; if (!options.IsDryRun) { IKustoIngestionResult result = await client.IngestFromStreamAsync(csv, properties, sourceOptions); IngestionStatus ingestionStatus = result.GetIngestionStatusBySourceId(sourceOptions.SourceId); for (int i = 0; i < 10 && ingestionStatus.Status == Status.Pending; i++) { await Task.Delay(TimeSpan.FromSeconds(30)); ingestionStatus = result.GetIngestionStatusBySourceId(sourceOptions.SourceId); } if (ingestionStatus.Status != Status.Succeeded) { throw new InvalidOperationException( $"Failed to ingest Kusto data.{Environment.NewLine}{ingestionStatus.Details}"); } else if (ingestionStatus.Status == Status.Pending) { throw new InvalidOperationException($"Timeout while ingesting Kusto data."); } } } }
static void Main(string[] args) { // Ingest From Local File using KustoDirectIngestClient (only for test purposes): // Create Kusto connection string with App Authentication var kustoConnectionStringBuilderEngine = new KustoConnectionStringBuilder(@"https://{clusterNameAndRegion}.kusto.windows.net").WithAadApplicationKeyAuthentication( applicationClientId: "{Application Client ID}", applicationKey: "{Application Key (secret)}", authority: "{AAD TenantID or name}"); // Create a disposable client that will execute the ingestion using (IKustoIngestClient client = KustoIngestFactory.CreateDirectIngestClient(kustoConnectionStringBuilderEngine)) { //Ingest from blobs according to the required properties var kustoIngestionProperties = new KustoIngestionProperties(databaseName: "myDB", tableName: "myTable"); client.IngestFromStorageAsync(@"< Path to local file >", ingestionProperties: kustoIngestionProperties).GetAwaiter().GetResult(); } }
private static async Task IngestTriageItemsIntoKusto(TriageItem[] triageItems, ILogger log) { log.LogInformation("Entering IngestTriageItemIntoKusto"); string kustoIngestConnectionString = System.Environment.GetEnvironmentVariable("KustoIngestConnectionString"); string databaseName = System.Environment.GetEnvironmentVariable("KustoDatabaseName"); IKustoIngestClient ingestClient = KustoIngestFactory.CreateQueuedIngestClient(kustoIngestConnectionString); await KustoHelpers.WriteDataToKustoInMemoryAsync( ingestClient, databaseName, "TimelineIssuesTriage", log, triageItems, b => new[] { new KustoValue("ModifiedDateTime", b.ModifiedDateTime.ToString(), KustoDataTypes.DateTime), new KustoValue("BuildId", b.BuildId.ToString(), KustoDataTypes.Int), new KustoValue("RecordId", b.RecordId.ToString(), KustoDataTypes.Guid), new KustoValue("Index", b.Index.ToString(), KustoDataTypes.Int), new KustoValue("UpdatedCategory", b?.UpdatedCategory, KustoDataTypes.String), new KustoValue("Url", b?.Url, KustoDataTypes.String) }); }
private IKustoIngestClient GetKustoIngestClient(string appId, string appKey, EventSinkInfo eventSinkInfo) { IKustoIngestClient _client; try { var kcsb = new KustoConnectionStringBuilder(eventSinkInfo.Settings.EndpointUrl) { InitialCatalog = eventSinkInfo.Settings.Database, FederatedSecurity = true }; kcsb = kcsb.WithAadApplicationKeyAuthentication(appId, appKey, eventSinkInfo.Settings.Authority); _client = KustoIngestFactory.CreateQueuedIngestClient(kcsb); } catch (Exception e) { Console.WriteLine(e); throw; } return(_client); }