/// <inheritdoc /> public IBigBrother UseKusto(string kustoEngineName, string kustoEngineLocation, string kustoDb, string tenantId) { KustoDbName = kustoDb; var kustoUri = $"https://{kustoEngineName}.{kustoEngineLocation}.kusto.windows.net"; var kustoIngestUri = $"https://ingest-{kustoEngineName}.{kustoEngineLocation}.kusto.windows.net"; var token = new AzureServiceTokenProvider().GetAccessTokenAsync(kustoUri, string.Empty).Result; KustoAdminClient = KustoClientFactory.CreateCslAdminProvider( new KustoConnectionStringBuilder(kustoUri) { FederatedSecurity = true, InitialCatalog = KustoDbName, AuthorityId = tenantId, ApplicationToken = token }); KustoIngestClient = KustoIngestFactory.CreateQueuedIngestClient( new KustoConnectionStringBuilder(kustoIngestUri) { FederatedSecurity = true, InitialCatalog = KustoDbName, AuthorityId = tenantId, ApplicationToken = token }); SetupKustoSubscription(); return(this); }
static void Main(string[] args) { // Ingest From Local Files using KustoQueuedIngestClient and Ingestion Validation // Create Kusto connection string with App Authentication var kustoConnectionStringBuilderDM = new KustoConnectionStringBuilder(@"https://ingest-{clusterNameAndRegion}.kusto.windows.net").WithAadApplicationKeyAuthentication( applicationClientId: "{Application Client ID}", applicationKey: "{Application Key (secret)}", authority: "{AAD TenantID or name}"); // Create a disposable client that will execute the ingestion IKustoQueuedIngestClient client = KustoIngestFactory.CreateQueuedIngestClient(kustoConnectionStringBuilderDM); // Ingest from files according to the required properties var kustoIngestionProperties = new KustoIngestionProperties(databaseName: "myDB", tableName: "myTable"); client.IngestFromStorageAsync(@"ValidTestFile.csv", kustoIngestionProperties); client.IngestFromStorageAsync(@"InvalidTestFile.csv", kustoIngestionProperties); // Waiting for the aggregation Thread.Sleep(TimeSpan.FromMinutes(8)); // Retrieve and validate failures var ingestionFailures = client.PeekTopIngestionFailures().GetAwaiter().GetResult(); Ensure.IsTrue((ingestionFailures.Count() > 0), "Failures expected"); // Retrieve, delete and validate failures ingestionFailures = client.GetAndDiscardTopIngestionFailures().GetAwaiter().GetResult(); Ensure.IsTrue((ingestionFailures.Count() > 0), "Failures expected"); // Dispose of the client client.Dispose(); }
/// <summary> /// Constructor. Initializes this object and does nothing else. /// </summary> /// <param name="connectionString">Kusto connection string.</param> /// <param name="database">Database into which to ingest.</param> /// <param name="table">Table into which to ingest.</param> /// <param name="csvMapping">Csv mapping that applies to all CSV files to be uploaded via <see cref="UploadSingleCsvFile"/></param> /// <param name="deleteFilesOnSuccess">Whether to delete files upon successful upload.</param> /// <param name="checkForIngestionErrors"> /// Whether to check for ingestion errors before disposing this object. /// Note that at this time not all uploaded files have necessarily been ingested; this class /// does not wait for ingestions to complete, it only checks for failures of those that have completed. /// </param> /// <param name="log">Optional log to which to write some debug information.</param> public KustoUploader ( string connectionString, string database, string table, IEnumerable <CsvColumnMapping> csvMapping, bool deleteFilesOnSuccess, bool checkForIngestionErrors, ILog log = null ) { _log = log; _deleteFilesOnSuccess = deleteFilesOnSuccess; _checkForIngestionErrors = checkForIngestionErrors; _client = KustoIngestFactory.CreateQueuedIngestClient(connectionString); _hasUploadErrors = false; _ingestionProperties = new KustoQueuedIngestionProperties(database, table) { CSVMapping = csvMapping, ReportLevel = IngestionReportLevel.FailuresOnly, ReportMethod = IngestionReportMethod.Queue, }; _block = new ActionBlock <FileDescription> ( UploadSingleCsvFile, new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = 1, BoundedCapacity = DataflowBlockOptions.Unbounded } ); }
static async Task Main(string[] args) { // Ingest From a Local File using KustoQueuedIngestClient and report status to a table // Create Kusto connection string with App Authentication var kustoConnectionStringBuilderDM = new KustoConnectionStringBuilder(@"https://ingest-{clusterNameAndRegion}.kusto.windows.net").WithAadApplicationKeyAuthentication( applicationClientId: "{Application Client ID}", applicationKey: "{Application Key (secret)}", authority: "{AAD TenantID or name}"); // Create a disposable client that will execute the ingestion IKustoQueuedIngestClient client = KustoIngestFactory.CreateQueuedIngestClient(kustoConnectionStringBuilderDM); // Ingest from a file according to the required properties var kustoIngestionProperties = new KustoQueuedIngestionProperties(databaseName: "myDB", tableName: "myDB") { // Setting the report level to FailuresAndSuccesses will cause both successful and failed ingestions to be reported // (Rather than the default "FailuresOnly" level) ReportLevel = IngestionReportLevel.FailuresAndSuccesses, // Choose the report method of choice ReportMethod = IngestionReportMethod.Table }; var filePath = @"< Path to file >"; var fileIdentifier = Guid.NewGuid(); var fileDescription = new FileDescription() { FilePath = filePath, SourceId = fileIdentifier }; var sourceOptions = new StorageSourceOptions() { SourceId = fileDescription.SourceId.Value }; // Execute the ingest operation and save the result. var clientResult = await client.IngestFromStorageAsync(fileDescription.FilePath, ingestionProperties : kustoIngestionProperties, sourceOptions); // Use the fileIdentifier you supplied to get the status of your ingestion var ingestionStatus = clientResult.GetIngestionStatusBySourceId(fileIdentifier); while (ingestionStatus.Status == Status.Pending) { // Wait a minute... Thread.Sleep(TimeSpan.FromMinutes(1)); // Try again ingestionStatus = clientResult.GetIngestionStatusBySourceId(fileIdentifier); } // Verify the results of the ingestion Ensure.ConditionIsMet(ingestionStatus.Status == Status.Succeeded, "The file should have been ingested successfully"); // Dispose of the client client.Dispose(); }
protected FI(ILogger iLog, Microsoft.Azure.Management.Fluent.IAzure iAzure, string iRGName, string kustoConn, string kustoDBName, string kustoTableName) { curRGName = iRGName; log = iLog; curSubName = iAzure.SubscriptionId; ingestClient = KustoIngestFactory.CreateQueuedIngestClient(kustoConn); ingestProps = new KustoIngestionProperties(kustoDBName, kustoTableName); ingestProps.Format = Kusto.Data.Common.DataSourceFormat.csv; }
public async Task InsertRowAsync(T row, CancellationToken cancellationToken = default) { using IKustoQueuedIngestClient ingestClient = KustoIngestFactory.CreateQueuedIngestClient(_connectionStringBuilder); string serializedData = JsonConvert.SerializeObject(row); byte[] serializedBytes = Encoding.UTF8.GetBytes(serializedData); using MemoryStream dataStream = new MemoryStream(serializedBytes); // IKustoQueuedIngestClient doesn't support cancellation at the moment. Update the line below if it does in the future. await ingestClient.IngestFromStreamAsync(dataStream, _ingestionProperties, leaveOpen : true); }
private static bool TryInitialize(ILogger log) { lock (_lock) { if (!_initialized) { string kustoIngestUrl = GetEnvVariable("KustoIngestUrl"); string clientId = GetEnvVariable("ClientId"); string clientSecret = GetEnvVariable("ClientSecret"); string tenantId = GetEnvVariable("TenantId"); if (String.IsNullOrWhiteSpace(kustoIngestUrl) || String.IsNullOrWhiteSpace(clientId) || String.IsNullOrWhiteSpace(clientSecret) || String.IsNullOrWhiteSpace(tenantId)) { log.LogError($"Could not initialize the Kusto client because the connection parameters are wrong (url: {kustoIngestUrl} clientId: {clientId}, tenant: {tenantId}"); return(false); } //Initialize adx connection = new KustoConnectionStringBuilder(GetEnvVariable("KustoIngestUrl")).WithAadApplicationKeyAuthentication( applicationClientId: GetEnvVariable("ClientId"), applicationKey: GetEnvVariable("ClientSecret"), authority: GetEnvVariable("TenantId")); adx = KustoIngestFactory.CreateQueuedIngestClient(connection); if (adx != null) { _initialized = true; log.LogInformation("Function successfully initialized"); return(true); } else { log.LogWarning("Function not successfully initialized"); } } else { return(true); } } return(false); }
static void Main(string[] args) { // Ingest From a Local Files using KustoQueuedIngestClient and report status to a queue // Create Kusto connection string with App Authentication var kustoConnectionStringBuilderDM = new KustoConnectionStringBuilder(@"https://ingest-{clusterNameAndRegion}.kusto.windows.net").WithAadApplicationKeyAuthentication( applicationClientId: "{Application Client ID}", applicationKey: "{Application Key (secret)}", authority: "{AAD TenantID or name}"); // Create a disposable client that will execute the ingestion IKustoQueuedIngestClient client = KustoIngestFactory.CreateQueuedIngestClient(kustoConnectionStringBuilderDM); // Ingest from a file according to the required properties var kustoIngestionProperties = new KustoQueuedIngestionProperties(databaseName: "myDB", tableName: "myTable") { // Setting the report level to FailuresAndSuccesses will cause both successful and failed ingestions to be reported // (Rather than the default "FailuresOnly" level - which is demonstrated in the // 'Ingest From Local File(s) using KustoQueuedIngestClient and Ingestion Validation' section) ReportLevel = IngestionReportLevel.FailuresAndSuccesses, // Choose the report method of choice. 'Queue' is the default method. // For the sake of the example, we will choose it anyway. ReportMethod = IngestionReportMethod.Queue }; client.IngestFromStorageAsync("ValidTestFile.csv", kustoIngestionProperties); client.IngestFromStorageAsync("InvalidTestFile.csv", kustoIngestionProperties); // Waiting for the aggregation Thread.Sleep(TimeSpan.FromMinutes(8)); // Retrieve and validate failures var ingestionFailures = client.PeekTopIngestionFailures().GetAwaiter().GetResult(); Ensure.IsTrue((ingestionFailures.Count() > 0), "The failed ingestion should have been reported to the failed ingestions queue"); // Retrieve, delete and validate failures ingestionFailures = client.GetAndDiscardTopIngestionFailures().GetAwaiter().GetResult(); Ensure.IsTrue((ingestionFailures.Count() > 0), "The failed ingestion should have been reported to the failed ingestions queue"); // Verify the success has also been reported to the queue var ingestionSuccesses = client.GetAndDiscardTopIngestionSuccesses().GetAwaiter().GetResult(); Ensure.ConditionIsMet((ingestionSuccesses.Count() > 0), "The successful ingestion should have been reported to the successful ingestions queue"); // Dispose of the client client.Dispose(); }
/// <summary> /// Saves the details of the <see cref="HttpResponseMessage"/> to /// </summary> /// <param name="monitorName"></param> /// <param name="httpResponse"></param> /// <returns></returns> public async Task ReportUrlAccessAsync(string monitorName, HttpResponseMessage httpResponse, CancellationToken cancellationToken = default) { HttpRequestLogEntry logEntry = new HttpRequestLogEntry() { MonitorName = monitorName, EventTime = DateTime.UtcNow, RequestedUrl = httpResponse.RequestMessage.RequestUri.AbsoluteUri, HttpStatusCode = (int)httpResponse.StatusCode }; KustoConnectionStringBuilder kcsb = new KustoConnectionStringBuilder($"https://ingest-{ServiceNameAndRegion}.kusto.windows.net") .WithAadManagedIdentity("system"); using IKustoQueuedIngestClient ingestClient = KustoIngestFactory.CreateQueuedIngestClient(kcsb); KustoQueuedIngestionProperties ingestProps = new KustoQueuedIngestionProperties(DatabaseName, TableName) { ReportLevel = IngestionReportLevel.FailuresOnly, ReportMethod = IngestionReportMethod.Queue, IngestionMapping = new IngestionMapping() { IngestionMappingKind = Kusto.Data.Ingestion.IngestionMappingKind.Json, IngestionMappings = HttpRequestLogColumnMapping }, Format = DataSourceFormat.json }; using MemoryStream memStream = new MemoryStream(); using StreamWriter writer = new StreamWriter(memStream); writer.WriteLine(JsonConvert.SerializeObject(logEntry)); writer.Flush(); memStream.Seek(0, SeekOrigin.Begin); // IKustoQueuedIngestClient doesn't support cancellation at the moment. Update the line below if it does in the future. await ingestClient.IngestFromStreamAsync(memStream, ingestProps, leaveOpen : true); }
protected AzureDataExplorerService(IKustoClientFactory kustoClientFactory) { this._ingestionClient = kustoClientFactory.GetQueuedIngestClient(); this._kustoConnectionStringBuilder = kustoClientFactory.KustoConnectionStringBuilder; }