public Monitor(Configuration configuration) { Contract.RequiresNotNull(configuration); _configuration = configuration; _logger = CreateLogger(); Contract.RequiresNotNullOrEmpty(_configuration.KustoIngestionClusterUrl); Contract.RequiresNotNullOrEmpty(_configuration.ApplicationClientId); Contract.RequiresNotNullOrEmpty(_configuration.ApplicationKey); Contract.RequiresNotNullOrEmpty(_configuration.Authority); var kustoIngestConnectionString = new KustoConnectionStringBuilder(_configuration.KustoIngestionClusterUrl) .WithAadApplicationKeyAuthentication(_configuration.ApplicationClientId, _configuration.ApplicationKey, _configuration.Authority); _kustoIngestClient = KustoIngestFactory.CreateDirectIngestClient(kustoIngestConnectionString); var kustoConnectionString = new KustoConnectionStringBuilder(_configuration.KustoClusterUrl) .WithAadApplicationKeyAuthentication(_configuration.ApplicationClientId, _configuration.ApplicationKey, _configuration.Authority); _cslQueryProvider = KustoClientFactory.CreateCslQueryProvider(kustoConnectionString); Contract.RequiresNotNull(_configuration.KustoNotifier); _alertNotifier = new KustoWriter <Notification>(_configuration.KustoNotifier, _logger, _kustoIngestClient); Contract.RequiresNotNull(_configuration.SchedulerKustoNotifier); _schedulerLogWriter = new KustoWriter <Scheduler.LogEntry>(_configuration.SchedulerKustoNotifier, _logger, _kustoIngestClient); _scheduler = new Scheduler(_configuration.Scheduler, _logger, _clock, _schedulerLogWriter); }
public BlockingKustoUploader( KustoConnectionStringBuilder kscb, string tableName, int batchSize, TimeSpan flushDuration) { csb = kscb; TableName = TableName; BatchSize = batchSize; _flushDuration = flushDuration; _lastUploadTime = DateTime.Now; Completed = new AutoResetEvent(false); _ingestionProperties = new KustoIngestionProperties(csb.InitialCatalog, tableName); _fields = typeof(EtwEvent).GetFields().Select(f => f.Name).ToArray(); if (csb.DataSource.StartsWith("https://ingest-")) { _ingestClient = KustoIngestFactory.CreateQueuedIngestClient(csb); } else { _ingestClient = KustoIngestFactory.CreateDirectIngestClient(csb); } _nextBatch = new List <T>(); }
private static string CreateADXTable(string databaseName, string table, KustoConnectionStringBuilder kustoConnectionStringBuilder) { var command = ""; using (var kustoClient = KustoClientFactory.CreateCslAdminProvider(kustoConnectionStringBuilder)) { command = CslCommandGenerator.GenerateTableCreateCommand( table, new[] { Tuple.Create("id", "System.Int32"), Tuple.Create("date", "System.DateTime"), Tuple.Create("time", "System.DateTime"), Tuple.Create("sym", "System.String"), Tuple.Create("qty", "System.Double"), Tuple.Create("px", "System.Double") }); kustoClient.ExecuteControlCommand(databaseName, command); //if (!isBatch) //{ // var tablePolicyAlterCommand = // CslCommandGenerator.GenerateTableAlterStreamingIngestionPolicyCommand(table, isEnabled: true); // kustoClient.ExecuteControlCommand(databaseName, tablePolicyAlterCommand); //} return(command); // kustoClient.ExecuteControlCommand(databaseName, ".create table StreamingDataTable (['id']:int)"); } }
} // - function Read private static void InitializeKustoClient(ILogger log) { lock (Lock) { if (!_isInitialized) { log.LogInformation("[PrometheusRead] [InitializeKustoClient] Initialize"); var connection = new KustoConnectionStringBuilder( Environment.GetEnvironmentVariable("kustoUrl", EnvironmentVariableTarget.Process)) .WithAadApplicationKeyAuthentication( applicationClientId: Environment.GetEnvironmentVariable("appClientId", EnvironmentVariableTarget.Process), applicationKey: Environment.GetEnvironmentVariable("appClientSecret", EnvironmentVariableTarget.Process), authority: Environment.GetEnvironmentVariable("tenantId", EnvironmentVariableTarget.Process)); _adx = KustoClientFactory.CreateCslQueryProvider(connection); _isInitialized = true; } log.LogInformation("[PrometheusRead] [InitializeKustoClient] Ok"); } } // - InitializeKustoClient
static void Main(string[] args) { var builder = new KustoConnectionStringBuilder("https://masvaas.kusto.windows.net/") .WithAadUserPromptAuthentication(); using (var adminProvider = KustoClientFactory.CreateCslAdminProvider(builder)) { var command = CslCommandGenerator.GenerateDatabasesShowCommand(); var objectReader = new ObjectReader <DatabasesShowCommandResult>(adminProvider.ExecuteControlCommand(command)); foreach (var temp in objectReader) { var db = temp.DatabaseName; var databaseJournalCommand = CslCommandGenerator.GenerateDatabaseJournalShowCommand(db); databaseJournalCommand += " | where Event == 'ADD-DATABASE' | project EventTimestamp"; using (var journalCmdResult = adminProvider.ExecuteControlCommand(db, databaseJournalCommand)) { if (journalCmdResult.Read() && DateTime.TryParse(journalCmdResult["EventTimestamp"].ToString(), out var createdTime)) { //ValhallaLogger.LogInformationalMessage(operationId, nameof(DatabaseInfoProvider), $"Database {database} Created time {createdTime.ToUniversalTime():o}"); Console.WriteLine($"Database: {db}, CreationTime: {createdTime}"); } } } } }
public BigBrotherUseKustoTest(ITestOutputHelper output) { _output = output; _kustoName = Environment.GetEnvironmentVariable("kusto_name"); _kustoLocation = Environment.GetEnvironmentVariable("kusto_location"); _kustoDatabase = Environment.GetEnvironmentVariable("kusto_database"); _kustoTenantId = Environment.GetEnvironmentVariable("kusto_tenant_id"); if (_kustoName != null && _kustoLocation != null && _kustoDatabase != null && _kustoTenantId != null) { var kustoUri = $"https://{_kustoName}.{_kustoLocation}.kusto.windows.net"; var token = new AzureServiceTokenProvider().GetAccessTokenAsync(kustoUri, _kustoTenantId).Result; var kustoConnectionString = new KustoConnectionStringBuilder(kustoUri) { FederatedSecurity = true, InitialCatalog = _kustoDatabase, Authority = _kustoTenantId, ApplicationToken = token }; _kustoQueryClient = KustoClientFactory.CreateCslQueryProvider(kustoConnectionString); _kustoAdminClient = KustoClientFactory.CreateCslAdminProvider(kustoConnectionString); } }
static void Main(string[] args) { // Ingest From Local Files using KustoQueuedIngestClient and Ingestion Validation // Create Kusto connection string with App Authentication var kustoConnectionStringBuilderDM = new KustoConnectionStringBuilder(@"https://ingest-{clusterNameAndRegion}.kusto.windows.net").WithAadApplicationKeyAuthentication( applicationClientId: "{Application Client ID}", applicationKey: "{Application Key (secret)}", authority: "{AAD TenantID or name}"); // Create a disposable client that will execute the ingestion IKustoQueuedIngestClient client = KustoIngestFactory.CreateQueuedIngestClient(kustoConnectionStringBuilderDM); // Ingest from files according to the required properties var kustoIngestionProperties = new KustoIngestionProperties(databaseName: "myDB", tableName: "myTable"); client.IngestFromStorageAsync(@"ValidTestFile.csv", kustoIngestionProperties); client.IngestFromStorageAsync(@"InvalidTestFile.csv", kustoIngestionProperties); // Waiting for the aggregation Thread.Sleep(TimeSpan.FromMinutes(8)); // Retrieve and validate failures var ingestionFailures = client.PeekTopIngestionFailures().GetAwaiter().GetResult(); Ensure.IsTrue((ingestionFailures.Count() > 0), "Failures expected"); // Retrieve, delete and validate failures ingestionFailures = client.GetAndDiscardTopIngestionFailures().GetAwaiter().GetResult(); Ensure.IsTrue((ingestionFailures.Count() > 0), "Failures expected"); // Dispose of the client client.Dispose(); }
static void UploadRealTime( string _sessionName, string _queryFile, string _outputFileName, KustoConnectionStringBuilder kscbAdmin, KustoConnectionStringBuilder kscbIngest, bool _demoMode, string _tableName, bool _resetTable) { var etw = Tx.Windows.EtwTdhObservable.FromSession(_sessionName); Console.WriteLine(); Console.WriteLine("Listening to real-time session '{0}'. Press Enter to terminate", _sessionName); var ku = CreateUploader(UploadTimespan, _outputFileName, kscbAdmin, kscbIngest, _demoMode, _tableName, _resetTable); Task task = Task.Factory.StartNew(() => { RunUploader(ku, etw, _queryFile); }); string readline = Console.ReadLine(); ku.OnCompleted(); }
public static void CreateTableMappingFromDefinition(string databaseName, string table, string tableMappingName, KustoConnectionStringBuilder kustoConnectionStringBuilder, IDictionary <string, string> tableDefinition) { using (var kustoClient = KustoClientFactory.CreateCslAdminProvider(kustoConnectionStringBuilder)) { var columnMappings = new List <ColumnMapping>(); int cnt = 0; foreach (var keyvaluepair in tableDefinition) { columnMappings.Add(new ColumnMapping() { ColumnName = keyvaluepair.Key, Properties = new Dictionary <string, string>() { { MappingConsts.Ordinal, cnt.ToString() } } }); cnt++; } var command = CslCommandGenerator.GenerateTableMappingCreateCommand( Kusto.Data.Ingestion.IngestionMappingKind.Csv, table, tableMappingName, columnMappings); kustoClient.ExecuteControlCommand(databaseName, command); } }
static void MainImpl(Arguments arguments) { // 1. Create a connection string to a cluster/database with AAD user authentication var cluster = "https://help.kusto.windows.net/"; var database = "Samples"; var kcsb = new KustoConnectionStringBuilder(cluster, database) { FederatedSecurity = true }; // 2. Connect to the Kusto query endpoint and create a query provider using (var queryProvider = KustoClientFactory.CreateCslQueryProvider(kcsb)) { // 3. Send a query using the V2 API var query = "print Welcome='Hello, World!'; print PI=pi()"; var properties = new ClientRequestProperties() { ClientRequestId = "kusto_samples_query_v2;" + Guid.NewGuid().ToString() }; if (arguments.ProgressiveMode) { properties.SetOption(ClientRequestProperties.OptionResultsProgressiveEnabled, true); } var queryTask = queryProvider.ExecuteQueryV2Async(database, query, properties); // 4. Parse and print the results of the query WriteResultsToConsole(queryTask); } }
private static ICslQueryProvider CreateClient(AppConfigHelper appConfigHelper) { string adxClustorName = appConfigHelper.GetValue(ADXClusterNameKey); string location = appConfigHelper.GetValue(LocationKey); string applicationId = appConfigHelper.GetValue(ApplicationIdKey); string applicationSecret = appConfigHelper.GetValue(ApplicationSecretKey); string appTenant = appConfigHelper.GetValue(AppTenantKey); try { var kustoUri = $"https://{adxClustorName}.{location}.kusto.windows.net/"; var kustoConnectionStringBuilder = new KustoConnectionStringBuilder(kustoUri) .WithAadApplicationKeyAuthentication( applicationClientId: applicationId, applicationKey: applicationSecret, authority: appTenant); return KustoClientFactory.CreateCslQueryProvider(kustoConnectionStringBuilder); } catch (Exception ex) { throw new ApplicationException("DocumentClient creation failed in the helper class", ex); } }
private ICslAdminProvider GetKustoAdminClient() { if (this.client == null) { try { var kustoUri = $"https://{this.config.Global.DataExplorer.Name}.{this.config.Global.Location}.kusto.windows.net/"; var kustoConnectionStringBuilder = new KustoConnectionStringBuilder(kustoUri) .WithAadApplicationKeyAuthentication( applicationClientId: this.config.Global.AzureActiveDirectory.AppId, applicationKey: this.config.Global.AzureActiveDirectory.AppSecret, authority: this.config.Global.AzureActiveDirectory.TenantId); this.client = KustoClientFactory.CreateCslAdminProvider(kustoConnectionStringBuilder); } catch (Exception e) { var msg = "Unable to retrieve kusto with Active Directory properties"; throw new InvalidConfigurationException(msg, e); } if (this.client == null) { // this.logger.LogError(new Exception(errorMessage), errorMessage); throw new InvalidConfigurationException("Could not connect to kusto client"); } } return(this.client); }
static void UploadSyslogRealTime( string listenerAdapterName, int listenerUdpPort, string queryFile, string outputFileName, KustoConnectionStringBuilder kscbAdmin, KustoConnectionStringBuilder kscbIngest, bool quickIngest, string tableName, bool resetTable) { var parser = CreateSIEMfxSyslogParser(); IPAddress localIp = null; if (!string.IsNullOrEmpty(listenerAdapterName)) { localIp = GetLocalIp(listenerAdapterName); } localIp ??= IPAddress.IPv6Any; var endPoint = new IPEndPoint(localIp, listenerUdpPort); var PortListener = new UdpClient(AddressFamily.InterNetworkV6); PortListener.Client.DualMode = true; PortListener.Client.Bind(endPoint); PortListener.Client.ReceiveBufferSize = 10 * 1024 * 1024; using var listener = new SyslogListener(parser, PortListener); var filter = new SyslogFilter(); if (filter != null) { listener.Filter = filter.Allow; } listener.Error += Listener_Error; listener.EntryReceived += Listener_EntryReceived; var _converter = new SyslogEntryToRecordConverter(); listener.Subscribe(_converter); listener.Start(); Console.WriteLine(); Console.WriteLine("Listening to Syslog events. Press any key to terminate"); var ku = CreateUploader(UploadTimespan, outputFileName, kscbAdmin, kscbIngest, quickIngest, tableName, resetTable); Task task = Task.Factory.StartNew(() => { RunUploader(ku, _converter, queryFile); }); string readline = Console.ReadLine(); listener.Stop(); ku.OnCompleted(); }
private static string CreateADXTableFromDefinition(string databaseName, string table, KustoConnectionStringBuilder kustoConnectionStringBuilder, IDictionary <string, string> tableDefinition) { var command = ""; var tuple = new Tuple <string, string> [tableDefinition.Count()]; int cnt = 0; foreach (var keyvaluepair in tableDefinition) { tuple[cnt] = new Tuple <string, string>(keyvaluepair.Key, keyvaluepair.Value); cnt++; } using (var kustoClient = KustoClientFactory.CreateCslAdminProvider(kustoConnectionStringBuilder)) { command = CslCommandGenerator.GenerateTableCreateCommand( table, tuple); var tablePolicyAlterCommand = CslCommandGenerator.GenerateTableAlterStreamingIngestionPolicyCommand(table, isEnabled: true); kustoClient.ExecuteControlCommand(databaseName, command); kustoClient.ExecuteControlCommand(databaseName, tablePolicyAlterCommand); } return(command); }
public KustoClientLogger(string databaseName, string tableName, List <KustoColumnMapping> _schema, string clientId, string clientKey) { var csvColumnMappings = new List <CsvColumnMapping>(); foreach (var mapping in _schema) { csvColumnMappings.Add(new CsvColumnMapping { ColumnName = mapping.ColumnName, CslDataType = mapping.DataType, Ordinal = mapping.ColumnNumber }); } _kustoProperties = new KustoIngestionProperties(databaseName, tableName) { Format = DataSourceFormat.csv, CSVMapping = csvColumnMappings }; var kustoConnectionStringBuilderDM = new KustoConnectionStringBuilder(@"https://wawstest.kusto.windows.net:443") { FederatedSecurity = true, InitialCatalog = databaseName, ApplicationKey = clientKey, ApplicationClientId = clientId }; _kustoClient = KustoIngestFactory.CreateDirectIngestClient(kustoConnectionStringBuilderDM); }
static async Task Main(string[] args) { // Async Ingestion From a Single Azure Blob using KustoQueuedIngestClient with (optional) RetryPolicy: //Create Kusto connection string with App Authentication var kustoConnectionStringBuilderDM = new KustoConnectionStringBuilder(@"https://ingest-{clusterNameAndRegion}.kusto.windows.net").WithAadApplicationKeyAuthentication( applicationClientId: "{Application Client ID}", applicationKey: "{Application Key (secret)}", authority: "{AAD TenantID or name}"); // Create an ingest client // Note, that creating a separate instance per ingestion operation is an anti-pattern. // IngestClient classes are thread-safe and intended for reuse IKustoIngestClient client = KustoIngestFactory.CreateQueuedIngestClient(kustoConnectionStringBuilderDM); // Ingest from blobs according to the required properties var kustoIngestionProperties = new KustoIngestionProperties(databaseName: "myDB", tableName: "myTable"); var sourceOptions = new StorageSourceOptions() { DeleteSourceOnSuccess = true }; //// Create your custom implementation of IRetryPolicy, which will affect how the ingest client handles retrying on transient failures IRetryPolicy retryPolicy = new NoRetry(); //// This line sets the retry policy on the ingest client that will be enforced on every ingest call from here on ((IKustoQueuedIngestClient)client).QueueRetryPolicy = retryPolicy; await client.IngestFromStorageAsync(uri : @"BLOB-URI-WITH-SAS-KEY", ingestionProperties : kustoIngestionProperties, sourceOptions); client.Dispose(); }
/// <summary> /// Constructor which gets ready to make queries to Kusto /// </summary> /// <param name="kustoConnectionStringBuilder">The connection string builder to connect to Kusto</param> public QueryEngine(KustoConnectionStringBuilder kustoConnectionStringBuilder) { _adminClient = KustoClientFactory.CreateCslAdminProvider(kustoConnectionStringBuilder); _queryClient = KustoClientFactory.CreateCslQueryProvider(kustoConnectionStringBuilder); _databaseName = kustoConnectionStringBuilder.InitialCatalog; _cluster = kustoConnectionStringBuilder.DataSource; }
} // = new Timer(DisposeClient, kustoQueryClient, maxKustoClientTimeMs, maxKustoClientTimeMs); public void Authenticate(bool throwOnError = false, bool prompt = false) { if (Config.IsKustoConfigured() && _arm.Authenticate(throwOnError, ClusterIngestUrl, prompt ? PromptBehavior.Auto : PromptBehavior.Never)) { DatabaseConnection = new KustoConnectionStringBuilder(ClusterIngestUrl) { FederatedSecurity = true, InitialCatalog = DatabaseName, UserToken = _arm.BearerToken }; ManagementConnection = new KustoConnectionStringBuilder(ManagementUrl) { FederatedSecurity = true, InitialCatalog = DatabaseName, UserToken = _arm.BearerToken }; } else { DatabaseConnection = new KustoConnectionStringBuilder(ClusterIngestUrl) { FederatedSecurity = true, InitialCatalog = DatabaseName }; ManagementConnection = new KustoConnectionStringBuilder(ManagementUrl) { FederatedSecurity = true, InitialCatalog = DatabaseName }; } IdentityToken = RetrieveKustoIdentityToken(); IngestionResources = RetrieveIngestionResources(); }
/// <summary> /// Get kusto query record into Datatable. /// </summary> /// <returns>Datatable</returns> public DataTable ExecuteQueryOnKusto(KustoConnectionStringBuilder kustoConnectionStringBuilder, string query) { Console.WriteLine("\n[System]: In Kusto Execute Query to get DT"); DataTable dt = new DataTable(); if (kustoConnectionStringBuilder != null && !string.IsNullOrWhiteSpace(query)) { try { var clients = KustoClientFactory.CreateCslQueryProvider(kustoConnectionStringBuilder); using (var record = clients.ExecuteQuery(query)) { if (record != null) { dt.Load(record); } } } catch (Exception ex) { Console.WriteLine("\n[System]: Exception :" + ex.Message); } } return(dt); }
public static Tuple <KustoConnectionStringBuilder, KustoConnectionStringBuilder> GetKustoConnectionStrings( string authority, string clusterAddress, string database, string appClientId, string appKey) { KustoConnectionStringBuilder kscbAdmin = null; KustoConnectionStringBuilder kscbIngest = null; if (!string.IsNullOrEmpty(authority)) { if (!string.IsNullOrEmpty(appClientId) && !string.IsNullOrEmpty(appKey)) { kscbIngest = new KustoConnectionStringBuilder($"https://ingest-{clusterAddress}", database).WithAadApplicationKeyAuthentication(appClientId, appKey, authority); kscbAdmin = new KustoConnectionStringBuilder($"https://{clusterAddress}", database).WithAadApplicationKeyAuthentication(appClientId, appKey, authority); } #if NET462 else { kscbIngest = new KustoConnectionStringBuilder($"https://ingest-{clusterAddress}", database).WithAadUserPromptAuthentication(authority); kscbAdmin = new KustoConnectionStringBuilder($"https://{clusterAddress}", database).WithAadUserPromptAuthentication(authority); } #endif } return(new Tuple <KustoConnectionStringBuilder, KustoConnectionStringBuilder>(kscbIngest, kscbAdmin)); }
public Monitor(Configuration configuration) { Contract.RequiresNotNull(configuration); _configuration = configuration; _logger = CreateLogger(); // TODO(jubayard): use streaming ingestion instead of direct ingestion. There seems to be some assembly // issues when attempting to do that Contract.RequiresNotNullOrEmpty(_configuration.KustoIngestionClusterUrl); Contract.RequiresNotNullOrEmpty(_configuration.ApplicationClientId); Contract.RequiresNotNullOrEmpty(_configuration.ApplicationKey); Contract.RequiresNotNullOrEmpty(_configuration.Authority); var kustoIngestConnectionString = new KustoConnectionStringBuilder(_configuration.KustoIngestionClusterUrl) .WithAadApplicationKeyAuthentication(_configuration.ApplicationClientId, _configuration.ApplicationKey, _configuration.Authority); _kustoIngestClient = KustoIngestFactory.CreateDirectIngestClient(kustoIngestConnectionString); var kustoConnectionString = new KustoConnectionStringBuilder(_configuration.KustoClusterUrl) .WithAadApplicationKeyAuthentication(_configuration.ApplicationClientId, _configuration.ApplicationKey, _configuration.Authority); _cslQueryProvider = KustoClientFactory.CreateCslQueryProvider(kustoConnectionString); Contract.RequiresNotNull(_configuration.KustoNotifier); _alertNotifier = new KustoWriter <Notification>(_configuration.KustoNotifier, _logger, _kustoIngestClient); Contract.RequiresNotNull(_configuration.SchedulerKustoNotifier); _schedulerLogWriter = new KustoWriter <Scheduler.LogEntry>(_configuration.SchedulerKustoNotifier, _logger, _kustoIngestClient); _scheduler = new Scheduler(_configuration.Scheduler, _logger, _clock, _schedulerLogWriter); }
private static void UploadEvtxFiles( string _filePattern, string _queryFile, string consoleLogOption, string _outputFileName, string blobConnectionString, string blobContainerName, KustoConnectionStringBuilder kscbAdmin, KustoConnectionStringBuilder kscbIngest, bool _demoMode, string _tableName, bool _resetTable) { string[] files; if (Path.IsPathRooted(_filePattern)) { string dir = Path.GetDirectoryName(Path.GetFullPath(_filePattern)); string pattern = Path.GetFileName(_filePattern); files = Directory.GetFiles(dir, pattern); } else { // input string rootDir = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); // Get directory and file parts of complete relative pattern string pattern = Path.GetFileName(_filePattern); string relDir = pattern.Substring(0, _filePattern.Length - pattern.Length); // Get absolute path (root+relative) string absPath = Path.GetFullPath(Path.Combine(rootDir, relDir)); // Search files mathing the pattern files = Directory.GetFiles(absPath, pattern, SearchOption.TopDirectoryOnly); } if (files != null && files.Length > 0) { var etw = EvtxAsDictionaryObservable.FromFiles(files); if (kscbAdmin != null) { // output to kusto var ku = CreateUploader(UploadTimespan, blobConnectionString, blobContainerName, kscbAdmin, kscbIngest, _demoMode, _tableName, _resetTable); RunUploader(ku, etw, _queryFile); } else if (!string.IsNullOrEmpty(_outputFileName)) { // output to file var fileOutput = new FileOutput(_outputFileName); RunFileOutput(fileOutput, etw, _queryFile); } else { // output to console bool tableFormat = consoleLogOption == "table" ? true : false; var consoleOutput = new ConsoleOutput(tableFormat); RunConsoleOutput(consoleOutput, etw, _queryFile); } } }
public KustoHandler(string tenantId, string token, string url) { KustoConnectionString = new KustoConnectionStringBuilder(url, tenantId) { UserID = tenantId, Password = token }; }
public static void CreateTableMapping(string databaseName, string table, string tableMappingName, KustoConnectionStringBuilder kustoConnectionStringBuilder) { using (var kustoClient = KustoClientFactory.CreateCslAdminProvider(kustoConnectionStringBuilder)) { var command = CslCommandGenerator.GenerateTableMappingCreateCommand( Kusto.Data.Ingestion.IngestionMappingKind.Csv, table, tableMappingName, new[] { new ColumnMapping() { ColumnName = "id", Properties = new Dictionary <string, string>() { { MappingConsts.Ordinal, "0" } } }, new ColumnMapping() { ColumnName = "date", Properties = new Dictionary <string, string>() { { MappingConsts.Ordinal, "1" } } }, new ColumnMapping() { ColumnName = "time", Properties = new Dictionary <string, string>() { { MappingConsts.Ordinal, "2" } } }, new ColumnMapping() { ColumnName = "sym", Properties = new Dictionary <string, string>() { { MappingConsts.Ordinal, "3" } } }, new ColumnMapping() { ColumnName = "qty", Properties = new Dictionary <string, string>() { { MappingConsts.Ordinal, "4" } } }, new ColumnMapping() { ColumnName = "px", Properties = new Dictionary <string, string>() { { MappingConsts.Ordinal, "5" } } } }); kustoClient.ExecuteControlCommand(databaseName, command); } }
public AdxOutput( BaseLogger logger, string authority, string appclientId, string appKey, string cluster, string database, string table, bool createOrResetTable = false, bool directIngest = false) { _logger = logger; _table = table; _createOrResetTable = createOrResetTable; _batchSize = 10000; _flushDuration = TimeSpan.FromMilliseconds(5); _lastUploadTime = DateTime.UtcNow; _initializeTable = false; _nextBatch = new List <IDictionary <string, object> >(); Completed = new AutoResetEvent(false); // Setting up kusto connection if (!string.IsNullOrEmpty(authority)) { if (!string.IsNullOrEmpty(appclientId) && !string.IsNullOrEmpty(appKey)) { kscbIngest = new KustoConnectionStringBuilder($"https://ingest-{cluster}", database).WithAadApplicationKeyAuthentication(appclientId, appKey, authority); kscbAdmin = new KustoConnectionStringBuilder($"https://{cluster}", database).WithAadApplicationKeyAuthentication(appclientId, appKey, authority); } else { kscbIngest = new KustoConnectionStringBuilder($"https://ingest-{cluster}", database).WithAadUserPromptAuthentication(authority); kscbAdmin = new KustoConnectionStringBuilder($"https://{cluster}", database).WithAadUserPromptAuthentication(authority); } } if (kscbAdmin != null) { _ingestionProperties = new KustoIngestionProperties(kscbIngest.InitialCatalog, table); if (directIngest) { _ingestClient = KustoIngestFactory.CreateDirectIngestClient(this.kscbAdmin); } else { _ingestClient = KustoIngestFactory.CreateQueuedIngestClient(kscbIngest); } } else { _logger.Log(LogLevel.ERROR, "ERROR getting ADX connection strings. Please double check the information provided."); _error = true; } }
static async Task Main(string[] args) { // Ingest From a Local File using KustoQueuedIngestClient and report status to a table // Create Kusto connection string with App Authentication var kustoConnectionStringBuilderDM = new KustoConnectionStringBuilder(@"https://ingest-{clusterNameAndRegion}.kusto.windows.net").WithAadApplicationKeyAuthentication( applicationClientId: "{Application Client ID}", applicationKey: "{Application Key (secret)}", authority: "{AAD TenantID or name}"); // Create a disposable client that will execute the ingestion IKustoQueuedIngestClient client = KustoIngestFactory.CreateQueuedIngestClient(kustoConnectionStringBuilderDM); // Ingest from a file according to the required properties var kustoIngestionProperties = new KustoQueuedIngestionProperties(databaseName: "myDB", tableName: "myDB") { // Setting the report level to FailuresAndSuccesses will cause both successful and failed ingestions to be reported // (Rather than the default "FailuresOnly" level) ReportLevel = IngestionReportLevel.FailuresAndSuccesses, // Choose the report method of choice ReportMethod = IngestionReportMethod.Table }; var filePath = @"< Path to file >"; var fileIdentifier = Guid.NewGuid(); var fileDescription = new FileDescription() { FilePath = filePath, SourceId = fileIdentifier }; var sourceOptions = new StorageSourceOptions() { SourceId = fileDescription.SourceId.Value }; // Execute the ingest operation and save the result. var clientResult = await client.IngestFromStorageAsync(fileDescription.FilePath, ingestionProperties : kustoIngestionProperties, sourceOptions); // Use the fileIdentifier you supplied to get the status of your ingestion var ingestionStatus = clientResult.GetIngestionStatusBySourceId(fileIdentifier); while (ingestionStatus.Status == Status.Pending) { // Wait a minute... Thread.Sleep(TimeSpan.FromMinutes(1)); // Try again ingestionStatus = clientResult.GetIngestionStatusBySourceId(fileIdentifier); } // Verify the results of the ingestion Ensure.ConditionIsMet(ingestionStatus.Status == Status.Succeeded, "The file should have been ingested successfully"); // Dispose of the client client.Dispose(); }
public static void Run([CosmosDBTrigger( databaseName: "data", collectionName: "device", ConnectionStringSetting = "rbdemocosmosdb_DOCUMENTDB", LeaseCollectionName = "leases", CreateLeaseCollectionIfNotExists = true)] IReadOnlyList <Document> input, ILogger log) { if (input != null && input.Count > 0) { log.LogInformation("Documents modified " + input.Count); log.LogInformation("First document Id " + input[0].Id); //input[0].ToString() is the data that needs to be ingested. var tenantId = System.Environment.GetEnvironmentVariable($"tenantId"); var applicationClientId = System.Environment.GetEnvironmentVariable($"applicationClientId"); var applicationKey = System.Environment.GetEnvironmentVariable($"applicationKey"); var serviceNameAndRegion = System.Environment.GetEnvironmentVariable($"serviceNameAndRegion"); var kustoDatabase = "device"; var kustoTableName = "Staging"; var mappingName = "deviceMapping"; var kustoIngestConnectionStringBuilder = new KustoConnectionStringBuilder($"https://ingest-{serviceNameAndRegion}.kusto.windows.net") { FederatedSecurity = true, InitialCatalog = kustoDatabase, ApplicationClientId = applicationClientId, ApplicationKey = applicationKey, Authority = tenantId }; using (var ingestClient = KustoIngestFactory.CreateQueuedIngestClient(kustoIngestConnectionStringBuilder)) { var properties = new KustoQueuedIngestionProperties(kustoDatabase, kustoTableName) { Format = DataSourceFormat.json, JSONMappingReference = mappingName, ReportLevel = IngestionReportLevel.FailuresAndSuccesses, ReportMethod = IngestionReportMethod.Queue }; string inputValue = input[0].ToString().Replace("\r\n", ""); //string inputValue = "{ \"Id\":\"1\", \"Timestamp\":\"2\", \"Message\":\"This is a test message\" }"; //object val = Newtonsoft.Json.JsonConvert.DeserializeObject(inputValue); var stream = new System.IO.MemoryStream(System.Text.Encoding.UTF8.GetBytes(inputValue)); stream.Seek(0, SeekOrigin.Begin); // Post ingestion message ingestClient.IngestFromStream(stream, properties, leaveOpen: true); // Wait and retrieve all notifications // - Actual duration should be decided based on the effective Ingestion Batching Policy set on the table/database Thread.Sleep(10000); var errors = ingestClient.GetAndDiscardTopIngestionFailures().GetAwaiter().GetResult(); var successes = ingestClient.GetAndDiscardTopIngestionSuccesses().GetAwaiter().GetResult(); errors.ForEach((f) => { Console.WriteLine($"Ingestion error: {f.Info.Details}"); }); successes.ForEach((s) => { Console.WriteLine($"Ingested: {s.Info.IngestionSourcePath}"); }); } } }
internal KustoDataService() { KustoConnectionStringBuilder kcsb = new KustoConnectionStringBuilder($"https://ingest-{ServiceNameAndRegion}.kusto.windows.net") .WithAadManagedIdentity("system"); DirectJsonMappingResolver directJsonMappingResolver = new DirectJsonMappingResolver(); _httpRequestLogsTable = new KustoTable <HttpRequestLogEntry>(kcsb, DatabaseName, "UrlAccessLogs", directJsonMappingResolver); _scriptExecutionLogsTable = new KustoTable <ScriptExecutionLogEntry>(kcsb, DatabaseName, "ScriptExecLogs", directJsonMappingResolver); }
private static void ValidateIngestion(string databaseName, string table, KustoConnectionStringBuilder kustoConnectionStringBuilder) { using (var cslQueryProvider = KustoClientFactory.CreateCslQueryProvider(kustoConnectionStringBuilder)) { var query = $"{table} | count"; var results = cslQueryProvider.ExecuteQuery <long>(databaseName, query); Console.WriteLine(results.Single()); } }
public AdxClientTool() { connection = new KustoConnectionStringBuilder(Properties.Connection.Default.kustoURL).WithAadApplicationKeyAuthentication( applicationClientId: Properties.Connection.Default.appClientId, applicationKey: Properties.Connection.Default.appClientSecret, authority: Properties.Connection.Default.appAadTenantId); adx = KustoClientFactory.CreateCslQueryProvider(connection); }