public Monitor(Configuration configuration)
{
Contract.RequiresNotNull(configuration);
_configuration = configuration;
_logger = CreateLogger();
Contract.RequiresNotNullOrEmpty(_configuration.KustoIngestionClusterUrl);
Contract.RequiresNotNullOrEmpty(_configuration.ApplicationClientId);
Contract.RequiresNotNullOrEmpty(_configuration.ApplicationKey);
Contract.RequiresNotNullOrEmpty(_configuration.Authority);
var kustoIngestConnectionString = new KustoConnectionStringBuilder(_configuration.KustoIngestionClusterUrl)
.WithAadApplicationKeyAuthentication(_configuration.ApplicationClientId, _configuration.ApplicationKey, _configuration.Authority);
_kustoIngestClient = KustoIngestFactory.CreateDirectIngestClient(kustoIngestConnectionString);
var kustoConnectionString = new KustoConnectionStringBuilder(_configuration.KustoClusterUrl)
.WithAadApplicationKeyAuthentication(_configuration.ApplicationClientId, _configuration.ApplicationKey, _configuration.Authority);
_cslQueryProvider = KustoClientFactory.CreateCslQueryProvider(kustoConnectionString);
Contract.RequiresNotNull(_configuration.KustoNotifier);
_alertNotifier = new KustoWriter <Notification>(_configuration.KustoNotifier, _logger, _kustoIngestClient);
Contract.RequiresNotNull(_configuration.SchedulerKustoNotifier);
_schedulerLogWriter = new KustoWriter <Scheduler.LogEntry>(_configuration.SchedulerKustoNotifier, _logger, _kustoIngestClient);
_scheduler = new Scheduler(_configuration.Scheduler, _logger, _clock, _schedulerLogWriter);
}
public BlockingKustoUploader(
KustoConnectionStringBuilder kscb,
string tableName,
int batchSize,
TimeSpan flushDuration)
{
csb = kscb;
TableName = TableName;
BatchSize = batchSize;
_flushDuration = flushDuration;
_lastUploadTime = DateTime.Now;
Completed = new AutoResetEvent(false);
_ingestionProperties = new KustoIngestionProperties(csb.InitialCatalog, tableName);
_fields = typeof(EtwEvent).GetFields().Select(f => f.Name).ToArray();
if (csb.DataSource.StartsWith("https://ingest-"))
{
_ingestClient = KustoIngestFactory.CreateQueuedIngestClient(csb);
}
else
{
_ingestClient = KustoIngestFactory.CreateDirectIngestClient(csb);
}
_nextBatch = new List <T>();
}
private static string CreateADXTable(string databaseName, string table,
KustoConnectionStringBuilder kustoConnectionStringBuilder)
{
var command = "";
using (var kustoClient = KustoClientFactory.CreateCslAdminProvider(kustoConnectionStringBuilder))
{
command =
CslCommandGenerator.GenerateTableCreateCommand(
table,
new[]
{
Tuple.Create("id", "System.Int32"),
Tuple.Create("date", "System.DateTime"),
Tuple.Create("time", "System.DateTime"),
Tuple.Create("sym", "System.String"),
Tuple.Create("qty", "System.Double"),
Tuple.Create("px", "System.Double")
});
kustoClient.ExecuteControlCommand(databaseName, command);
//if (!isBatch)
//{
// var tablePolicyAlterCommand =
// CslCommandGenerator.GenerateTableAlterStreamingIngestionPolicyCommand(table, isEnabled: true);
// kustoClient.ExecuteControlCommand(databaseName, tablePolicyAlterCommand);
//}
return(command);
// kustoClient.ExecuteControlCommand(databaseName, ".create table StreamingDataTable (['id']:int)");
}
}
} // - function Read
private static void InitializeKustoClient(ILogger log)
{
lock (Lock)
{
if (!_isInitialized)
{
log.LogInformation("[PrometheusRead] [InitializeKustoClient] Initialize");
var connection =
new KustoConnectionStringBuilder(
Environment.GetEnvironmentVariable("kustoUrl", EnvironmentVariableTarget.Process))
.WithAadApplicationKeyAuthentication(
applicationClientId: Environment.GetEnvironmentVariable("appClientId",
EnvironmentVariableTarget.Process),
applicationKey: Environment.GetEnvironmentVariable("appClientSecret",
EnvironmentVariableTarget.Process),
authority: Environment.GetEnvironmentVariable("tenantId",
EnvironmentVariableTarget.Process));
_adx = KustoClientFactory.CreateCslQueryProvider(connection);
_isInitialized = true;
}
log.LogInformation("[PrometheusRead] [InitializeKustoClient] Ok");
}
} // - InitializeKustoClient
static void Main(string[] args)
{
var builder = new KustoConnectionStringBuilder("https://masvaas.kusto.windows.net/")
.WithAadUserPromptAuthentication();
using (var adminProvider = KustoClientFactory.CreateCslAdminProvider(builder))
{
var command = CslCommandGenerator.GenerateDatabasesShowCommand();
var objectReader = new ObjectReader <DatabasesShowCommandResult>(adminProvider.ExecuteControlCommand(command));
foreach (var temp in objectReader)
{
var db = temp.DatabaseName;
var databaseJournalCommand = CslCommandGenerator.GenerateDatabaseJournalShowCommand(db);
databaseJournalCommand += " | where Event == 'ADD-DATABASE' | project EventTimestamp";
using (var journalCmdResult = adminProvider.ExecuteControlCommand(db, databaseJournalCommand))
{
if (journalCmdResult.Read() && DateTime.TryParse(journalCmdResult["EventTimestamp"].ToString(), out var createdTime))
{
//ValhallaLogger.LogInformationalMessage(operationId, nameof(DatabaseInfoProvider), $"Database {database} Created time {createdTime.ToUniversalTime():o}");
Console.WriteLine($"Database: {db}, CreationTime: {createdTime}");
}
}
}
}
}
public BigBrotherUseKustoTest(ITestOutputHelper output)
{
_output = output;
_kustoName = Environment.GetEnvironmentVariable("kusto_name");
_kustoLocation = Environment.GetEnvironmentVariable("kusto_location");
_kustoDatabase = Environment.GetEnvironmentVariable("kusto_database");
_kustoTenantId = Environment.GetEnvironmentVariable("kusto_tenant_id");
if (_kustoName != null && _kustoLocation != null && _kustoDatabase != null && _kustoTenantId != null)
{
var kustoUri = $"https://{_kustoName}.{_kustoLocation}.kusto.windows.net";
var token = new AzureServiceTokenProvider().GetAccessTokenAsync(kustoUri, _kustoTenantId).Result;
var kustoConnectionString =
new KustoConnectionStringBuilder(kustoUri)
{
FederatedSecurity = true,
InitialCatalog = _kustoDatabase,
Authority = _kustoTenantId,
ApplicationToken = token
};
_kustoQueryClient = KustoClientFactory.CreateCslQueryProvider(kustoConnectionString);
_kustoAdminClient = KustoClientFactory.CreateCslAdminProvider(kustoConnectionString);
}
}
static void Main(string[] args)
{
// Ingest From Local Files using KustoQueuedIngestClient and Ingestion Validation
// Create Kusto connection string with App Authentication
var kustoConnectionStringBuilderDM =
new KustoConnectionStringBuilder(@"https://ingest-{clusterNameAndRegion}.kusto.windows.net").WithAadApplicationKeyAuthentication(
applicationClientId: "{Application Client ID}",
applicationKey: "{Application Key (secret)}",
authority: "{AAD TenantID or name}");
// Create a disposable client that will execute the ingestion
IKustoQueuedIngestClient client = KustoIngestFactory.CreateQueuedIngestClient(kustoConnectionStringBuilderDM);
// Ingest from files according to the required properties
var kustoIngestionProperties = new KustoIngestionProperties(databaseName: "myDB", tableName: "myTable");
client.IngestFromStorageAsync(@"ValidTestFile.csv", kustoIngestionProperties);
client.IngestFromStorageAsync(@"InvalidTestFile.csv", kustoIngestionProperties);
// Waiting for the aggregation
Thread.Sleep(TimeSpan.FromMinutes(8));
// Retrieve and validate failures
var ingestionFailures = client.PeekTopIngestionFailures().GetAwaiter().GetResult();
Ensure.IsTrue((ingestionFailures.Count() > 0), "Failures expected");
// Retrieve, delete and validate failures
ingestionFailures = client.GetAndDiscardTopIngestionFailures().GetAwaiter().GetResult();
Ensure.IsTrue((ingestionFailures.Count() > 0), "Failures expected");
// Dispose of the client
client.Dispose();
}
static void UploadRealTime(
string _sessionName,
string _queryFile,
string _outputFileName,
KustoConnectionStringBuilder kscbAdmin,
KustoConnectionStringBuilder kscbIngest,
bool _demoMode,
string _tableName,
bool _resetTable)
{
var etw = Tx.Windows.EtwTdhObservable.FromSession(_sessionName);
Console.WriteLine();
Console.WriteLine("Listening to real-time session '{0}'. Press Enter to terminate", _sessionName);
var ku = CreateUploader(UploadTimespan, _outputFileName, kscbAdmin, kscbIngest, _demoMode, _tableName, _resetTable);
Task task = Task.Factory.StartNew(() =>
{
RunUploader(ku, etw, _queryFile);
});
string readline = Console.ReadLine();
ku.OnCompleted();
}
public static void CreateTableMappingFromDefinition(string databaseName, string table,
string tableMappingName, KustoConnectionStringBuilder kustoConnectionStringBuilder,
IDictionary <string, string> tableDefinition)
{
using (var kustoClient = KustoClientFactory.CreateCslAdminProvider(kustoConnectionStringBuilder))
{
var columnMappings = new List <ColumnMapping>();
int cnt = 0;
foreach (var keyvaluepair in tableDefinition)
{
columnMappings.Add(new ColumnMapping()
{
ColumnName = keyvaluepair.Key,
Properties = new Dictionary <string, string>()
{
{ MappingConsts.Ordinal, cnt.ToString() }
}
});
cnt++;
}
var command =
CslCommandGenerator.GenerateTableMappingCreateCommand(
Kusto.Data.Ingestion.IngestionMappingKind.Csv,
table,
tableMappingName, columnMappings);
kustoClient.ExecuteControlCommand(databaseName, command);
}
}
static void MainImpl(Arguments arguments)
{
// 1. Create a connection string to a cluster/database with AAD user authentication
var cluster = "https://help.kusto.windows.net/";
var database = "Samples";
var kcsb = new KustoConnectionStringBuilder(cluster, database)
{
FederatedSecurity = true
};
// 2. Connect to the Kusto query endpoint and create a query provider
using (var queryProvider = KustoClientFactory.CreateCslQueryProvider(kcsb))
{
// 3. Send a query using the V2 API
var query = "print Welcome='Hello, World!'; print PI=pi()";
var properties = new ClientRequestProperties()
{
ClientRequestId = "kusto_samples_query_v2;" + Guid.NewGuid().ToString()
};
if (arguments.ProgressiveMode)
{
properties.SetOption(ClientRequestProperties.OptionResultsProgressiveEnabled, true);
}
var queryTask = queryProvider.ExecuteQueryV2Async(database, query, properties);
// 4. Parse and print the results of the query
WriteResultsToConsole(queryTask);
}
}
private static ICslQueryProvider CreateClient(AppConfigHelper appConfigHelper)
{
string adxClustorName = appConfigHelper.GetValue(ADXClusterNameKey);
string location = appConfigHelper.GetValue(LocationKey);
string applicationId = appConfigHelper.GetValue(ApplicationIdKey);
string applicationSecret = appConfigHelper.GetValue(ApplicationSecretKey);
string appTenant = appConfigHelper.GetValue(AppTenantKey);
try
{
var kustoUri = $"https://{adxClustorName}.{location}.kusto.windows.net/";
var kustoConnectionStringBuilder = new KustoConnectionStringBuilder(kustoUri)
.WithAadApplicationKeyAuthentication(
applicationClientId: applicationId,
applicationKey: applicationSecret,
authority: appTenant);
return KustoClientFactory.CreateCslQueryProvider(kustoConnectionStringBuilder);
}
catch (Exception ex)
{
throw new ApplicationException("DocumentClient creation failed in the helper class", ex);
}
}
private ICslAdminProvider GetKustoAdminClient()
{
if (this.client == null)
{
try
{
var kustoUri = $"https://{this.config.Global.DataExplorer.Name}.{this.config.Global.Location}.kusto.windows.net/";
var kustoConnectionStringBuilder = new KustoConnectionStringBuilder(kustoUri)
.WithAadApplicationKeyAuthentication(
applicationClientId: this.config.Global.AzureActiveDirectory.AppId,
applicationKey: this.config.Global.AzureActiveDirectory.AppSecret,
authority: this.config.Global.AzureActiveDirectory.TenantId);
this.client = KustoClientFactory.CreateCslAdminProvider(kustoConnectionStringBuilder);
}
catch (Exception e)
{
var msg = "Unable to retrieve kusto with Active Directory properties";
throw new InvalidConfigurationException(msg, e);
}
if (this.client == null)
{
// this.logger.LogError(new Exception(errorMessage), errorMessage);
throw new InvalidConfigurationException("Could not connect to kusto client");
}
}
return(this.client);
}
static void UploadSyslogRealTime(
string listenerAdapterName,
int listenerUdpPort,
string queryFile,
string outputFileName,
KustoConnectionStringBuilder kscbAdmin,
KustoConnectionStringBuilder kscbIngest,
bool quickIngest,
string tableName,
bool resetTable)
{
var parser = CreateSIEMfxSyslogParser();
IPAddress localIp = null;
if (!string.IsNullOrEmpty(listenerAdapterName))
{
localIp = GetLocalIp(listenerAdapterName);
}
localIp ??= IPAddress.IPv6Any;
var endPoint = new IPEndPoint(localIp, listenerUdpPort);
var PortListener = new UdpClient(AddressFamily.InterNetworkV6);
PortListener.Client.DualMode = true;
PortListener.Client.Bind(endPoint);
PortListener.Client.ReceiveBufferSize = 10 * 1024 * 1024;
using var listener = new SyslogListener(parser, PortListener);
var filter = new SyslogFilter();
if (filter != null)
{
listener.Filter = filter.Allow;
}
listener.Error += Listener_Error;
listener.EntryReceived += Listener_EntryReceived;
var _converter = new SyslogEntryToRecordConverter();
listener.Subscribe(_converter);
listener.Start();
Console.WriteLine();
Console.WriteLine("Listening to Syslog events. Press any key to terminate");
var ku = CreateUploader(UploadTimespan, outputFileName, kscbAdmin, kscbIngest, quickIngest, tableName, resetTable);
Task task = Task.Factory.StartNew(() =>
{
RunUploader(ku, _converter, queryFile);
});
string readline = Console.ReadLine();
listener.Stop();
ku.OnCompleted();
}
private static string CreateADXTableFromDefinition(string databaseName, string table,
KustoConnectionStringBuilder kustoConnectionStringBuilder, IDictionary <string, string> tableDefinition)
{
var command = "";
var tuple = new Tuple <string, string> [tableDefinition.Count()];
int cnt = 0;
foreach (var keyvaluepair in tableDefinition)
{
tuple[cnt] = new Tuple <string, string>(keyvaluepair.Key, keyvaluepair.Value);
cnt++;
}
using (var kustoClient = KustoClientFactory.CreateCslAdminProvider(kustoConnectionStringBuilder))
{
command =
CslCommandGenerator.GenerateTableCreateCommand(
table, tuple);
var tablePolicyAlterCommand = CslCommandGenerator.GenerateTableAlterStreamingIngestionPolicyCommand(table, isEnabled: true);
kustoClient.ExecuteControlCommand(databaseName, command);
kustoClient.ExecuteControlCommand(databaseName, tablePolicyAlterCommand);
}
return(command);
}
public KustoClientLogger(string databaseName, string tableName, List <KustoColumnMapping> _schema, string clientId, string clientKey)
{
var csvColumnMappings = new List <CsvColumnMapping>();
foreach (var mapping in _schema)
{
csvColumnMappings.Add(new CsvColumnMapping {
ColumnName = mapping.ColumnName, CslDataType = mapping.DataType, Ordinal = mapping.ColumnNumber
});
}
_kustoProperties = new KustoIngestionProperties(databaseName, tableName)
{
Format = DataSourceFormat.csv,
CSVMapping = csvColumnMappings
};
var kustoConnectionStringBuilderDM = new KustoConnectionStringBuilder(@"https://wawstest.kusto.windows.net:443")
{
FederatedSecurity = true,
InitialCatalog = databaseName,
ApplicationKey = clientKey,
ApplicationClientId = clientId
};
_kustoClient = KustoIngestFactory.CreateDirectIngestClient(kustoConnectionStringBuilderDM);
}
static async Task Main(string[] args)
{
// Async Ingestion From a Single Azure Blob using KustoQueuedIngestClient with (optional) RetryPolicy:
//Create Kusto connection string with App Authentication
var kustoConnectionStringBuilderDM =
new KustoConnectionStringBuilder(@"https://ingest-{clusterNameAndRegion}.kusto.windows.net").WithAadApplicationKeyAuthentication(
applicationClientId: "{Application Client ID}",
applicationKey: "{Application Key (secret)}",
authority: "{AAD TenantID or name}");
// Create an ingest client
// Note, that creating a separate instance per ingestion operation is an anti-pattern.
// IngestClient classes are thread-safe and intended for reuse
IKustoIngestClient client = KustoIngestFactory.CreateQueuedIngestClient(kustoConnectionStringBuilderDM);
// Ingest from blobs according to the required properties
var kustoIngestionProperties = new KustoIngestionProperties(databaseName: "myDB", tableName: "myTable");
var sourceOptions = new StorageSourceOptions()
{
DeleteSourceOnSuccess = true
};
//// Create your custom implementation of IRetryPolicy, which will affect how the ingest client handles retrying on transient failures
IRetryPolicy retryPolicy = new NoRetry();
//// This line sets the retry policy on the ingest client that will be enforced on every ingest call from here on
((IKustoQueuedIngestClient)client).QueueRetryPolicy = retryPolicy;
await client.IngestFromStorageAsync(uri : @"BLOB-URI-WITH-SAS-KEY", ingestionProperties : kustoIngestionProperties, sourceOptions);
client.Dispose();
}
/// <summary>
/// Constructor which gets ready to make queries to Kusto
/// </summary>
/// <param name="kustoConnectionStringBuilder">The connection string builder to connect to Kusto</param>
public QueryEngine(KustoConnectionStringBuilder kustoConnectionStringBuilder)
{
_adminClient = KustoClientFactory.CreateCslAdminProvider(kustoConnectionStringBuilder);
_queryClient = KustoClientFactory.CreateCslQueryProvider(kustoConnectionStringBuilder);
_databaseName = kustoConnectionStringBuilder.InitialCatalog;
_cluster = kustoConnectionStringBuilder.DataSource;
}
} // = new Timer(DisposeClient, kustoQueryClient, maxKustoClientTimeMs, maxKustoClientTimeMs);
public void Authenticate(bool throwOnError = false, bool prompt = false)
{
if (Config.IsKustoConfigured() && _arm.Authenticate(throwOnError, ClusterIngestUrl, prompt ? PromptBehavior.Auto : PromptBehavior.Never))
{
DatabaseConnection = new KustoConnectionStringBuilder(ClusterIngestUrl)
{
FederatedSecurity = true, InitialCatalog = DatabaseName, UserToken = _arm.BearerToken
};
ManagementConnection = new KustoConnectionStringBuilder(ManagementUrl)
{
FederatedSecurity = true, InitialCatalog = DatabaseName, UserToken = _arm.BearerToken
};
}
else
{
DatabaseConnection = new KustoConnectionStringBuilder(ClusterIngestUrl)
{
FederatedSecurity = true, InitialCatalog = DatabaseName
};
ManagementConnection = new KustoConnectionStringBuilder(ManagementUrl)
{
FederatedSecurity = true, InitialCatalog = DatabaseName
};
}
IdentityToken = RetrieveKustoIdentityToken();
IngestionResources = RetrieveIngestionResources();
}
/// <summary>
/// Get kusto query record into Datatable.
/// </summary>
/// <returns>Datatable</returns>
public DataTable ExecuteQueryOnKusto(KustoConnectionStringBuilder kustoConnectionStringBuilder, string query)
{
Console.WriteLine("\n[System]: In Kusto Execute Query to get DT");
DataTable dt = new DataTable();
if (kustoConnectionStringBuilder != null && !string.IsNullOrWhiteSpace(query))
{
try
{
var clients = KustoClientFactory.CreateCslQueryProvider(kustoConnectionStringBuilder);
using (var record = clients.ExecuteQuery(query))
{
if (record != null)
{
dt.Load(record);
}
}
}
catch (Exception ex)
{
Console.WriteLine("\n[System]: Exception :" + ex.Message);
}
}
return(dt);
}
public static Tuple <KustoConnectionStringBuilder, KustoConnectionStringBuilder> GetKustoConnectionStrings(
string authority,
string clusterAddress,
string database,
string appClientId,
string appKey)
{
KustoConnectionStringBuilder kscbAdmin = null;
KustoConnectionStringBuilder kscbIngest = null;
if (!string.IsNullOrEmpty(authority))
{
if (!string.IsNullOrEmpty(appClientId) && !string.IsNullOrEmpty(appKey))
{
kscbIngest = new KustoConnectionStringBuilder($"https://ingest-{clusterAddress}", database).WithAadApplicationKeyAuthentication(appClientId, appKey, authority);
kscbAdmin = new KustoConnectionStringBuilder($"https://{clusterAddress}", database).WithAadApplicationKeyAuthentication(appClientId, appKey, authority);
}
#if NET462
else
{
kscbIngest = new KustoConnectionStringBuilder($"https://ingest-{clusterAddress}", database).WithAadUserPromptAuthentication(authority);
kscbAdmin = new KustoConnectionStringBuilder($"https://{clusterAddress}", database).WithAadUserPromptAuthentication(authority);
}
#endif
}
return(new Tuple <KustoConnectionStringBuilder, KustoConnectionStringBuilder>(kscbIngest, kscbAdmin));
}
public Monitor(Configuration configuration)
{
Contract.RequiresNotNull(configuration);
_configuration = configuration;
_logger = CreateLogger();
// TODO(jubayard): use streaming ingestion instead of direct ingestion. There seems to be some assembly
// issues when attempting to do that
Contract.RequiresNotNullOrEmpty(_configuration.KustoIngestionClusterUrl);
Contract.RequiresNotNullOrEmpty(_configuration.ApplicationClientId);
Contract.RequiresNotNullOrEmpty(_configuration.ApplicationKey);
Contract.RequiresNotNullOrEmpty(_configuration.Authority);
var kustoIngestConnectionString = new KustoConnectionStringBuilder(_configuration.KustoIngestionClusterUrl)
.WithAadApplicationKeyAuthentication(_configuration.ApplicationClientId, _configuration.ApplicationKey, _configuration.Authority);
_kustoIngestClient = KustoIngestFactory.CreateDirectIngestClient(kustoIngestConnectionString);
var kustoConnectionString = new KustoConnectionStringBuilder(_configuration.KustoClusterUrl)
.WithAadApplicationKeyAuthentication(_configuration.ApplicationClientId, _configuration.ApplicationKey, _configuration.Authority);
_cslQueryProvider = KustoClientFactory.CreateCslQueryProvider(kustoConnectionString);
Contract.RequiresNotNull(_configuration.KustoNotifier);
_alertNotifier = new KustoWriter <Notification>(_configuration.KustoNotifier, _logger, _kustoIngestClient);
Contract.RequiresNotNull(_configuration.SchedulerKustoNotifier);
_schedulerLogWriter = new KustoWriter <Scheduler.LogEntry>(_configuration.SchedulerKustoNotifier, _logger, _kustoIngestClient);
_scheduler = new Scheduler(_configuration.Scheduler, _logger, _clock, _schedulerLogWriter);
}
private static void UploadEvtxFiles(
string _filePattern,
string _queryFile,
string consoleLogOption,
string _outputFileName,
string blobConnectionString,
string blobContainerName,
KustoConnectionStringBuilder kscbAdmin,
KustoConnectionStringBuilder kscbIngest,
bool _demoMode,
string _tableName,
bool _resetTable)
{
string[] files;
if (Path.IsPathRooted(_filePattern))
{
string dir = Path.GetDirectoryName(Path.GetFullPath(_filePattern));
string pattern = Path.GetFileName(_filePattern);
files = Directory.GetFiles(dir, pattern);
}
else
{
// input
string rootDir = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
// Get directory and file parts of complete relative pattern
string pattern = Path.GetFileName(_filePattern);
string relDir = pattern.Substring(0, _filePattern.Length - pattern.Length);
// Get absolute path (root+relative)
string absPath = Path.GetFullPath(Path.Combine(rootDir, relDir));
// Search files mathing the pattern
files = Directory.GetFiles(absPath, pattern, SearchOption.TopDirectoryOnly);
}
if (files != null && files.Length > 0)
{
var etw = EvtxAsDictionaryObservable.FromFiles(files);
if (kscbAdmin != null)
{
// output to kusto
var ku = CreateUploader(UploadTimespan, blobConnectionString, blobContainerName, kscbAdmin, kscbIngest, _demoMode, _tableName, _resetTable);
RunUploader(ku, etw, _queryFile);
}
else if (!string.IsNullOrEmpty(_outputFileName))
{
// output to file
var fileOutput = new FileOutput(_outputFileName);
RunFileOutput(fileOutput, etw, _queryFile);
}
else
{
// output to console
bool tableFormat = consoleLogOption == "table" ? true : false;
var consoleOutput = new ConsoleOutput(tableFormat);
RunConsoleOutput(consoleOutput, etw, _queryFile);
}
}
}
public KustoHandler(string tenantId, string token, string url)
{
KustoConnectionString = new KustoConnectionStringBuilder(url, tenantId)
{
UserID = tenantId,
Password = token
};
}
public static void CreateTableMapping(string databaseName, string table, string tableMappingName,
KustoConnectionStringBuilder kustoConnectionStringBuilder)
{
using (var kustoClient = KustoClientFactory.CreateCslAdminProvider(kustoConnectionStringBuilder))
{
var command =
CslCommandGenerator.GenerateTableMappingCreateCommand(
Kusto.Data.Ingestion.IngestionMappingKind.Csv,
table,
tableMappingName,
new[] {
new ColumnMapping()
{
ColumnName = "id", Properties = new Dictionary <string, string>()
{
{ MappingConsts.Ordinal, "0" }
}
},
new ColumnMapping()
{
ColumnName = "date", Properties = new Dictionary <string, string>()
{
{ MappingConsts.Ordinal, "1" }
}
},
new ColumnMapping()
{
ColumnName = "time", Properties = new Dictionary <string, string>()
{
{ MappingConsts.Ordinal, "2" }
}
},
new ColumnMapping()
{
ColumnName = "sym", Properties = new Dictionary <string, string>()
{
{ MappingConsts.Ordinal, "3" }
}
},
new ColumnMapping()
{
ColumnName = "qty", Properties = new Dictionary <string, string>()
{
{ MappingConsts.Ordinal, "4" }
}
},
new ColumnMapping()
{
ColumnName = "px", Properties = new Dictionary <string, string>()
{
{ MappingConsts.Ordinal, "5" }
}
}
});
kustoClient.ExecuteControlCommand(databaseName, command);
}
}
public AdxOutput(
BaseLogger logger,
string authority,
string appclientId,
string appKey,
string cluster,
string database,
string table,
bool createOrResetTable = false,
bool directIngest = false)
{
_logger = logger;
_table = table;
_createOrResetTable = createOrResetTable;
_batchSize = 10000;
_flushDuration = TimeSpan.FromMilliseconds(5);
_lastUploadTime = DateTime.UtcNow;
_initializeTable = false;
_nextBatch = new List <IDictionary <string, object> >();
Completed = new AutoResetEvent(false);
// Setting up kusto connection
if (!string.IsNullOrEmpty(authority))
{
if (!string.IsNullOrEmpty(appclientId) && !string.IsNullOrEmpty(appKey))
{
kscbIngest = new KustoConnectionStringBuilder($"https://ingest-{cluster}", database).WithAadApplicationKeyAuthentication(appclientId, appKey, authority);
kscbAdmin = new KustoConnectionStringBuilder($"https://{cluster}", database).WithAadApplicationKeyAuthentication(appclientId, appKey, authority);
}
else
{
kscbIngest = new KustoConnectionStringBuilder($"https://ingest-{cluster}", database).WithAadUserPromptAuthentication(authority);
kscbAdmin = new KustoConnectionStringBuilder($"https://{cluster}", database).WithAadUserPromptAuthentication(authority);
}
}
if (kscbAdmin != null)
{
_ingestionProperties = new KustoIngestionProperties(kscbIngest.InitialCatalog, table);
if (directIngest)
{
_ingestClient = KustoIngestFactory.CreateDirectIngestClient(this.kscbAdmin);
}
else
{
_ingestClient = KustoIngestFactory.CreateQueuedIngestClient(kscbIngest);
}
}
else
{
_logger.Log(LogLevel.ERROR, "ERROR getting ADX connection strings. Please double check the information provided.");
_error = true;
}
}
static async Task Main(string[] args)
{
// Ingest From a Local File using KustoQueuedIngestClient and report status to a table
// Create Kusto connection string with App Authentication
var kustoConnectionStringBuilderDM =
new KustoConnectionStringBuilder(@"https://ingest-{clusterNameAndRegion}.kusto.windows.net").WithAadApplicationKeyAuthentication(
applicationClientId: "{Application Client ID}",
applicationKey: "{Application Key (secret)}",
authority: "{AAD TenantID or name}");
// Create a disposable client that will execute the ingestion
IKustoQueuedIngestClient client = KustoIngestFactory.CreateQueuedIngestClient(kustoConnectionStringBuilderDM);
// Ingest from a file according to the required properties
var kustoIngestionProperties = new KustoQueuedIngestionProperties(databaseName: "myDB", tableName: "myDB")
{
// Setting the report level to FailuresAndSuccesses will cause both successful and failed ingestions to be reported
// (Rather than the default "FailuresOnly" level)
ReportLevel = IngestionReportLevel.FailuresAndSuccesses,
// Choose the report method of choice
ReportMethod = IngestionReportMethod.Table
};
var filePath = @"< Path to file >";
var fileIdentifier = Guid.NewGuid();
var fileDescription = new FileDescription()
{
FilePath = filePath, SourceId = fileIdentifier
};
var sourceOptions = new StorageSourceOptions()
{
SourceId = fileDescription.SourceId.Value
};
// Execute the ingest operation and save the result.
var clientResult = await client.IngestFromStorageAsync(fileDescription.FilePath,
ingestionProperties : kustoIngestionProperties, sourceOptions);
// Use the fileIdentifier you supplied to get the status of your ingestion
var ingestionStatus = clientResult.GetIngestionStatusBySourceId(fileIdentifier);
while (ingestionStatus.Status == Status.Pending)
{
// Wait a minute...
Thread.Sleep(TimeSpan.FromMinutes(1));
// Try again
ingestionStatus = clientResult.GetIngestionStatusBySourceId(fileIdentifier);
}
// Verify the results of the ingestion
Ensure.ConditionIsMet(ingestionStatus.Status == Status.Succeeded,
"The file should have been ingested successfully");
// Dispose of the client
client.Dispose();
}
public static void Run([CosmosDBTrigger(
databaseName: "data",
collectionName: "device",
ConnectionStringSetting = "rbdemocosmosdb_DOCUMENTDB",
LeaseCollectionName = "leases", CreateLeaseCollectionIfNotExists = true)] IReadOnlyList <Document> input, ILogger log)
{
if (input != null && input.Count > 0)
{
log.LogInformation("Documents modified " + input.Count);
log.LogInformation("First document Id " + input[0].Id);
//input[0].ToString() is the data that needs to be ingested.
var tenantId = System.Environment.GetEnvironmentVariable($"tenantId");
var applicationClientId = System.Environment.GetEnvironmentVariable($"applicationClientId");
var applicationKey = System.Environment.GetEnvironmentVariable($"applicationKey");
var serviceNameAndRegion = System.Environment.GetEnvironmentVariable($"serviceNameAndRegion");
var kustoDatabase = "device";
var kustoTableName = "Staging";
var mappingName = "deviceMapping";
var kustoIngestConnectionStringBuilder =
new KustoConnectionStringBuilder($"https://ingest-{serviceNameAndRegion}.kusto.windows.net")
{
FederatedSecurity = true,
InitialCatalog = kustoDatabase,
ApplicationClientId = applicationClientId,
ApplicationKey = applicationKey,
Authority = tenantId
};
using (var ingestClient = KustoIngestFactory.CreateQueuedIngestClient(kustoIngestConnectionStringBuilder))
{
var properties = new KustoQueuedIngestionProperties(kustoDatabase, kustoTableName)
{
Format = DataSourceFormat.json,
JSONMappingReference = mappingName,
ReportLevel = IngestionReportLevel.FailuresAndSuccesses,
ReportMethod = IngestionReportMethod.Queue
};
string inputValue = input[0].ToString().Replace("\r\n", "");
//string inputValue = "{ \"Id\":\"1\", \"Timestamp\":\"2\", \"Message\":\"This is a test message\" }";
//object val = Newtonsoft.Json.JsonConvert.DeserializeObject(inputValue);
var stream = new System.IO.MemoryStream(System.Text.Encoding.UTF8.GetBytes(inputValue));
stream.Seek(0, SeekOrigin.Begin);
// Post ingestion message
ingestClient.IngestFromStream(stream, properties, leaveOpen: true);
// Wait and retrieve all notifications
// - Actual duration should be decided based on the effective Ingestion Batching Policy set on the table/database
Thread.Sleep(10000);
var errors = ingestClient.GetAndDiscardTopIngestionFailures().GetAwaiter().GetResult();
var successes = ingestClient.GetAndDiscardTopIngestionSuccesses().GetAwaiter().GetResult();
errors.ForEach((f) => { Console.WriteLine($"Ingestion error: {f.Info.Details}"); });
successes.ForEach((s) => { Console.WriteLine($"Ingested: {s.Info.IngestionSourcePath}"); });
}
}
}
internal KustoDataService()
{
KustoConnectionStringBuilder kcsb = new KustoConnectionStringBuilder($"https://ingest-{ServiceNameAndRegion}.kusto.windows.net")
.WithAadManagedIdentity("system");
DirectJsonMappingResolver directJsonMappingResolver = new DirectJsonMappingResolver();
_httpRequestLogsTable = new KustoTable <HttpRequestLogEntry>(kcsb, DatabaseName, "UrlAccessLogs", directJsonMappingResolver);
_scriptExecutionLogsTable = new KustoTable <ScriptExecutionLogEntry>(kcsb, DatabaseName, "ScriptExecLogs", directJsonMappingResolver);
}
private static void ValidateIngestion(string databaseName, string table, KustoConnectionStringBuilder kustoConnectionStringBuilder)
{
using (var cslQueryProvider = KustoClientFactory.CreateCslQueryProvider(kustoConnectionStringBuilder))
{
var query = $"{table} | count";
var results = cslQueryProvider.ExecuteQuery <long>(databaseName, query);
Console.WriteLine(results.Single());
}
}
public AdxClientTool()
{
connection =
new KustoConnectionStringBuilder(Properties.Connection.Default.kustoURL).WithAadApplicationKeyAuthentication(
applicationClientId: Properties.Connection.Default.appClientId,
applicationKey: Properties.Connection.Default.appClientSecret,
authority: Properties.Connection.Default.appAadTenantId);
adx = KustoClientFactory.CreateCslQueryProvider(connection);
}