public static async Task <string> AdxExportStatusCheck(
[ActivityTrigger] string operationId, ILogger log)
{
using (var client = KustoClientFactory.CreateCslAdminProvider(await GetKustoConnectionStringBuilder()))
{
var operationQuery = CslCommandGenerator.GenerateOperationsShowCommand(Guid.Parse(operationId));
var resultReader = new ObjectReader <OperationsShowCommandResult>(client.ExecuteControlCommand(adxDatabaseName, operationQuery));
var res = resultReader?.FirstOrDefault();
var state = res?.State;
if (state == "Completed")
{
// When the state is completed, we can query the export details which contains the path to the file on blob storage
var operationDetailsQuery = CslCommandGenerator.GenerateOperationDetailsShowCommand(Guid.Parse(operationId));
var resultReader2 = new ObjectReader <DataExportToBlobCommandResult>(client.ExecuteControlCommand(adxDatabaseName, operationDetailsQuery));
var res2 = resultReader2?.FirstOrDefault();
var path = res2?.Path;
return(path);
}
else if (state == "Cancelled")
{
return("Error");
}
else
{
return(null);
}
}
}
public BigBrotherUseKustoTest(ITestOutputHelper output)
{
_output = output;
_kustoName = Environment.GetEnvironmentVariable("kusto_name");
_kustoLocation = Environment.GetEnvironmentVariable("kusto_location");
_kustoDatabase = Environment.GetEnvironmentVariable("kusto_database");
_kustoTenantId = Environment.GetEnvironmentVariable("kusto_tenant_id");
if (_kustoName != null && _kustoLocation != null && _kustoDatabase != null && _kustoTenantId != null)
{
var kustoUri = $"https://{_kustoName}.{_kustoLocation}.kusto.windows.net";
var token = new AzureServiceTokenProvider().GetAccessTokenAsync(kustoUri, _kustoTenantId).Result;
var kustoConnectionString =
new KustoConnectionStringBuilder(kustoUri)
{
FederatedSecurity = true,
InitialCatalog = _kustoDatabase,
Authority = _kustoTenantId,
ApplicationToken = token
};
_kustoQueryClient = KustoClientFactory.CreateCslQueryProvider(kustoConnectionString);
_kustoAdminClient = KustoClientFactory.CreateCslAdminProvider(kustoConnectionString);
}
}
} // - function Read
private static void InitializeKustoClient(ILogger log)
{
lock (Lock)
{
if (!_isInitialized)
{
log.LogInformation("[PrometheusRead] [InitializeKustoClient] Initialize");
var connection =
new KustoConnectionStringBuilder(
Environment.GetEnvironmentVariable("kustoUrl", EnvironmentVariableTarget.Process))
.WithAadApplicationKeyAuthentication(
applicationClientId: Environment.GetEnvironmentVariable("appClientId",
EnvironmentVariableTarget.Process),
applicationKey: Environment.GetEnvironmentVariable("appClientSecret",
EnvironmentVariableTarget.Process),
authority: Environment.GetEnvironmentVariable("tenantId",
EnvironmentVariableTarget.Process));
_adx = KustoClientFactory.CreateCslQueryProvider(connection);
_isInitialized = true;
}
log.LogInformation("[PrometheusRead] [InitializeKustoClient] Ok");
}
} // - InitializeKustoClient
public Monitor(Configuration configuration)
{
Contract.RequiresNotNull(configuration);
_configuration = configuration;
_logger = CreateLogger();
Contract.RequiresNotNullOrEmpty(_configuration.KustoIngestionClusterUrl);
Contract.RequiresNotNullOrEmpty(_configuration.ApplicationClientId);
Contract.RequiresNotNullOrEmpty(_configuration.ApplicationKey);
Contract.RequiresNotNullOrEmpty(_configuration.Authority);
var kustoIngestConnectionString = new KustoConnectionStringBuilder(_configuration.KustoIngestionClusterUrl)
.WithAadApplicationKeyAuthentication(_configuration.ApplicationClientId, _configuration.ApplicationKey, _configuration.Authority);
_kustoIngestClient = KustoIngestFactory.CreateDirectIngestClient(kustoIngestConnectionString);
var kustoConnectionString = new KustoConnectionStringBuilder(_configuration.KustoClusterUrl)
.WithAadApplicationKeyAuthentication(_configuration.ApplicationClientId, _configuration.ApplicationKey, _configuration.Authority);
_cslQueryProvider = KustoClientFactory.CreateCslQueryProvider(kustoConnectionString);
Contract.RequiresNotNull(_configuration.KustoNotifier);
_alertNotifier = new KustoWriter <Notification>(_configuration.KustoNotifier, _logger, _kustoIngestClient);
Contract.RequiresNotNull(_configuration.SchedulerKustoNotifier);
_schedulerLogWriter = new KustoWriter <Scheduler.LogEntry>(_configuration.SchedulerKustoNotifier, _logger, _kustoIngestClient);
_scheduler = new Scheduler(_configuration.Scheduler, _logger, _clock, _schedulerLogWriter);
}
/// <inheritdoc />
public IBigBrother UseKusto(string kustoEngineName, string kustoEngineLocation, string kustoDb, string tenantId)
{
KustoDbName = kustoDb;
var kustoUri = $"https://{kustoEngineName}.{kustoEngineLocation}.kusto.windows.net";
var kustoIngestUri = $"https://ingest-{kustoEngineName}.{kustoEngineLocation}.kusto.windows.net";
var token = new AzureServiceTokenProvider().GetAccessTokenAsync(kustoUri, string.Empty).Result;
KustoAdminClient = KustoClientFactory.CreateCslAdminProvider(
new KustoConnectionStringBuilder(kustoUri)
{
FederatedSecurity = true,
InitialCatalog = KustoDbName,
AuthorityId = tenantId,
ApplicationToken = token
});
KustoIngestClient = KustoIngestFactory.CreateQueuedIngestClient(
new KustoConnectionStringBuilder(kustoIngestUri)
{
FederatedSecurity = true,
InitialCatalog = KustoDbName,
AuthorityId = tenantId,
ApplicationToken = token
});
SetupKustoSubscription();
return(this);
}
/// <summary>
/// After querying, if we need alert, return true, otherwise return false
/// </summary>
/// <param name="rule">The specified MonitorRule instance.</param>
/// <returns>If need alert, return true, otherwise return false.</returns>
public override bool ExecuteRule(MonitorRule rule, out string result)
{
object value = null;
result = string.Empty;
try
{
//Then there is the code to execute the Kusto query based on rule
var client = KustoClientFactory.CreateCslQueryProvider(rule.DataSource);
IDataReader reader = client.ExecuteQuery(rule.AlertQuery);
while (reader.Read())
{
value = reader.GetValue(0);
}
}
catch (Exception ex)
{
Log.WriteErrorLog("{0} throw exception : {1}", rule.RuleUniqueIdentity, ex.ToString());
return(false);
}
result = value.ToString();
Double.TryParse(result, out double returnObj);
return(CommonHelper.CheckResult(rule.Operation, returnObj, rule.Threshold, rule.RuleUniqueIdentity));
}
static void MainImpl(Arguments arguments)
{
// 1. Create a connection string to a cluster/database with AAD user authentication
var cluster = "https://help.kusto.windows.net/";
var database = "Samples";
var kcsb = new KustoConnectionStringBuilder(cluster, database)
{
FederatedSecurity = true
};
// 2. Connect to the Kusto query endpoint and create a query provider
using (var queryProvider = KustoClientFactory.CreateCslQueryProvider(kcsb))
{
// 3. Send a query using the V2 API
var query = "print Welcome='Hello, World!'; print PI=pi()";
var properties = new ClientRequestProperties()
{
ClientRequestId = "kusto_samples_query_v2;" + Guid.NewGuid().ToString()
};
if (arguments.ProgressiveMode)
{
properties.SetOption(ClientRequestProperties.OptionResultsProgressiveEnabled, true);
}
var queryTask = queryProvider.ExecuteQueryV2Async(database, query, properties);
// 4. Parse and print the results of the query
WriteResultsToConsole(queryTask);
}
}
private static string CreateADXTable(string databaseName, string table,
KustoConnectionStringBuilder kustoConnectionStringBuilder)
{
var command = "";
using (var kustoClient = KustoClientFactory.CreateCslAdminProvider(kustoConnectionStringBuilder))
{
command =
CslCommandGenerator.GenerateTableCreateCommand(
table,
new[]
{
Tuple.Create("id", "System.Int32"),
Tuple.Create("date", "System.DateTime"),
Tuple.Create("time", "System.DateTime"),
Tuple.Create("sym", "System.String"),
Tuple.Create("qty", "System.Double"),
Tuple.Create("px", "System.Double")
});
kustoClient.ExecuteControlCommand(databaseName, command);
//if (!isBatch)
//{
// var tablePolicyAlterCommand =
// CslCommandGenerator.GenerateTableAlterStreamingIngestionPolicyCommand(table, isEnabled: true);
// kustoClient.ExecuteControlCommand(databaseName, tablePolicyAlterCommand);
//}
return(command);
// kustoClient.ExecuteControlCommand(databaseName, ".create table StreamingDataTable (['id']:int)");
}
}
public static void CreateTableMappingFromDefinition(string databaseName, string table,
string tableMappingName, KustoConnectionStringBuilder kustoConnectionStringBuilder,
IDictionary <string, string> tableDefinition)
{
using (var kustoClient = KustoClientFactory.CreateCslAdminProvider(kustoConnectionStringBuilder))
{
var columnMappings = new List <ColumnMapping>();
int cnt = 0;
foreach (var keyvaluepair in tableDefinition)
{
columnMappings.Add(new ColumnMapping()
{
ColumnName = keyvaluepair.Key,
Properties = new Dictionary <string, string>()
{
{ MappingConsts.Ordinal, cnt.ToString() }
}
});
cnt++;
}
var command =
CslCommandGenerator.GenerateTableMappingCreateCommand(
Kusto.Data.Ingestion.IngestionMappingKind.Csv,
table,
tableMappingName, columnMappings);
kustoClient.ExecuteControlCommand(databaseName, command);
}
}
static void Main(string[] args)
{
var builder = new KustoConnectionStringBuilder("https://masvaas.kusto.windows.net/")
.WithAadUserPromptAuthentication();
using (var adminProvider = KustoClientFactory.CreateCslAdminProvider(builder))
{
var command = CslCommandGenerator.GenerateDatabasesShowCommand();
var objectReader = new ObjectReader <DatabasesShowCommandResult>(adminProvider.ExecuteControlCommand(command));
foreach (var temp in objectReader)
{
var db = temp.DatabaseName;
var databaseJournalCommand = CslCommandGenerator.GenerateDatabaseJournalShowCommand(db);
databaseJournalCommand += " | where Event == 'ADD-DATABASE' | project EventTimestamp";
using (var journalCmdResult = adminProvider.ExecuteControlCommand(db, databaseJournalCommand))
{
if (journalCmdResult.Read() && DateTime.TryParse(journalCmdResult["EventTimestamp"].ToString(), out var createdTime))
{
//ValhallaLogger.LogInformationalMessage(operationId, nameof(DatabaseInfoProvider), $"Database {database} Created time {createdTime.ToUniversalTime():o}");
Console.WriteLine($"Database: {db}, CreationTime: {createdTime}");
}
}
}
}
}
/// <summary>
/// Constructor which gets ready to make queries to Kusto
/// </summary>
/// <param name="kustoConnectionStringBuilder">The connection string builder to connect to Kusto</param>
public QueryEngine(KustoConnectionStringBuilder kustoConnectionStringBuilder)
{
_adminClient = KustoClientFactory.CreateCslAdminProvider(kustoConnectionStringBuilder);
_queryClient = KustoClientFactory.CreateCslQueryProvider(kustoConnectionStringBuilder);
_databaseName = kustoConnectionStringBuilder.InitialCatalog;
_cluster = kustoConnectionStringBuilder.DataSource;
}
/// <summary>
/// Get kusto query record into Datatable.
/// </summary>
/// <returns>Datatable</returns>
public DataTable ExecuteQueryOnKusto(KustoConnectionStringBuilder kustoConnectionStringBuilder, string query)
{
Console.WriteLine("\n[System]: In Kusto Execute Query to get DT");
DataTable dt = new DataTable();
if (kustoConnectionStringBuilder != null && !string.IsNullOrWhiteSpace(query))
{
try
{
var clients = KustoClientFactory.CreateCslQueryProvider(kustoConnectionStringBuilder);
using (var record = clients.ExecuteQuery(query))
{
if (record != null)
{
dt.Load(record);
}
}
}
catch (Exception ex)
{
Console.WriteLine("\n[System]: Exception :" + ex.Message);
}
}
return(dt);
}
public IKustoManagementGateway CreateGateway(Uri clusterUri, string database)
{
ICslAdminProvider?commandProvider;
// Make the command provider singleton as they hold HTTP connections
if (!_providerCache.TryGetValue(clusterUri, out commandProvider))
{
lock (_providerCache)
{ // Double-check within lock
if (!_providerCache.TryGetValue(clusterUri, out commandProvider))
{
var kustoConnectionStringBuilder = CreateKustoConnectionStringBuilder(
clusterUri,
_tokenProvider);
commandProvider =
KustoClientFactory.CreateCslCmAdminProvider(kustoConnectionStringBuilder);
_providerCache.Add(clusterUri, commandProvider);
}
}
}
return(new KustoManagementGateway(
clusterUri,
database,
commandProvider,
_tracer));
}
public Monitor(Configuration configuration)
{
Contract.RequiresNotNull(configuration);
_configuration = configuration;
_logger = CreateLogger();
// TODO(jubayard): use streaming ingestion instead of direct ingestion. There seems to be some assembly
// issues when attempting to do that
Contract.RequiresNotNullOrEmpty(_configuration.KustoIngestionClusterUrl);
Contract.RequiresNotNullOrEmpty(_configuration.ApplicationClientId);
Contract.RequiresNotNullOrEmpty(_configuration.ApplicationKey);
Contract.RequiresNotNullOrEmpty(_configuration.Authority);
var kustoIngestConnectionString = new KustoConnectionStringBuilder(_configuration.KustoIngestionClusterUrl)
.WithAadApplicationKeyAuthentication(_configuration.ApplicationClientId, _configuration.ApplicationKey, _configuration.Authority);
_kustoIngestClient = KustoIngestFactory.CreateDirectIngestClient(kustoIngestConnectionString);
var kustoConnectionString = new KustoConnectionStringBuilder(_configuration.KustoClusterUrl)
.WithAadApplicationKeyAuthentication(_configuration.ApplicationClientId, _configuration.ApplicationKey, _configuration.Authority);
_cslQueryProvider = KustoClientFactory.CreateCslQueryProvider(kustoConnectionString);
Contract.RequiresNotNull(_configuration.KustoNotifier);
_alertNotifier = new KustoWriter <Notification>(_configuration.KustoNotifier, _logger, _kustoIngestClient);
Contract.RequiresNotNull(_configuration.SchedulerKustoNotifier);
_schedulerLogWriter = new KustoWriter <Scheduler.LogEntry>(_configuration.SchedulerKustoNotifier, _logger, _kustoIngestClient);
_scheduler = new Scheduler(_configuration.Scheduler, _logger, _clock, _schedulerLogWriter);
}
private static ICslQueryProvider CreateClient(AppConfigHelper appConfigHelper)
{
string adxClustorName = appConfigHelper.GetValue(ADXClusterNameKey);
string location = appConfigHelper.GetValue(LocationKey);
string applicationId = appConfigHelper.GetValue(ApplicationIdKey);
string applicationSecret = appConfigHelper.GetValue(ApplicationSecretKey);
string appTenant = appConfigHelper.GetValue(AppTenantKey);
try
{
var kustoUri = $"https://{adxClustorName}.{location}.kusto.windows.net/";
var kustoConnectionStringBuilder = new KustoConnectionStringBuilder(kustoUri)
.WithAadApplicationKeyAuthentication(
applicationClientId: applicationId,
applicationKey: applicationSecret,
authority: appTenant);
return KustoClientFactory.CreateCslQueryProvider(kustoConnectionStringBuilder);
}
catch (Exception ex)
{
throw new ApplicationException("DocumentClient creation failed in the helper class", ex);
}
}
private ICslAdminProvider GetKustoAdminClient()
{
if (this.client == null)
{
try
{
var kustoUri = $"https://{this.config.Global.DataExplorer.Name}.{this.config.Global.Location}.kusto.windows.net/";
var kustoConnectionStringBuilder = new KustoConnectionStringBuilder(kustoUri)
.WithAadApplicationKeyAuthentication(
applicationClientId: this.config.Global.AzureActiveDirectory.AppId,
applicationKey: this.config.Global.AzureActiveDirectory.AppSecret,
authority: this.config.Global.AzureActiveDirectory.TenantId);
this.client = KustoClientFactory.CreateCslAdminProvider(kustoConnectionStringBuilder);
}
catch (Exception e)
{
var msg = "Unable to retrieve kusto with Active Directory properties";
throw new InvalidConfigurationException(msg, e);
}
if (this.client == null)
{
// this.logger.LogError(new Exception(errorMessage), errorMessage);
throw new InvalidConfigurationException("Could not connect to kusto client");
}
}
return(this.client);
}
private static string CreateADXTableFromDefinition(string databaseName, string table,
KustoConnectionStringBuilder kustoConnectionStringBuilder, IDictionary <string, string> tableDefinition)
{
var command = "";
var tuple = new Tuple <string, string> [tableDefinition.Count()];
int cnt = 0;
foreach (var keyvaluepair in tableDefinition)
{
tuple[cnt] = new Tuple <string, string>(keyvaluepair.Key, keyvaluepair.Value);
cnt++;
}
using (var kustoClient = KustoClientFactory.CreateCslAdminProvider(kustoConnectionStringBuilder))
{
command =
CslCommandGenerator.GenerateTableCreateCommand(
table, tuple);
var tablePolicyAlterCommand = CslCommandGenerator.GenerateTableAlterStreamingIngestionPolicyCommand(table, isEnabled: true);
kustoClient.ExecuteControlCommand(databaseName, command);
kustoClient.ExecuteControlCommand(databaseName, tablePolicyAlterCommand);
}
return(command);
}
public List <string> Query(string query)
{
Log.Info($"query:{query}", ConsoleColor.Blue);
if (_kustoQueryClient == null)
{
_kustoQueryClient = KustoClientFactory.CreateCslQueryProvider(ManagementConnection);
queryTimer = new Timer(DisposeQueryClient, null, maxKustoClientTimeMs, maxKustoClientTimeMs);
}
try
{
queryTimer.Change(maxKustoClientTimeMs, maxKustoClientTimeMs);
// unable to parse multiple tables v1 or v2 using kusto so using httpclient and rest
string requestBody = "{ \"db\": \"" + DatabaseName + "\", \"csl\": \"" + query + "\" }";
string requestId = new Guid().ToString();
Dictionary <string, string> headers = new Dictionary <string, string>();
headers.Add("accept", "application/json");
headers.Add("host", HostName);
headers.Add("x-ms-client-request-id", requestId);
Log.Info($"query:", requestBody);
_httpClient.DisplayResponse = Config.LogDebug >= LoggingLevel.Verbose;
_httpClient.SendRequest(uri: RestQueryUri, authToken: _arm.BearerToken, jsonBody: requestBody, httpMethod: HttpMethod.Post, headers: headers);
ResponseDataSet = JsonConvert.DeserializeObject <KustoRestResponseV1>(_httpClient.ResponseStreamString);
if (!ResponseDataSet.HasData())
{
Log.Info($"no tables:", ResponseDataSet);
return(new List <string>());
}
KustoRestTableOfContentsV1 toc = SetTableOfContents(ResponseDataSet);
if (toc.HasData)
{
SetExtendedProperties();
long index = toc.Rows.FirstOrDefault(x => x.Kind.Equals("QueryResult")).Ordinal;
PrimaryResultTable = new KustoRestTable(ResponseDataSet.Tables[index]);
return(PrimaryResultTable.RecordsCsv());
}
else
{
TableOfContents = new KustoRestTableOfContentsV1();
Cursor = "''";
ExtendedPropertiesTable = new KustoRestTable();
PrimaryResultTable = new KustoRestTable(ResponseDataSet.Tables[0]);
return(PrimaryResultTable.RecordsCsv());
}
}
catch (Exception e)
{
Log.Exception($"exception executing query: {query}\r\n{e}");
return(new List <string>());
}
}
public static void CreateTableMapping(string databaseName, string table, string tableMappingName,
KustoConnectionStringBuilder kustoConnectionStringBuilder)
{
using (var kustoClient = KustoClientFactory.CreateCslAdminProvider(kustoConnectionStringBuilder))
{
var command =
CslCommandGenerator.GenerateTableMappingCreateCommand(
Kusto.Data.Ingestion.IngestionMappingKind.Csv,
table,
tableMappingName,
new[] {
new ColumnMapping()
{
ColumnName = "id", Properties = new Dictionary <string, string>()
{
{ MappingConsts.Ordinal, "0" }
}
},
new ColumnMapping()
{
ColumnName = "date", Properties = new Dictionary <string, string>()
{
{ MappingConsts.Ordinal, "1" }
}
},
new ColumnMapping()
{
ColumnName = "time", Properties = new Dictionary <string, string>()
{
{ MappingConsts.Ordinal, "2" }
}
},
new ColumnMapping()
{
ColumnName = "sym", Properties = new Dictionary <string, string>()
{
{ MappingConsts.Ordinal, "3" }
}
},
new ColumnMapping()
{
ColumnName = "qty", Properties = new Dictionary <string, string>()
{
{ MappingConsts.Ordinal, "4" }
}
},
new ColumnMapping()
{
ColumnName = "px", Properties = new Dictionary <string, string>()
{
{ MappingConsts.Ordinal, "5" }
}
}
});
kustoClient.ExecuteControlCommand(databaseName, command);
}
}
private void CreateOrResetTable(IDictionary <string, object> value)
{
using (var admin = KustoClientFactory.CreateCslAdminProvider(kscbAdmin))
{
string dropTable = CslCommandGenerator.GenerateTableDropCommand(_table, true);
admin.ExecuteControlCommand(dropTable);
CreateMergeKustoTable(admin, value);
}
}
private List <string> Command(string command)
{
Log.Info($"command:{command}", ConsoleColor.Blue);
using (ICslAdminProvider kustoAdminClient = KustoClientFactory.CreateCslAdminProvider(ManagementConnection))
{
return(EnumerateResults(kustoAdminClient.ExecuteControlCommand(command)));
}
}
public static void CreateClient()
{
using (kustoAdminClient = KustoClientFactory.CreateCslAdminProvider(Kusto()))
{
PopulateFunctionsData();
}
functionFullName = $"{KustoDatabase()}:fn_countries_and_airports";
}
public static void CreateClient()
{
using (kustoAdminClient = KustoClientFactory.CreateCslAdminProvider(Kusto()))
{
PopulateTypesIndexData();
}
typesIndexFullName = $"{KustoDatabase()}:{TypesIndex}";
}
public static void IngestCsvFile(string connectionString, string tableName, string path, IEnumerable <string> tags)
{
using (var adminProvider = KustoClientFactory.CreateCslAdminProvider(connectionString))
{
var csvData = File.ReadAllText(path);
var command = CslCommandGenerator.GenerateTableIngestPushCommand(
tableName, /* compressed: */ true, csvData, tags);
adminProvider.ExecuteControlCommand(command);
}
}
private static void ValidateIngestion(string databaseName, string table, KustoConnectionStringBuilder kustoConnectionStringBuilder)
{
using (var cslQueryProvider = KustoClientFactory.CreateCslQueryProvider(kustoConnectionStringBuilder))
{
var query = $"{table} | count";
var results = cslQueryProvider.ExecuteQuery <long>(databaseName, query);
Console.WriteLine(results.Single());
}
}
public AdxClientTool()
{
connection =
new KustoConnectionStringBuilder(Properties.Connection.Default.kustoURL).WithAadApplicationKeyAuthentication(
applicationClientId: Properties.Connection.Default.appClientId,
applicationKey: Properties.Connection.Default.appClientSecret,
authority: Properties.Connection.Default.appAadTenantId);
adx = KustoClientFactory.CreateCslQueryProvider(connection);
}
public List <string> Query(string query)
{
Log.Info($"query:{query}", ConsoleColor.Blue);
List <string> list = new List <string>();
using (ICslQueryProvider kustoQueryClient = KustoClientFactory.CreateCslQueryProvider(ManagementConnection))
{
return(EnumerateResults(kustoQueryClient.ExecuteQuery(query)));
}
}
/// <summary>
/// Populate the Kusto backend with test data.
/// </summary>
/// <param name="kusto"></param>
/// <param name="db"></param>
/// <param name="table"></param>
/// <param name="mapping"></param>
/// <param name="structure">JSON file containing the Elasticsearch index structure. Elasticsearch types will be converted to Kusto types. Note that the method only supported a small set of Elasticsearch types.</param>
/// <param name="dataFile">Gzipped JSON file containing the data to be loaded.</param>
/// <returns>Bulk Insert operation result.</returns>
public static async Task <IKustoIngestionResult> Populate(KustoConnectionStringBuilder kusto, string db, string table, string mapping, string structure, string dataFile)
{
var struc = JObject.Parse(structure);
var properties = struc["mappings"]["_doc"]["properties"] as JObject;
// Build list of columns and mappings to provision Kusto
var kustoColumns = new List <string>();
var columnMappings = new List <ColumnMapping>();
foreach (var prop in properties)
{
string name = prop.Key;
JObject value = prop.Value as JObject;
string type = (string)value["type"];
if (ES2KUSTOTYPE.ContainsKey(type))
{
type = ES2KUSTOTYPE[type];
}
kustoColumns.Add($"{name}:{type}");
columnMappings.Add(new ColumnMapping()
{
ColumnName = name,
Properties = new Dictionary <string, string>
{
["Path"] = $"$.{name}",
},
});
}
using (var kustoAdminClient = KustoClientFactory.CreateCslAdminProvider(kusto))
{
// Send drop table ifexists command to Kusto
var command = CslCommandGenerator.GenerateTableDropCommand(table, true);
kustoAdminClient.ExecuteControlCommand(command);
// Send create table command to Kusto
command = $".create table {table} ({string.Join(", ", kustoColumns)})";
Console.WriteLine(command);
kustoAdminClient.ExecuteControlCommand(command);
// Send create table mapping command to Kusto
command = CslCommandGenerator.GenerateTableMappingCreateCommand(IngestionMappingKind.Json, table, mapping, columnMappings);
kustoAdminClient.ExecuteControlCommand(command);
}
// Log information to console.
// Can't use Console.WriteLine here: https://github.com/nunit/nunit3-vs-adapter/issues/266
TestContext.Progress.WriteLine($"Ingesting {dataFile} as compressed data into Kusto");
// Populate Kusto
using Stream fs = File.OpenRead(dataFile);
return(await KustoIngest(kusto, db, table, mapping, fs));
}
private ICslQueryProvider GetKustoConnection(KustoConnectionConfiguration conf)
{
var kustoConnectionStringBuilder = new KustoConnectionStringBuilder(conf.Cluster, conf.DefaultDB)
{
UserID = conf.User,
Authority = conf.AuthorityId,
FederatedSecurity = true
};
return(KustoClientFactory.CreateCslQueryProvider(kustoConnectionStringBuilder));
}
public static Result <IKustoClient> CreateKustoQueryClient(string kustoClusterUrl, string azureTenantId, string azureAppId, string azureAppKey)
{
Contract.RequiresNotNullOrEmpty(kustoClusterUrl);
Contract.RequiresNotNullOrEmpty(azureAppId);
Contract.RequiresNotNullOrEmpty(azureAppKey);
Contract.RequiresNotNullOrEmpty(azureTenantId);
var kustoConnectionString = new KustoConnectionStringBuilder(kustoClusterUrl)
.WithAadApplicationKeyAuthentication(azureAppId, azureAppKey, azureTenantId);
return(new Result <IKustoClient>(new KustoClient(KustoClientFactory.CreateCslQueryProvider(kustoConnectionString))));
}
