/// <summary>
/// Generates a Kusto table mapping for a specific <see cref="Type"/>, by mapping it's properties to column mappings.
/// </summary>
/// <param name="client">The <see cref="ICslAdminProvider"/> client that we are extending.</param>
/// <param name="type">The <see cref="Type"/> that we are generating the JSON mapping for.</param>
/// <returns>The name of the mapping created.</returns>
public static string GenerateTableJsonMappingFromType(this ICslAdminProvider client, Type type)
{
var tableName = type.Name;
var mappingName = $"{tableName}_mapping";
var command = CslCommandGenerator.GenerateTableJsonMappingShowCommand(tableName, mappingName);
try
{
client.ExecuteControlCommand(command);
return(mappingName);
}
catch (KustoBadRequestException ex) when(ex.ErrorMessage.Contains("'JsonMappingPersistent' was not found"))
{
// soak
}
var mappings = type.GetProperties().Select(property => new JsonColumnMapping {
ColumnName = property.Name, JsonPath = $"$.{property.Name}"
}).ToList();
command = CslCommandGenerator.GenerateTableJsonMappingCreateCommand(tableName, mappingName, mappings);
client.ExecuteControlCommand(command);
return(mappingName);
}
public KustoContext(ICslQueryProvider query, ICslAdminProvider admin, string databaseName, ILogger logger)
{
_query = query;
_admin = admin;
_databaseName = databaseName;
_logger = logger;
}
/// <summary>
/// Constructor which gets ready to make queries to Kusto
/// </summary>
/// <param name="kustoConnectionStringBuilder">The connection string builder to connect to Kusto</param>
public QueryEngine(KustoConnectionStringBuilder kustoConnectionStringBuilder)
{
_adminClient = KustoClientFactory.CreateCslAdminProvider(kustoConnectionStringBuilder);
_queryClient = KustoClientFactory.CreateCslQueryProvider(kustoConnectionStringBuilder);
_databaseName = kustoConnectionStringBuilder.InitialCatalog;
_cluster = kustoConnectionStringBuilder.DataSource;
}
/// <summary>
/// Generates a Kusto table mapping for a specific <see cref="Type"/>, by mapping it's properties to column mappings.
/// </summary>
/// <param name="client">The <see cref="ICslAdminProvider"/> client that we are extending.</param>
/// <param name="type">The <see cref="Type"/> that we are generating the JSON mapping for.</param>
/// <returns>The name of the mapping created.</returns>
public static async Task <string> GenerateTableJsonMappingFromType(this ICslAdminProvider client, Type type)
{
var tableName = type.Name;
var mappingName = $"{tableName}_mapping";
var tableMappings = new List <string>();
var command = CslCommandGenerator.GenerateTableJsonMappingsShowCommand(tableName);
var reader = await client.ExecuteControlCommandAsync(client.DefaultDatabaseName, command);
while (reader.Read())
{
tableMappings.Add(reader.GetString(0));
}
if (tableMappings.Contains(mappingName))
{
return(mappingName);
}
var mappings = type.GetProperties().Select(property => new JsonColumnMapping {
ColumnName = property.Name, JsonPath = $"$.{property.Name}"
}).ToList();
command = CslCommandGenerator.GenerateTableJsonMappingCreateCommand(tableName, mappingName, mappings);
await client.ExecuteControlCommandAsync(client.DefaultDatabaseName, command);
return(mappingName);
}
private ICslAdminProvider GetKustoAdminClient()
{
if (this.client == null)
{
try
{
var kustoUri = $"https://{this.config.Global.DataExplorer.Name}.{this.config.Global.Location}.kusto.windows.net/";
var kustoConnectionStringBuilder = new KustoConnectionStringBuilder(kustoUri)
.WithAadApplicationKeyAuthentication(
applicationClientId: this.config.Global.AzureActiveDirectory.AppId,
applicationKey: this.config.Global.AzureActiveDirectory.AppSecret,
authority: this.config.Global.AzureActiveDirectory.TenantId);
this.client = KustoClientFactory.CreateCslAdminProvider(kustoConnectionStringBuilder);
}
catch (Exception e)
{
var msg = "Unable to retrieve kusto with Active Directory properties";
throw new InvalidConfigurationException(msg, e);
}
if (this.client == null)
{
// this.logger.LogError(new Exception(errorMessage), errorMessage);
throw new InvalidConfigurationException("Could not connect to kusto client");
}
}
return(this.client);
}
/// <summary>
/// Generates a Kusto table for a specific <see cref="Type"/>, by mapping it's properties to columns.
/// </summary>
/// <param name="client">The <see cref="ICslAdminProvider"/> that we are extending.</param>
/// <param name="type">The <see cref="Type"/> that we are generating a table for.</param>
/// <returns>The name of the table created.</returns>
public static async Task <string> GenerateTableFromType(this ICslAdminProvider client, Type type)
{
var tableName = type.Name;
var tables = new List <string>();
var command = CslCommandGenerator.GenerateTablesShowCommand();
var reader = await client.ExecuteControlCommandAsync(client.DefaultDatabaseName, command);
while (reader.Read())
{
tables.Add(reader.GetString(0));
}
if (tables.Contains(tableName))
{
return(tableName);
}
var columns = type.GetProperties().Select(property => Tuple.Create(property.Name, property.PropertyType.FullName)).ToList();
command = CslCommandGenerator.GenerateTableCreateCommand(tableName, columns);
await client.ExecuteControlCommandAsync(client.DefaultDatabaseName, command);
return(tableName);
}
/// <inheritdoc />
public IBigBrother UseKusto(string kustoEngineName, string kustoEngineLocation, string kustoDb, string tenantId)
{
KustoDbName = kustoDb;
var kustoUri = $"https://{kustoEngineName}.{kustoEngineLocation}.kusto.windows.net";
var kustoIngestUri = $"https://ingest-{kustoEngineName}.{kustoEngineLocation}.kusto.windows.net";
var token = new AzureServiceTokenProvider().GetAccessTokenAsync(kustoUri, string.Empty).Result;
KustoAdminClient = KustoClientFactory.CreateCslAdminProvider(
new KustoConnectionStringBuilder(kustoUri)
{
FederatedSecurity = true,
InitialCatalog = KustoDbName,
AuthorityId = tenantId,
ApplicationToken = token
});
KustoIngestClient = KustoIngestFactory.CreateQueuedIngestClient(
new KustoConnectionStringBuilder(kustoIngestUri)
{
FederatedSecurity = true,
InitialCatalog = KustoDbName,
AuthorityId = tenantId,
ApplicationToken = token
});
SetupKustoSubscription();
return(this);
}
private async Task <IReadOnlyDictionary <string, IngestionMapping[]> > GetIngestionMappingsAsync(
string databaseName,
ICslAdminProvider client)
{
_log.LogDebug($"Getting ingestion mappings for database '{databaseName}'");
var showIngestionMappingsCsl = $".show databases (['{databaseName}']) ingestion mappings with (onlyLatestPerTable=False)";
var ingestionMappings = new List <IngestionMapping>();
using (var reader = await client.ExecuteControlCommandAsync(databaseName, showIngestionMappingsCsl))
{
while (reader.Read())
{
var mapping = new IngestionMapping(
reader.GetString(5),
reader.GetString(0),
reader.GetString(1),
JsonConvert.DeserializeObject <IngestionMappingColumnMapping[]>(reader.GetString(2)));
ingestionMappings.Add(mapping);
}
}
_log.LogDebug($"Successfully retrieved ingestion mappings for database '{databaseName}'");
return(ingestionMappings.GroupBy(m => m.TableName).ToDictionary(grp => grp.Key, grp => grp.ToArray()));
}
private static void DisposeAdminClient(object state)
{
if (_kustoAdminClient != null)
{
Log.Info("Disposing kusto admin client");
_kustoAdminClient.Dispose();
_kustoAdminClient = null;
}
}
private List <string> Command(string command)
{
Log.Info($"command:{command}", ConsoleColor.Blue);
using (ICslAdminProvider kustoAdminClient = KustoClientFactory.CreateCslAdminProvider(ManagementConnection))
{
return(EnumerateResults(kustoAdminClient.ExecuteControlCommand(command)));
}
}
public static void CreateClient()
{
using (kustoAdminClient = KustoClientFactory.CreateCslAdminProvider(Kusto()))
{
PopulateTypesIndexData();
}
typesIndexFullName = $"{KustoDatabase()}:{TypesIndex}";
}
public KustoClient(IConfiguration configuration, ILoggerFactory loggerFactory)
{
_logger = loggerFactory.CreateLogger <KustoClient>();
_kustoSettings = configuration.GetConfiguredSettings <KustoSettings>();
var clientFactory = new ClientFactory(configuration, loggerFactory);
_queryClient = clientFactory.QueryClient;
_adminClient = clientFactory.AdminClient;
}
public static void CreateClient()
{
using (kustoAdminClient = KustoClientFactory.CreateCslAdminProvider(Kusto()))
{
PopulateFunctionsData();
}
functionFullName = $"{KustoDatabase()}:fn_countries_and_airports";
}
/// <summary>
/// Initializes a new instance of the <see cref="KustoQueryExecutor"/> class.
/// </summary>
/// <param name="adminClient">Admin client.</param>
/// <param name="queryClient">Query client.</param>
/// <param name="logger">A logger.</param>
/// <param name="metricsHistograms">The instance of the class to record metrics.</param>
public KustoQueryExecutor(
ICslQueryProvider queryClient,
ICslAdminProvider adminClient,
ILogger <KustoQueryExecutor> logger,
Metrics metricsHistograms)
{
Logger = logger;
this.queryClient = queryClient ?? throw new ArgumentNullException(nameof(queryClient));
this.adminClient = adminClient ?? throw new ArgumentNullException(nameof(adminClient));
this.metricsHistograms = metricsHistograms;
}
public KustoManagementGateway(
Uri clusterUri,
string database,
ICslAdminProvider commandProvider,
ITracer tracer)
{
_clusterUri = clusterUri;
_database = database;
_commandProvider = commandProvider;
_tracer = tracer;
}
private List <string> Command(string command)
{
Log.Info($"command:{command}", ConsoleColor.Blue);
if (_kustoAdminClient == null)
{
_kustoAdminClient = KustoClientFactory.CreateCslAdminProvider(ManagementConnection);
adminTimer = new Timer(DisposeAdminClient, null, maxKustoClientTimeMs, maxKustoClientTimeMs);
}
adminTimer.Change(maxKustoClientTimeMs, maxKustoClientTimeMs);
return(EnumerateResults(_kustoAdminClient.ExecuteControlCommand(command)));
}
public KustoClient(IServiceProvider serviceProvider, ILoggerFactory loggerFactory,
KustoSettings kustoSettings = null)
{
logger = loggerFactory.CreateLogger <KustoClient>();
var configuration = serviceProvider.GetRequiredService <IConfiguration>();
this.kustoSettings = kustoSettings ?? configuration.GetConfiguredSettings <KustoSettings>();
var clientFactory = new KustoClientFactory(serviceProvider, kustoSettings);
queryClient = clientFactory.QueryQueryClient;
adminClient = clientFactory.AdminClient;
ingestClient = clientFactory.IngestClient;
}
private void CreateMergeKustoTable(ICslAdminProvider admin, IDictionary <string, object> value)
{
TableSchema tableSchema = new TableSchema(TableName);
foreach (var pair in value)
{
tableSchema.AddColumnIfMissing(new ColumnSchema(pair.Key, _columnType[pair.Value != null ? pair.Value.GetType() : typeof(string)]));
}
string createTable = CslCommandGenerator.GenerateTableCreateMergeCommand(tableSchema);
admin.ExecuteControlCommand(createTable);
string enableIngestTime = CslCommandGenerator.GenerateIngestionTimePolicyAlterCommand(TableName, true);
admin.ExecuteControlCommand(enableIngestTime);
}
/// <summary>
/// Constructor which creates a connection to the workspace cluster and uses the workspace database to temporarily load the schema
/// </summary>
public QueryEngine()
{
if (string.IsNullOrEmpty(SettingsWrapper.KustoClusterForTempDatabases))
{
throw new ArgumentNullException(nameof(SettingsWrapper.KustoClusterForTempDatabases));
}
_databaseName = SettingsWrapper.TemporaryKustoDatabase;
var connString = new KustoConnectionStringBuilder(SettingsWrapper.KustoClusterForTempDatabases)
{
FederatedSecurity = true,
InitialCatalog = _databaseName,
Authority = SettingsWrapper.AADAuthority
};
_adminClient = KustoClientFactory.CreateCslAdminProvider(connString);
_queryClient = KustoClientFactory.CreateCslQueryProvider(connString);
_tempDatabaseUsed = true;
_cluster = connString.DataSource;
CleanDatabase();
}
/// <summary>
/// Generates a Kusto table for a specific <see cref="Type"/>, by mapping it's properties to columns.
/// </summary>
/// <param name="client">The <see cref="ICslAdminProvider"/> that we are extending.</param>
/// <param name="type">The <see cref="Type"/> that we are generating a table for.</param>
/// <returns>The name of the table created.</returns>
public static string GenerateTableFromType(this ICslAdminProvider client, Type type)
{
var tableName = type.Name;
var command = CslCommandGenerator.GenerateTableShowCommand(tableName);
try
{
client.ExecuteControlCommand(command);
return(tableName);
}
catch (KustoBadRequestException ex) when(ex.ErrorMessage.Contains("'Table' was not found"))
{
// soak
}
var columns = type.GetProperties().Select(property => new Tuple <string, string>(property.Name, property.PropertyType.FullName)).ToList();
command = CslCommandGenerator.GenerateTableCreateCommand(tableName, columns);
client.ExecuteControlCommand(command);
return(tableName);
}
public ClientFactory(IConfiguration configuration, ILoggerFactory loggerFactory)
{
_logger = loggerFactory.CreateLogger <ClientFactory>();
var aadSettings = configuration.GetConfiguredSettings <AadSettings>();
var kustoSettings = configuration.GetConfiguredSettings <KustoSettings>();
var authBuilder = new AadAuthBuilder(aadSettings);
var clientSecretCert = authBuilder.GetClientSecretOrCert();
KustoConnectionStringBuilder kcsb;
if (kustoSettings.AuthMode == AuthMode.User)
{
kcsb = new KustoConnectionStringBuilder(kustoSettings.ClusterUrl, kustoSettings.DbName)
{
FederatedSecurity = true,
Authority = aadSettings.Authority
}.WithAadUserPromptAuthentication();
}
else if (clientSecretCert.secret != null)
{
kcsb = new KustoConnectionStringBuilder($"{kustoSettings.ClusterUrl}")
.WithAadApplicationKeyAuthentication(
aadSettings.ClientId,
clientSecretCert.secret,
aadSettings.Authority);
}
else
{
kcsb = new KustoConnectionStringBuilder($"{kustoSettings.ClusterUrl}")
.WithAadApplicationCertificateAuthentication(
aadSettings.ClientId,
clientSecretCert.cert,
aadSettings.Authority);
}
_client = KustoClientFactory.CreateCslQueryProvider(kcsb);
_adminClient = KustoClientFactory.CreateCslAdminProvider(kcsb);
}
private async Task <DatabaseSchema> GetDatabaseSchemaAsync(string databaseName, ICslAdminProvider client)
{
_log.LogDebug($"Getting schema for database '{databaseName}'");
var showDatabaseSchemaCsl = $".show database ['{databaseName}'] schema as json";
ClusterSchema clusterSchema;
using (var reader = await client.ExecuteControlCommandAsync(databaseName, showDatabaseSchemaCsl))
{
reader.Read();
clusterSchema = JsonConvert.DeserializeObject <ClusterSchema>(reader[0].ToString());
}
_log.LogDebug($"Successfully retrieved schema for database '{databaseName}'");
return(clusterSchema.Databases[databaseName]);
}
public KustoTableManagementClient(AppConfig config, ILogger <KustoTableManagementClient> logger)
{
this.config = config;
this.logger = logger;
this.client = this.GetKustoAdminClient();
}