public KustoContext(ICslQueryProvider query, ICslAdminProvider admin, string databaseName, ILogger logger)
{
_query = query;
_admin = admin;
_databaseName = databaseName;
_logger = logger;
}
private ICslQueryProvider GetKustoQueryClient()
{
if (this.client == null)
{
try
{
var kustoUri = $"https://{this.config.Global.DataExplorer.Name}.{this.config.Global.Location}.kusto.windows.net/";
var kustoConnectionStringBuilder = new KustoConnectionStringBuilder(kustoUri)
.WithAadApplicationKeyAuthentication(
applicationClientId: this.applicationId,
applicationKey: this.applicationSecret,
authority: this.tenant);
this.client = KustoClientFactory.CreateCslQueryProvider(kustoConnectionStringBuilder);
}
catch (Exception e)
{
var msg = "Unable to retrieve kusto with Active Directory properties" +
$"'{this.applicationId}'," +
$"'{this.applicationSecret}' and" +
$"'{this.tenant}'.";
throw new InvalidConfigurationException(msg, e);
}
if (this.client == null)
{
// this.logger.LogError(new Exception(errorMessage), errorMessage);
throw new InvalidConfigurationException("Could not connect to kusto client");
}
}
return(this.client);
}
/// <inheritdoc />
protected override void Setup(DownloadMonthlyQueueDataInput input)
{
// prepare query, calculate last day of requested month
var lastDayOfMonth = DateTime.DaysInMonth(input.Year, input.Month);
// try to connect..
m_queryProvider = GetKustoConnection(m_configuration.KustoConfig);
// and also parse the query
m_query = File.ReadAllText(Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), constants.GetMonthlyQueueDataQuery))
.Replace("{0}", Convert.ToString(input.Year))
.Replace("{1}", Convert.ToString(input.Month))
.Replace("{2}", Convert.ToString(lastDayOfMonth))
.Replace("{3}", m_configuration.UseCBTest ? constants.CBTestDatabaseName : constants.ProdDatabaseName);
s_logger.Debug($"Target Query: {m_query}");
// and prepare the directory with the output
m_outputDir = Path.Combine(input.OutputDirectory, constants.ResultDirectoryName);
m_distanceMaps = Path.Combine(m_outputDir, constants.QueueMapDirectoryName);
// create these dirs
Directory.CreateDirectory(m_outputDir);
Directory.CreateDirectory(m_distanceMaps);
m_outputFile = Path.Combine(m_outputDir, $"queues-{input.Year}-{input.Month}.csv");
m_writer = new StreamWriter(m_outputFile);
Contract.Requires(File.Exists(m_outputFile), $"Could not create output file [{m_outputFile}]");
Contract.Requires(Directory.Exists(m_distanceMaps), $"Could not create output directory [{m_distanceMaps}]");
}
public Monitor(Configuration configuration)
{
Contract.RequiresNotNull(configuration);
_configuration = configuration;
_logger = CreateLogger();
Contract.RequiresNotNullOrEmpty(_configuration.KustoIngestionClusterUrl);
Contract.RequiresNotNullOrEmpty(_configuration.ApplicationClientId);
Contract.RequiresNotNullOrEmpty(_configuration.ApplicationKey);
Contract.RequiresNotNullOrEmpty(_configuration.Authority);
var kustoIngestConnectionString = new KustoConnectionStringBuilder(_configuration.KustoIngestionClusterUrl)
.WithAadApplicationKeyAuthentication(_configuration.ApplicationClientId, _configuration.ApplicationKey, _configuration.Authority);
_kustoIngestClient = KustoIngestFactory.CreateDirectIngestClient(kustoIngestConnectionString);
var kustoConnectionString = new KustoConnectionStringBuilder(_configuration.KustoClusterUrl)
.WithAadApplicationKeyAuthentication(_configuration.ApplicationClientId, _configuration.ApplicationKey, _configuration.Authority);
_cslQueryProvider = KustoClientFactory.CreateCslQueryProvider(kustoConnectionString);
Contract.RequiresNotNull(_configuration.KustoNotifier);
_alertNotifier = new KustoWriter <Notification>(_configuration.KustoNotifier, _logger, _kustoIngestClient);
Contract.RequiresNotNull(_configuration.SchedulerKustoNotifier);
_schedulerLogWriter = new KustoWriter <Scheduler.LogEntry>(_configuration.SchedulerKustoNotifier, _logger, _kustoIngestClient);
_scheduler = new Scheduler(_configuration.Scheduler, _logger, _clock, _schedulerLogWriter);
}
/// <summary>
/// Constructor which gets ready to make queries to Kusto
/// </summary>
/// <param name="kustoConnectionStringBuilder">The connection string builder to connect to Kusto</param>
public QueryEngine(KustoConnectionStringBuilder kustoConnectionStringBuilder)
{
_adminClient = KustoClientFactory.CreateCslAdminProvider(kustoConnectionStringBuilder);
_queryClient = KustoClientFactory.CreateCslQueryProvider(kustoConnectionStringBuilder);
_databaseName = kustoConnectionStringBuilder.InitialCatalog;
_cluster = kustoConnectionStringBuilder.DataSource;
}
public static async Task <Watchlist> CreateAsync(ILogger logger, ICslQueryProvider cslQueryProvider, IReadOnlyDictionary <CloudBuildEnvironment, EnvironmentConfiguration> environments)
{
var watchlist = new Watchlist(logger, cslQueryProvider, environments);
await watchlist.RefreshAsync();
return(watchlist);
}
} // - function Read
private static void InitializeKustoClient(ILogger log)
{
lock (Lock)
{
if (!_isInitialized)
{
log.LogInformation("[PrometheusRead] [InitializeKustoClient] Initialize");
var connection =
new KustoConnectionStringBuilder(
Environment.GetEnvironmentVariable("kustoUrl", EnvironmentVariableTarget.Process))
.WithAadApplicationKeyAuthentication(
applicationClientId: Environment.GetEnvironmentVariable("appClientId",
EnvironmentVariableTarget.Process),
applicationKey: Environment.GetEnvironmentVariable("appClientSecret",
EnvironmentVariableTarget.Process),
authority: Environment.GetEnvironmentVariable("tenantId",
EnvironmentVariableTarget.Process));
_adx = KustoClientFactory.CreateCslQueryProvider(connection);
_isInitialized = true;
}
log.LogInformation("[PrometheusRead] [InitializeKustoClient] Ok");
}
} // - InitializeKustoClient
private void ClearProviderCache(KustoClientProviderOptions arg1, string arg2)
{
lock (_updateLock)
{
_kustoQueryProvider = null;
}
}
/// <summary>
/// Execute monitored ADX query.
/// </summary>
/// <param name="client">Query provider.</param>
/// <param name="query">ADX query string.</param>
/// <param name="clientRequestProperties">An object that represents properties that will be sent to Kusto.</param>
/// <param name="metrics">Prometheus query duration metric.</param>
/// <returns>Tuple of timeTaken and the reader result.</returns>
public static async Task <(TimeSpan timeTaken, IDataReader reader)> ExecuteMonitoredQueryAsync(
this ICslQueryProvider client,
string query,
ClientRequestProperties clientRequestProperties,
Metrics metrics)
{
Ensure.IsNotNull(client, nameof(client));
Ensure.IsNotNull(metrics, nameof(metrics));
Ensure.IsNotNullOrEmpty(query, nameof(query));
// Timer to be used to report the duration of a query to.
var stopwatch = new Stopwatch();
stopwatch.Start();
var reader = await client?.ExecuteQueryAsync(string.Empty, query, clientRequestProperties);
stopwatch.Stop();
var duration = stopwatch.Elapsed;
if (metrics != null)
{
metrics.AdxQueryDurationMetric.Observe(duration.TotalSeconds);
}
return(duration, reader);
}
public static async Task <Watchlist> CreateAsync(ILogger logger, ICslQueryProvider cslQueryProvider)
{
var watchlist = new Watchlist(logger, cslQueryProvider);
await watchlist.RefreshAsync();
return(watchlist);
}
public Monitor(Configuration configuration)
{
Contract.RequiresNotNull(configuration);
_configuration = configuration;
_logger = CreateLogger();
// TODO(jubayard): use streaming ingestion instead of direct ingestion. There seems to be some assembly
// issues when attempting to do that
Contract.RequiresNotNullOrEmpty(_configuration.KustoIngestionClusterUrl);
Contract.RequiresNotNullOrEmpty(_configuration.ApplicationClientId);
Contract.RequiresNotNullOrEmpty(_configuration.ApplicationKey);
Contract.RequiresNotNullOrEmpty(_configuration.Authority);
var kustoIngestConnectionString = new KustoConnectionStringBuilder(_configuration.KustoIngestionClusterUrl)
.WithAadApplicationKeyAuthentication(_configuration.ApplicationClientId, _configuration.ApplicationKey, _configuration.Authority);
_kustoIngestClient = KustoIngestFactory.CreateDirectIngestClient(kustoIngestConnectionString);
var kustoConnectionString = new KustoConnectionStringBuilder(_configuration.KustoClusterUrl)
.WithAadApplicationKeyAuthentication(_configuration.ApplicationClientId, _configuration.ApplicationKey, _configuration.Authority);
_cslQueryProvider = KustoClientFactory.CreateCslQueryProvider(kustoConnectionString);
Contract.RequiresNotNull(_configuration.KustoNotifier);
_alertNotifier = new KustoWriter <Notification>(_configuration.KustoNotifier, _logger, _kustoIngestClient);
Contract.RequiresNotNull(_configuration.SchedulerKustoNotifier);
_schedulerLogWriter = new KustoWriter <Scheduler.LogEntry>(_configuration.SchedulerKustoNotifier, _logger, _kustoIngestClient);
_scheduler = new Scheduler(_configuration.Scheduler, _logger, _clock, _schedulerLogWriter);
}
public async Task <string> FetchZoneForTraceId(string uxoTraceId, ICslQueryProvider queryProvider)
{
var query = $"UxoTrace | where TraceId == \"{uxoTraceId}\" | project Zone | take 1";
var queryResult = await DatabaseQueryRunner.RunQuery(DatabaseConstants.Uxo_Connection, DatabaseConstants.Uxo_Database, query, queryProvider);
object[] result = (object[])queryResult.Results[0];
return(result[0].ToString());
}
public List <string> Query(string query)
{
Log.Info($"query:{query}", ConsoleColor.Blue);
if (_kustoQueryClient == null)
{
_kustoQueryClient = KustoClientFactory.CreateCslQueryProvider(ManagementConnection);
queryTimer = new Timer(DisposeQueryClient, null, maxKustoClientTimeMs, maxKustoClientTimeMs);
}
try
{
queryTimer.Change(maxKustoClientTimeMs, maxKustoClientTimeMs);
// unable to parse multiple tables v1 or v2 using kusto so using httpclient and rest
string requestBody = "{ \"db\": \"" + DatabaseName + "\", \"csl\": \"" + query + "\" }";
string requestId = new Guid().ToString();
Dictionary <string, string> headers = new Dictionary <string, string>();
headers.Add("accept", "application/json");
headers.Add("host", HostName);
headers.Add("x-ms-client-request-id", requestId);
Log.Info($"query:", requestBody);
_httpClient.DisplayResponse = Config.LogDebug >= LoggingLevel.Verbose;
_httpClient.SendRequest(uri: RestQueryUri, authToken: _arm.BearerToken, jsonBody: requestBody, httpMethod: HttpMethod.Post, headers: headers);
ResponseDataSet = JsonConvert.DeserializeObject <KustoRestResponseV1>(_httpClient.ResponseStreamString);
if (!ResponseDataSet.HasData())
{
Log.Info($"no tables:", ResponseDataSet);
return(new List <string>());
}
KustoRestTableOfContentsV1 toc = SetTableOfContents(ResponseDataSet);
if (toc.HasData)
{
SetExtendedProperties();
long index = toc.Rows.FirstOrDefault(x => x.Kind.Equals("QueryResult")).Ordinal;
PrimaryResultTable = new KustoRestTable(ResponseDataSet.Tables[index]);
return(PrimaryResultTable.RecordsCsv());
}
else
{
TableOfContents = new KustoRestTableOfContentsV1();
Cursor = "''";
ExtendedPropertiesTable = new KustoRestTable();
PrimaryResultTable = new KustoRestTable(ResponseDataSet.Tables[0]);
return(PrimaryResultTable.RecordsCsv());
}
}
catch (Exception e)
{
Log.Exception($"exception executing query: {query}\r\n{e}");
return(new List <string>());
}
}
public KustoClient(IConfiguration configuration, ILoggerFactory loggerFactory)
{
_logger = loggerFactory.CreateLogger <KustoClient>();
_kustoSettings = configuration.GetConfiguredSettings <KustoSettings>();
var clientFactory = new ClientFactory(configuration, loggerFactory);
_queryClient = clientFactory.QueryClient;
_adminClient = clientFactory.AdminClient;
}
private static void DisposeQueryClient(object state)
{
if (_kustoQueryClient != null)
{
Log.Info("Disposing kusto query client");
_kustoQueryClient.Dispose();
_kustoQueryClient = null;
}
}
private static KustoOperations CreateInstance()
{
string connectionString = Environment.GetEnvironmentVariable("AppConfigurationConnectionString", EnvironmentVariableTarget.Process);
AppConfigHelper appConfigHelper = new AppConfigHelper(connectionString);
ICslQueryProvider docClient = CreateClient(appConfigHelper);
return new KustoOperations(docClient);
}
public KustoRuleConfiguration(IClock clock, ILogger logger, INotifier <Notification> notifier, ICslQueryProvider cslQueryProvider, StampId stampId, string kustoDatabaseName, string cacheTableName)
{
Clock = clock;
Logger = logger;
Notifier = notifier;
CslQueryProvider = cslQueryProvider;
StampId = stampId;
KustoDatabaseName = kustoDatabaseName;
CacheTableName = cacheTableName;
}
public AdxClientTool()
{
connection =
new KustoConnectionStringBuilder(Properties.Connection.Default.kustoURL).WithAadApplicationKeyAuthentication(
applicationClientId: Properties.Connection.Default.appClientId,
applicationKey: Properties.Connection.Default.appClientSecret,
authority: Properties.Connection.Default.appAadTenantId);
adx = KustoClientFactory.CreateCslQueryProvider(connection);
}
public List <string> Query(string query)
{
Log.Info($"query:{query}", ConsoleColor.Blue);
List <string> list = new List <string>();
using (ICslQueryProvider kustoQueryClient = KustoClientFactory.CreateCslQueryProvider(ManagementConnection))
{
return(EnumerateResults(kustoQueryClient.ExecuteQuery(query)));
}
}
/// <summary>
/// Initializes a new instance of the <see cref="KustoQueryExecutor"/> class.
/// </summary>
/// <param name="adminClient">Admin client.</param>
/// <param name="queryClient">Query client.</param>
/// <param name="logger">A logger.</param>
/// <param name="metricsHistograms">The instance of the class to record metrics.</param>
public KustoQueryExecutor(
ICslQueryProvider queryClient,
ICslAdminProvider adminClient,
ILogger <KustoQueryExecutor> logger,
Metrics metricsHistograms)
{
Logger = logger;
this.queryClient = queryClient ?? throw new ArgumentNullException(nameof(queryClient));
this.adminClient = adminClient ?? throw new ArgumentNullException(nameof(adminClient));
this.metricsHistograms = metricsHistograms;
}
public static async Task <ObjectReader <T> > QuerySingleResultSetAsync <T>(this ICslQueryProvider cslQueryProvider, string query, string database, ClientRequestProperties?requestProperties = null)
{
Contract.RequiresNotNullOrEmpty(query);
Contract.RequiresNotNullOrEmpty(database);
requestProperties ??= new ClientRequestProperties();
var dataReader = await cslQueryProvider.ExecuteQueryAsync(database, query, requestProperties);
return(new ObjectReader <T>(dataReader, disposeReader: true, nameBasedColumnMapping: true));
}
public KustoQueryClient(
AppConfig config,
ILogger <KustoQueryClient> logger)
{
this.logger = logger;
this.config = config;
this.applicationId = config.Global.AzureActiveDirectory.AppId;
this.applicationSecret = config.Global.AzureActiveDirectory.AppSecret;
this.tenant = config.Global.AzureActiveDirectory.TenantId;
this.client = this.GetKustoQueryClient();
}
public KustoQueryClient(
AppConfig config,
ILogger <KustoQueryClient> logger,
ICslQueryProvider kustQueryClient)
{
this.logger = logger;
this.config = config;
this.applicationId = config.Global.AzureActiveDirectory.AppId;
this.applicationSecret = config.Global.AzureActiveDirectory.AppSecret;
this.tenant = config.Global.AzureActiveDirectory.TenantId;
this.client = kustQueryClient;
}
/// <inheritdoc />
protected override void Setup(GetKustoBuildInput input)
{
// try to connect..
m_queryProvider = GetKustoConnection(m_configuration.KustoConfig);
// and also parse the query
m_query = File.ReadAllText(Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), constants.GetBuildQuery))
.Replace("{0}", Convert.ToString(input.Year))
.Replace("{1}", Convert.ToString(input.Month))
.Replace("{2}", Convert.ToString(input.Day))
.Replace("{3}", Convert.ToString(input.NumBuilds * s_maxRetryBuilds))
.Replace("{4}", m_configuration.UseCBTest? constants.CBTestDatabaseName : constants.ProdDatabaseName);
s_logger.Debug($"Target Query: {m_query}");
}
public KustoClient(IServiceProvider serviceProvider, ILoggerFactory loggerFactory,
KustoSettings kustoSettings = null)
{
logger = loggerFactory.CreateLogger <KustoClient>();
var configuration = serviceProvider.GetRequiredService <IConfiguration>();
this.kustoSettings = kustoSettings ?? configuration.GetConfiguredSettings <KustoSettings>();
var clientFactory = new KustoClientFactory(serviceProvider, kustoSettings);
queryClient = clientFactory.QueryQueryClient;
adminClient = clientFactory.AdminClient;
ingestClient = clientFactory.IngestClient;
}
/// <summary>
/// Create a new GitHubNameResolver
/// </summary>
/// <param name="aadAppId">AAD App Id</param>
/// <param name="aadAppSecret">AAD App Secret</param>
/// <param name="aadTenant">AAD Tenant</param>
/// <param name="kustoUrl">Kusto URL</param>
/// <param name="kustoDatabaseName">Kusto DB Name</param>
/// <param name="kustoTable">Kusto Table Name</param>
/// <param name="logger">Logger</param>
public GitHubNameResolver(
string aadAppId,
string aadAppSecret,
string aadTenant,
string kustoUrl,
string kustoDatabaseName,
string kustoTable,
ILogger <GitHubNameResolver> logger)
{
this.client = GetKustoClient(aadAppId, aadAppSecret, aadTenant, kustoUrl, kustoDatabaseName);
this.kustoTable = kustoTable;
this.logger = logger;
}
public KustoClient(string clusterName, string database, int icm)
{
Log.Information("Creating Kusto connector for Cluster : {0} in database : {1}", clusterName, database);
var authority = Authentication.Instance.ServicePrincipal.tenant;
var applicationClientId = Authentication.Instance.ServicePrincipal.appId;
var applicationKey = Authentication.Instance.ServicePrincipal.password;
var kcsb = new KustoConnectionStringBuilder(String.Format("https://{0}.kusto.windows.net", clusterName), database)
.WithAadApplicationKeyAuthentication(applicationClientId, applicationKey, authority);
client = KustoClientFactory.CreateCslQueryProvider(kcsb);
Log.Information("Finished creating Kusto connector : {0}", kcsb);
}
private Monitor(Configuration configuration, IKustoIngestClient kustoIngestClient, ICslQueryProvider cslQueryProvider, IClock clock, IReadOnlyDictionary <CloudBuildEnvironment, EnvironmentResources> environmentResources, ILogger logger)
{
_configuration = configuration;
_clock = clock;
_logger = logger;
_kustoIngestClient = kustoIngestClient;
_cslQueryProvider = cslQueryProvider;
_environmentResources = environmentResources;
_alertNotifier = new KustoWriter <Notification>(_configuration.KustoNotifier, _logger, _kustoIngestClient);
_schedulerLogWriter = new KustoWriter <RuleScheduler.LogEntry>(_configuration.SchedulerKustoNotifier, _logger, _kustoIngestClient);
_scheduler = new RuleScheduler(_configuration.Scheduler, _logger, _clock, _schedulerLogWriter);
}
public KustoTimelineTelemetryRepository(ILogger <KustoTimelineTelemetryRepository> logger, IOptionsSnapshot <KustoTimelineTelemetryOptions> options)
{
_logger = logger;
// removing the IngestConnectionString was a default setup in local debugging
if (string.IsNullOrEmpty(options.Value.IngestConnectionString))
{
_logger.LogDebug("No ingest connection string provided; will ignore ingest operations");
_ingest = new NullKustoIngestClient();
}
else
{
_ingest = KustoIngestFactory.CreateQueuedIngestClient(options.Value.IngestConnectionString);
}
_query = KustoClientFactory.CreateCslQueryProvider(options.Value.QueryConnectionString);
_database = options.Value.Database;
}
private static void Initialize()
{
lock (_lock)
{
if (!_isInitialized)
{
KustoConnectionStringBuilder connection =
new KustoConnectionStringBuilder(Environment.GetEnvironmentVariable("kustoUrl", EnvironmentVariableTarget.Process)).WithAadApplicationKeyAuthentication(
applicationClientId: Environment.GetEnvironmentVariable("appClientId", EnvironmentVariableTarget.Process),
applicationKey: Environment.GetEnvironmentVariable("appClientSecret", EnvironmentVariableTarget.Process),
authority: Environment.GetEnvironmentVariable("tenantId", EnvironmentVariableTarget.Process));
adx = KustoClientFactory.CreateCslQueryProvider(connection);
_isInitialized = true;
}
}
}
