static async Task Main(string[] args)
{
// Async Ingestion From a Single Azure Blob using KustoQueuedIngestClient with (optional) RetryPolicy:
//Create Kusto connection string with App Authentication
var kustoConnectionStringBuilderDM =
new KustoConnectionStringBuilder(@"https://ingest-{clusterNameAndRegion}.kusto.windows.net").WithAadApplicationKeyAuthentication(
applicationClientId: "{Application Client ID}",
applicationKey: "{Application Key (secret)}",
authority: "{AAD TenantID or name}");
// Create an ingest client
// Note, that creating a separate instance per ingestion operation is an anti-pattern.
// IngestClient classes are thread-safe and intended for reuse
IKustoIngestClient client = KustoIngestFactory.CreateQueuedIngestClient(kustoConnectionStringBuilderDM);
// Ingest from blobs according to the required properties
var kustoIngestionProperties = new KustoIngestionProperties(databaseName: "myDB", tableName: "myTable");
var sourceOptions = new StorageSourceOptions()
{
DeleteSourceOnSuccess = true
};
//// Create your custom implementation of IRetryPolicy, which will affect how the ingest client handles retrying on transient failures
IRetryPolicy retryPolicy = new NoRetry();
//// This line sets the retry policy on the ingest client that will be enforced on every ingest call from here on
((IKustoQueuedIngestClient)client).QueueRetryPolicy = retryPolicy;
await client.IngestFromStorageAsync(uri : @"BLOB-URI-WITH-SAS-KEY", ingestionProperties : kustoIngestionProperties, sourceOptions);
client.Dispose();
}
public Monitor(Configuration configuration)
{
Contract.RequiresNotNull(configuration);
_configuration = configuration;
_logger = CreateLogger();
Contract.RequiresNotNullOrEmpty(_configuration.KustoIngestionClusterUrl);
Contract.RequiresNotNullOrEmpty(_configuration.ApplicationClientId);
Contract.RequiresNotNullOrEmpty(_configuration.ApplicationKey);
Contract.RequiresNotNullOrEmpty(_configuration.Authority);
var kustoIngestConnectionString = new KustoConnectionStringBuilder(_configuration.KustoIngestionClusterUrl)
.WithAadApplicationKeyAuthentication(_configuration.ApplicationClientId, _configuration.ApplicationKey, _configuration.Authority);
_kustoIngestClient = KustoIngestFactory.CreateDirectIngestClient(kustoIngestConnectionString);
var kustoConnectionString = new KustoConnectionStringBuilder(_configuration.KustoClusterUrl)
.WithAadApplicationKeyAuthentication(_configuration.ApplicationClientId, _configuration.ApplicationKey, _configuration.Authority);
_cslQueryProvider = KustoClientFactory.CreateCslQueryProvider(kustoConnectionString);
Contract.RequiresNotNull(_configuration.KustoNotifier);
_alertNotifier = new KustoWriter <Notification>(_configuration.KustoNotifier, _logger, _kustoIngestClient);
Contract.RequiresNotNull(_configuration.SchedulerKustoNotifier);
_schedulerLogWriter = new KustoWriter <Scheduler.LogEntry>(_configuration.SchedulerKustoNotifier, _logger, _kustoIngestClient);
_scheduler = new Scheduler(_configuration.Scheduler, _logger, _clock, _schedulerLogWriter);
}
public KustoClientLogger(string databaseName, string tableName, List <KustoColumnMapping> _schema, string clientId, string clientKey)
{
var csvColumnMappings = new List <CsvColumnMapping>();
foreach (var mapping in _schema)
{
csvColumnMappings.Add(new CsvColumnMapping {
ColumnName = mapping.ColumnName, CslDataType = mapping.DataType, Ordinal = mapping.ColumnNumber
});
}
_kustoProperties = new KustoIngestionProperties(databaseName, tableName)
{
Format = DataSourceFormat.csv,
CSVMapping = csvColumnMappings
};
var kustoConnectionStringBuilderDM = new KustoConnectionStringBuilder(@"https://wawstest.kusto.windows.net:443")
{
FederatedSecurity = true,
InitialCatalog = databaseName,
ApplicationKey = clientKey,
ApplicationClientId = clientId
};
_kustoClient = KustoIngestFactory.CreateDirectIngestClient(kustoConnectionStringBuilderDM);
}
public BlockingKustoUploader(
KustoConnectionStringBuilder kscb,
string tableName,
int batchSize,
TimeSpan flushDuration)
{
csb = kscb;
TableName = TableName;
BatchSize = batchSize;
_flushDuration = flushDuration;
_lastUploadTime = DateTime.Now;
Completed = new AutoResetEvent(false);
_ingestionProperties = new KustoIngestionProperties(csb.InitialCatalog, tableName);
_fields = typeof(EtwEvent).GetFields().Select(f => f.Name).ToArray();
if (csb.DataSource.StartsWith("https://ingest-"))
{
_ingestClient = KustoIngestFactory.CreateQueuedIngestClient(csb);
}
else
{
_ingestClient = KustoIngestFactory.CreateDirectIngestClient(csb);
}
_nextBatch = new List <T>();
}
public Monitor(Configuration configuration)
{
Contract.RequiresNotNull(configuration);
_configuration = configuration;
_logger = CreateLogger();
// TODO(jubayard): use streaming ingestion instead of direct ingestion. There seems to be some assembly
// issues when attempting to do that
Contract.RequiresNotNullOrEmpty(_configuration.KustoIngestionClusterUrl);
Contract.RequiresNotNullOrEmpty(_configuration.ApplicationClientId);
Contract.RequiresNotNullOrEmpty(_configuration.ApplicationKey);
Contract.RequiresNotNullOrEmpty(_configuration.Authority);
var kustoIngestConnectionString = new KustoConnectionStringBuilder(_configuration.KustoIngestionClusterUrl)
.WithAadApplicationKeyAuthentication(_configuration.ApplicationClientId, _configuration.ApplicationKey, _configuration.Authority);
_kustoIngestClient = KustoIngestFactory.CreateDirectIngestClient(kustoIngestConnectionString);
var kustoConnectionString = new KustoConnectionStringBuilder(_configuration.KustoClusterUrl)
.WithAadApplicationKeyAuthentication(_configuration.ApplicationClientId, _configuration.ApplicationKey, _configuration.Authority);
_cslQueryProvider = KustoClientFactory.CreateCslQueryProvider(kustoConnectionString);
Contract.RequiresNotNull(_configuration.KustoNotifier);
_alertNotifier = new KustoWriter <Notification>(_configuration.KustoNotifier, _logger, _kustoIngestClient);
Contract.RequiresNotNull(_configuration.SchedulerKustoNotifier);
_schedulerLogWriter = new KustoWriter <Scheduler.LogEntry>(_configuration.SchedulerKustoNotifier, _logger, _kustoIngestClient);
_scheduler = new Scheduler(_configuration.Scheduler, _logger, _clock, _schedulerLogWriter);
}
public AdxOutput(
BaseLogger logger,
string authority,
string appclientId,
string appKey,
string cluster,
string database,
string table,
bool createOrResetTable = false,
bool directIngest = false)
{
_logger = logger;
_table = table;
_createOrResetTable = createOrResetTable;
_batchSize = 10000;
_flushDuration = TimeSpan.FromMilliseconds(5);
_lastUploadTime = DateTime.UtcNow;
_initializeTable = false;
_nextBatch = new List <IDictionary <string, object> >();
Completed = new AutoResetEvent(false);
// Setting up kusto connection
if (!string.IsNullOrEmpty(authority))
{
if (!string.IsNullOrEmpty(appclientId) && !string.IsNullOrEmpty(appKey))
{
kscbIngest = new KustoConnectionStringBuilder($"https://ingest-{cluster}", database).WithAadApplicationKeyAuthentication(appclientId, appKey, authority);
kscbAdmin = new KustoConnectionStringBuilder($"https://{cluster}", database).WithAadApplicationKeyAuthentication(appclientId, appKey, authority);
}
else
{
kscbIngest = new KustoConnectionStringBuilder($"https://ingest-{cluster}", database).WithAadUserPromptAuthentication(authority);
kscbAdmin = new KustoConnectionStringBuilder($"https://{cluster}", database).WithAadUserPromptAuthentication(authority);
}
}
if (kscbAdmin != null)
{
_ingestionProperties = new KustoIngestionProperties(kscbIngest.InitialCatalog, table);
if (directIngest)
{
_ingestClient = KustoIngestFactory.CreateDirectIngestClient(this.kscbAdmin);
}
else
{
_ingestClient = KustoIngestFactory.CreateQueuedIngestClient(kscbIngest);
}
}
else
{
_logger.Log(LogLevel.ERROR, "ERROR getting ADX connection strings. Please double check the information provided.");
_error = true;
}
}
public TelemetryController(
ILogger <TelemetryController> logger,
IOptionsSnapshot <KustoOptions> options,
IKustoIngestClient client)
{
_logger = logger;
_options = options;
_client = client;
}
private async Task <IKustoIngestionResult> IngestFromStreamAsync(
string csv, IKustoIngestClient client, KustoIngestionProperties properties, StreamSourceOptions sourceOptions)
{
using MemoryStream stream = new();
using StreamWriter writer = new(stream);
writer.Write(csv);
writer.Flush();
stream.Seek(0, SeekOrigin.Begin);
return(await client.IngestFromStreamAsync(stream, properties, sourceOptions));
}
public KustoClient(IServiceProvider serviceProvider, ILoggerFactory loggerFactory,
KustoSettings kustoSettings = null)
{
logger = loggerFactory.CreateLogger <KustoClient>();
var configuration = serviceProvider.GetRequiredService <IConfiguration>();
this.kustoSettings = kustoSettings ?? configuration.GetConfiguredSettings <KustoSettings>();
var clientFactory = new KustoClientFactory(serviceProvider, kustoSettings);
queryClient = clientFactory.QueryQueryClient;
adminClient = clientFactory.AdminClient;
ingestClient = clientFactory.IngestClient;
}
public async Task IngestFromCsvAsync(
string csv, ServicePrincipalOptions servicePrincipal, string cluster, string database, string table, bool isDryRun)
{
KustoConnectionStringBuilder connectionBuilder =
new KustoConnectionStringBuilder($"https://{cluster}.kusto.windows.net")
.WithAadApplicationKeyAuthentication(
servicePrincipal.ClientId,
servicePrincipal.Secret,
servicePrincipal.Tenant);
using (IKustoIngestClient client = KustoIngestFactory.CreateDirectIngestClient(connectionBuilder))
{
KustoIngestionProperties properties =
new(database, table) { Format = DataSourceFormat.csv };
StreamSourceOptions sourceOptions = new() { SourceId = Guid.NewGuid() };
if (!isDryRun)
{
AsyncRetryPolicy retryPolicy = Policy
.Handle <Kusto.Data.Exceptions.KustoException>()
.Or <Kusto.Ingest.Exceptions.KustoException>()
.WaitAndRetryAsync(
Backoff.DecorrelatedJitterBackoffV2(TimeSpan.FromSeconds(10), RetryHelper.MaxRetries),
RetryHelper.GetOnRetryDelegate(RetryHelper.MaxRetries, _loggerService));
IKustoIngestionResult result = await retryPolicy.ExecuteAsync(
() => IngestFromStreamAsync(csv, client, properties, sourceOptions));
IngestionStatus ingestionStatus = result.GetIngestionStatusBySourceId(sourceOptions.SourceId);
for (int i = 0; i < 10 && ingestionStatus.Status == Status.Pending; i++)
{
await Task.Delay(TimeSpan.FromSeconds(30));
ingestionStatus = result.GetIngestionStatusBySourceId(sourceOptions.SourceId);
}
if (ingestionStatus.Status == Status.Pending)
{
throw new InvalidOperationException($"Timeout while ingesting Kusto data.");
}
else if (ingestionStatus.Status != Status.Succeeded)
{
throw new InvalidOperationException(
$"Failed to ingest Kusto data.{Environment.NewLine}{ingestionStatus.Details}");
}
}
}
}
private Monitor(Configuration configuration, IKustoIngestClient kustoIngestClient, ICslQueryProvider cslQueryProvider, IClock clock, IReadOnlyDictionary <CloudBuildEnvironment, EnvironmentResources> environmentResources, ILogger logger)
{
_configuration = configuration;
_clock = clock;
_logger = logger;
_kustoIngestClient = kustoIngestClient;
_cslQueryProvider = cslQueryProvider;
_environmentResources = environmentResources;
_alertNotifier = new KustoWriter <Notification>(_configuration.KustoNotifier, _logger, _kustoIngestClient);
_schedulerLogWriter = new KustoWriter <RuleScheduler.LogEntry>(_configuration.SchedulerKustoNotifier, _logger, _kustoIngestClient);
_scheduler = new RuleScheduler(_configuration.Scheduler, _logger, _clock, _schedulerLogWriter);
}
/// <summary>
/// Ingest data into Kusto.
/// </summary>
/// <param name="table">Name of table to ingest into.</param>
/// <param name="mappingName">Name of table mapping to ingest with.</param>
/// <param name="stream">input JSON data stream.</param>
/// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
private static async Task <IKustoIngestionResult> KustoIngest(KustoConnectionStringBuilder kusto, string db, string table, string mappingName, Stream stream)
{
// Create a disposable client that will execute the ingestion
using IKustoIngestClient client = KustoIngestFactory.CreateDirectIngestClient(kusto);
var ingestProps = new KustoIngestionProperties(db, table)
{
JSONMappingReference = mappingName,
Format = DataSourceFormat.json,
};
var ssOptions = new StreamSourceOptions
{
CompressionType = DataSourceCompressionType.GZip,
};
return(await client.IngestFromStreamAsync(stream, ingestProps, ssOptions));
}
public BlockingKustoUploader(
string blobConnectionString,
string blobContainerName,
KustoConnectionStringBuilder adminCsb,
KustoConnectionStringBuilder kscb,
bool demoMode,
string tableName,
int batchSize,
TimeSpan flushDuration,
bool resetTable = false)
{
Csb = kscb;
AdminCsb = adminCsb;
TableName = tableName;
BatchSize = batchSize;
_flushDuration = flushDuration;
_lastUploadTime = DateTime.UtcNow;
_resetTable = resetTable;
_initializeTable = false;
Completed = new AutoResetEvent(false);
if (Csb != null)
{
_ingestionProperties = new KustoIngestionProperties(Csb.InitialCatalog, tableName);
if (demoMode)
{
_ingestClient = KustoIngestFactory.CreateDirectIngestClient(this.AdminCsb);
}
else
{
_ingestClient = KustoIngestFactory.CreateQueuedIngestClient(Csb);
}
}
if (!string.IsNullOrEmpty(blobConnectionString) &&
!string.IsNullOrEmpty(blobContainerName))
{
blobServiceClient = new BlobServiceClient(blobConnectionString);
blobContainerClient = blobServiceClient.GetBlobContainerClient(blobContainerName);
blobContainerClient.CreateIfNotExists();
}
_nextBatch = new List <IDictionary <string, object> >();
}
public KustoTimelineTelemetryRepository(ILogger <KustoTimelineTelemetryRepository> logger, IOptionsSnapshot <KustoTimelineTelemetryOptions> options)
{
_logger = logger;
// removing the IngestConnectionString was a default setup in local debugging
if (string.IsNullOrEmpty(options.Value.IngestConnectionString))
{
_logger.LogDebug("No ingest connection string provided; will ignore ingest operations");
_ingest = new NullKustoIngestClient();
}
else
{
_ingest = KustoIngestFactory.CreateQueuedIngestClient(options.Value.IngestConnectionString);
}
_query = KustoClientFactory.CreateCslQueryProvider(options.Value.QueryConnectionString);
_database = options.Value.Database;
}
public BlockingKustoUploader(
string outputFileName,
KustoConnectionStringBuilder adminCsb,
KustoConnectionStringBuilder kscb,
bool demoMode,
string tableName,
int batchSize,
TimeSpan flushDuration,
bool resetTable = false)
{
OutputFileName = outputFileName;
Csb = kscb;
AdminCsb = adminCsb;
TableName = tableName;
BatchSize = batchSize;
_flushDuration = flushDuration;
_lastUploadTime = DateTime.UtcNow;
_resetTable = resetTable;
_initializeTable = false;
Completed = new AutoResetEvent(false);
if (Csb != null)
{
_ingestionProperties = new KustoIngestionProperties(Csb.InitialCatalog, tableName);
if (demoMode)
{
_ingestClient = KustoIngestFactory.CreateDirectIngestClient(this.AdminCsb);
}
else
{
_ingestClient = KustoIngestFactory.CreateQueuedIngestClient(Csb);
}
}
if (!string.IsNullOrEmpty(OutputFileName))
{
outputFile = new StreamWriter(this.OutputFileName);
outputFile.Write($"[{Environment.NewLine}");
}
_nextBatch = new List <IDictionary <string, object> >();
}
public async Task IngestFromCsvStreamAsync(Stream csv, IngestKustoImageInfoOptions options)
{
KustoConnectionStringBuilder connectionBuilder =
new KustoConnectionStringBuilder($"https://{options.Cluster}.kusto.windows.net")
.WithAadApplicationKeyAuthentication(
options.ServicePrincipal.ClientId,
options.ServicePrincipal.Secret,
options.ServicePrincipal.Tenant);
using (IKustoIngestClient client = KustoIngestFactory.CreateDirectIngestClient(connectionBuilder))
{
KustoIngestionProperties properties =
new KustoIngestionProperties(options.Database, options.Table)
{
Format = DataSourceFormat.csv
};
StreamSourceOptions sourceOptions = new StreamSourceOptions {
SourceId = Guid.NewGuid()
};
if (!options.IsDryRun)
{
IKustoIngestionResult result = await client.IngestFromStreamAsync(csv, properties, sourceOptions);
IngestionStatus ingestionStatus = result.GetIngestionStatusBySourceId(sourceOptions.SourceId);
for (int i = 0; i < 10 && ingestionStatus.Status == Status.Pending; i++)
{
await Task.Delay(TimeSpan.FromSeconds(30));
ingestionStatus = result.GetIngestionStatusBySourceId(sourceOptions.SourceId);
}
if (ingestionStatus.Status != Status.Succeeded)
{
throw new InvalidOperationException(
$"Failed to ingest Kusto data.{Environment.NewLine}{ingestionStatus.Details}");
}
else if (ingestionStatus.Status == Status.Pending)
{
throw new InvalidOperationException($"Timeout while ingesting Kusto data.");
}
}
}
}
public KustoNotifier(Configuration configuration, ILogger logger, IKustoIngestClient kustoIngestClient)
{
_configuration = configuration;
_logger = logger;
_kustoIngestClient = kustoIngestClient;
_kustoIngestionProperties = new KustoIngestionProperties(_configuration.KustoDatabaseName, _configuration.KustoTableName)
{
Format = DataSourceFormat.json,
};
Contract.RequiresNotNullOrEmpty(_configuration.KustoTableIngestionMappingName,
"Kusto ingestion will fail to authenticate without a proper ingestion mapping.");
_kustoIngestionProperties.JSONMappingReference = _configuration.KustoTableIngestionMappingName;
_queue = NagleQueue <T> .Create(FlushAsync,
_configuration.MaxDegreeOfParallelism,
_configuration.FlushInterval,
_configuration.BatchSize);
}
static void Main(string[] args)
{
// Ingest From Local File using KustoDirectIngestClient (only for test purposes):
// Create Kusto connection string with App Authentication
var kustoConnectionStringBuilderEngine =
new KustoConnectionStringBuilder(@"https://{clusterNameAndRegion}.kusto.windows.net").WithAadApplicationKeyAuthentication(
applicationClientId: "{Application Client ID}",
applicationKey: "{Application Key (secret)}",
authority: "{AAD TenantID or name}");
// Create a disposable client that will execute the ingestion
using (IKustoIngestClient client = KustoIngestFactory.CreateDirectIngestClient(kustoConnectionStringBuilderEngine))
{
//Ingest from blobs according to the required properties
var kustoIngestionProperties = new KustoIngestionProperties(databaseName: "myDB", tableName: "myTable");
client.IngestFromStorageAsync(@"< Path to local file >", ingestionProperties: kustoIngestionProperties).GetAwaiter().GetResult();
}
}
private static async Task IngestTriageItemsIntoKusto(TriageItem[] triageItems, ILogger log)
{
log.LogInformation("Entering IngestTriageItemIntoKusto");
string kustoIngestConnectionString = System.Environment.GetEnvironmentVariable("KustoIngestConnectionString");
string databaseName = System.Environment.GetEnvironmentVariable("KustoDatabaseName");
IKustoIngestClient ingestClient = KustoIngestFactory.CreateQueuedIngestClient(kustoIngestConnectionString);
await KustoHelpers.WriteDataToKustoInMemoryAsync(
ingestClient,
databaseName,
"TimelineIssuesTriage",
log,
triageItems,
b => new[]
{
new KustoValue("ModifiedDateTime", b.ModifiedDateTime.ToString(), KustoDataTypes.DateTime),
new KustoValue("BuildId", b.BuildId.ToString(), KustoDataTypes.Int),
new KustoValue("RecordId", b.RecordId.ToString(), KustoDataTypes.Guid),
new KustoValue("Index", b.Index.ToString(), KustoDataTypes.Int),
new KustoValue("UpdatedCategory", b?.UpdatedCategory, KustoDataTypes.String),
new KustoValue("Url", b?.Url, KustoDataTypes.String)
});
}
private Monitor(Configuration configuration, IKustoIngestClient kustoIngestClient, IIcmClient icmClient, IClock clock, IReadOnlyDictionary <MonitorEnvironment, EnvironmentResources> environmentResources, ILogger logger)
{
_configuration = configuration;
_clock = clock;
_logger = logger;
_kustoIngestClient = kustoIngestClient;
_icmClient = icmClient;
_environmentResources = environmentResources;
if (configuration.ReadOnly)
{
_alertNotifier = new LogNotifier <Notification>(_logger);
_schedulerLogWriter = new LogNotifier <RuleScheduler.LogEntry>(_logger);
}
else
{
_alertNotifier = new KustoNotifier <Notification>(_configuration.KustoNotifier, _logger, _kustoIngestClient);
_schedulerLogWriter = new KustoNotifier <RuleScheduler.LogEntry>(_configuration.SchedulerKustoNotifier, _logger, _kustoIngestClient);
}
_scheduler = new RuleScheduler(_configuration.Scheduler, _logger, _clock, _schedulerLogWriter);
}
static void IngestData(KustoConnectionStringBuilder engineKustoConnectionStringBuilder, KustoConnectionStringBuilder dmKustoConnectionStringBuilderDM,
string databaseName, string tableName)
{
// 1. Ingest by connecting directly to the Kustolab cluster and sending a command
using (IKustoIngestClient directClient = KustoIngestFactory.CreateDirectIngestClient(engineKustoConnectionStringBuilder))
{
var kustoIngestionProperties = new KustoIngestionProperties(databaseName, tableName);
directClient.IngestFromDataReaderAsync(GetDataAsIDataReader(), kustoIngestionProperties);
}
// 2. Ingest by submitting the data to the Kustolab ingestion cluster
// Note that this is an async operation, so data might not appear immediately
using (IKustoIngestClient queuedClient = KustoIngestFactory.CreateQueuedIngestClient(dmKustoConnectionStringBuilderDM))
{
var kustoIngestionProperties = new KustoIngestionProperties(databaseName, tableName);
queuedClient.IngestFromDataReaderAsync(GetDataAsIDataReader(), kustoIngestionProperties);
}
// 3. Ingest by submitting the data to the Kustolab ingestion cluster -
// This time, update the report method and level so you can track the status of your ingestion
// using the IKustoIngestionResult returned from the ingest operation.
IKustoIngestionResult ingestionResult;
using (IKustoIngestClient queuedClient = KustoIngestFactory.CreateQueuedIngestClient(dmKustoConnectionStringBuilderDM))
{
var kustoIngestionProperties = new KustoQueuedIngestionProperties(databaseName, tableName)
{
// The default ReportLevel is set to FailuresOnly.
// In this case we want to check the status of successful ingestions as well.
ReportLevel = IngestionReportLevel.FailuresAndSuccesses,
// You can use either a queue or a table to track the status of your ingestion.
ReportMethod = IngestionReportMethod.Table
};
ingestionResult = queuedClient.IngestFromDataReaderAsync(GetDataAsIDataReader(), kustoIngestionProperties).Result;
}
// Obtain the status of our ingestion
var ingestionStatus = ingestionResult.GetIngestionStatusCollection().First();
var watch = System.Diagnostics.Stopwatch.StartNew();
var shouldContinue = true;
while ((ingestionStatus.Status == Status.Pending) && (shouldContinue))
{
// Wait a minute...
Thread.Sleep(TimeSpan.FromMinutes(1));
// Try again
ingestionStatus = ingestionResult.GetIngestionStatusBySourceId(ingestionStatus.IngestionSourceId);
shouldContinue = watch.ElapsedMilliseconds < TimeOutInMilliSeconds;
}
watch.Stop();
if (ingestionStatus.Status == Status.Pending)
{
// The status of the ingestion did not change.
Console.WriteLine(
"Ingestion with ID:'{0}' did not complete. Timed out after :'{1}' Milliseconds"
.FormatWithInvariantCulture(
ingestionStatus.IngestionSourceId, watch.ElapsedMilliseconds));
}
else
{
// The status of the ingestion has changed
Console.WriteLine(
"Ingestion with ID:'{0}' is complete. Ingestion Status:'{1}'".FormatWithInvariantCulture(
ingestionStatus.IngestionSourceId, ingestionStatus.Status));
}
// 4. Show the contents of the table
using (var client = Kusto.Data.Net.Client.KustoClientFactory.CreateCslQueryProvider(engineKustoConnectionStringBuilder))
{
while (true)
{
var query = string.Format("{0}", tableName);
var reader = client.ExecuteQuery(query);
Kusto.Cloud.Platform.Data.ExtendedDataReader.WriteAsText(reader, "Data ingested into the table:", tabify: true, firstOnly: true);
Console.WriteLine("Press 'r' to retry retrieving data from the table, any other key to quit");
var key = Console.ReadKey();
if (key.KeyChar != 'r' && key.KeyChar != 'R')
{
break;
}
Console.WriteLine();
}
}
}
public static void Main(string[] args)
{
KustoConnectionStringBuilder readStringBuilder;
KustoConnectionStringBuilder ingestStringBuilder;
readStringBuilder = new KustoConnectionStringBuilder(@"https://kuskusops.kustomfa.windows.net")
{
FederatedSecurity = true,
InitialCatalog = "testdb",
Authority = "72f988bf-86f1-41af-91ab-2d7cd011db47"
};
ingestStringBuilder = new KustoConnectionStringBuilder($"https://ingest-kuskusops.kustomfa.windows.net")
{
FederatedSecurity = true,
InitialCatalog = "testdb",
Authority = "72f988bf-86f1-41af-91ab-2d7cd011db47"
};
using (IKustoIngestClient kustoIngestClient = KustoIngestFactory.CreateQueuedIngestClient(ingestStringBuilder))
{
var kustoIngestionProperties = new KustoQueuedIngestionProperties("testdb", "test_table");
var dataReader = GetDataAsIDataReader();
IKustoIngestionResult ingestionResult =
kustoIngestClient.IngestFromDataReader(dataReader, kustoIngestionProperties);
var ingestionStatus = ingestionResult.GetIngestionStatusCollection().First();
var watch = System.Diagnostics.Stopwatch.StartNew();
var shouldContinue = true;
while ((ingestionStatus.Status == Status.Pending) && (shouldContinue))
{
// Wait a minute...
Thread.Sleep(TimeSpan.FromMinutes(1));
// Try again
ingestionStatus = ingestionResult.GetIngestionStatusBySourceId(ingestionStatus.IngestionSourceId);
shouldContinue = watch.ElapsedMilliseconds < 360000;
}
watch.Stop();
if (ingestionStatus.Status == Status.Pending)
{
// The status of the ingestion did not change.
Console.WriteLine(
"Ingestion with ID:'{0}' did not complete. Timed out after :'{1}' Milliseconds"
.FormatWithInvariantCulture(
ingestionStatus.IngestionSourceId, watch.ElapsedMilliseconds));
}
else
{
// The status of the ingestion has changed
Console.WriteLine(
"Ingestion with ID:'{0}' is complete. Ingestion Status:'{1}'".FormatWithInvariantCulture(
ingestionStatus.IngestionSourceId, ingestionStatus.Status));
}
}
// 4. Show the contents of the table
using (var client = Kusto.Data.Net.Client.KustoClientFactory.CreateCslQueryProvider(readStringBuilder))
{
while (true)
{
var query = "test_table | count";
var reader = client.ExecuteQuery(query);
Kusto.Cloud.Platform.Data.ExtendedDataReader.WriteAsText(reader, "Data ingested into the table:", tabify: true, firstOnly: true);
Console.WriteLine("Press 'r' to retry retrieving data from the table, any other key to quit");
var key = Console.ReadKey();
if (key.KeyChar != 'r' || key.KeyChar != 'R')
{
break;
}
}
}
}
public ActionResult Upload(string instanceUrl, string databaseName, string tenantId, string clientId, string secret)
{
ADXModel model = new ADXModel
{
InstanceUrl = instanceUrl,
DatabaseName = databaseName,
TenantId = tenantId,
ClientId = clientId,
Secret = secret,
};
try
{
var kustoConnectionStringBuilder = new KustoConnectionStringBuilder(instanceUrl).WithAadApplicationKeyAuthentication(clientId, secret, tenantId);
string tableName = DTDL._nodesetNamespaceURI.Replace("http://", "").Replace('/', '_').Replace('.', '_').TrimEnd('_');
using (var kustoClient = KustoClientFactory.CreateCslAdminProvider(kustoConnectionStringBuilder))
{
kustoClient.ExecuteControlCommand(databaseName, ".create table " + tableName + " (ExpandedNodeID: string, DisplayName: string, Type: string, ParentNodeID: string)");
string command = ".create table " + tableName + " ingestion json mapping 'Nodeset_Mapping' '["
+ "{ \"properties\": { \"path\": \"$.Key\" }, \"column\": \"ExpandedNodeID\", \"datatype\": \"string\" },"
+ "{ \"properties\": { \"path\": \"$.Value.Item1\" }, \"column\": \"DisplayName\", \"datatype\": \"string\" } ,"
+ "{ \"properties\": { \"path\": \"$.Value.Item2\" }, \"column\": \"Type\", \"datatype\": \"string\" },"
+ "{ \"properties\": { \"path\": \"$.Value.Item3\" }, \"column\": \"ParentNodeID\", \"datatype\": \"string\" } ]'";
kustoClient.ExecuteControlCommand(databaseName, command);
command = ".alter table " + tableName + " policy ingestionbatching @'{"
+ "\"MaximumBatchingTimeSpan\": \"00:00:05\","
+ "\"MaximumNumberOfItems\": 100,"
+ "\"MaximumRawDataSizeMB\": 1024 }'";
kustoClient.ExecuteControlCommand(databaseName, command);
}
using (IKustoIngestClient client = KustoIngestFactory.CreateDirectIngestClient(kustoConnectionStringBuilder))
{
var kustoIngestionProperties = new KustoIngestionProperties(databaseName, tableName);
kustoIngestionProperties.Format = DataSourceFormat.multijson;
kustoIngestionProperties.IngestionMapping = new IngestionMapping()
{
IngestionMappingReference = "Nodeset_Mapping",
IngestionMappingKind = IngestionMappingKind.Json
};
foreach (KeyValuePair <string, Tuple <string, string, string> > entry in DTDL._nodeList.ToArray(100))
{
string content = JsonConvert.SerializeObject(entry, Formatting.Indented);
MemoryStream stream = new MemoryStream(Encoding.UTF8.GetBytes(content));
client.IngestFromStream(stream, kustoIngestionProperties);
stream.Dispose();
}
}
using (var cslQueryProvider = KustoClientFactory.CreateCslQueryProvider(kustoConnectionStringBuilder))
{
string query = $"{tableName} | count";
var results = cslQueryProvider.ExecuteQuery <long>(databaseName, query);
foreach (long result in results)
{
model.StatusMessage = "ADX Ingestion succeeded.";
return(View("Index", model));
}
}
model.StatusMessage = "ADX Ingestion failed!";
return(View("Error", model));
}
catch (Exception ex)
{
model.StatusMessage = ex.Message;
return(View("Index", model));
}
}
public static async Task WriteDataToKustoInMemoryAsync <T>(
IKustoIngestClient client,
string databaseName,
string tableName,
ILogger logger,
IEnumerable <T> data,
Func <T, IList <KustoValue> > mapFunc)
{
CsvColumnMapping[] mappings = null;
int size = 5;
using (var stream = new MemoryStream())
{
using (var writer = new StreamWriter(stream, new UTF8Encoding(false), 1024, leaveOpen: true))
{
foreach (T d in data)
{
var dataList = new List <string>(size);
if (mappings == null)
{
var mapList = new List <CsvColumnMapping>();
foreach (KustoValue p in mapFunc(d))
{
mapList.Add(new CsvColumnMapping {
ColumnName = p.Column, CslDataType = p.DataType
});
dataList.Add(p.StringValue);
}
mappings = mapList.ToArray();
size = mappings.Length;
}
else
{
dataList.AddRange(mapFunc(d).Select(p => p.StringValue));
}
await writer.WriteCsvLineAsync(dataList);
}
}
if (mappings == null)
{
logger.LogInformation("No rows to upload.");
return;
}
for (int i = 0; i < mappings.Length; i++)
{
mappings[i].Ordinal = i;
}
stream.Seek(0, SeekOrigin.Begin);
logger.LogInformation($"Ingesting {mappings.Length} columns at {stream.Length} bytes...");
await client.IngestFromStreamAsync(
stream,
new KustoQueuedIngestionProperties(databaseName, tableName)
{
Format = DataSourceFormat.csv,
ReportLevel = IngestionReportLevel.FailuresOnly,
ReportMethod = IngestionReportMethod.Queue,
CSVMapping = mappings
});
logger.LogTrace("Ingest complete");
}
}
private async Task RunProject(
AzureDevOpsClient azureServer,
string project,
int buildBatchSize,
AzureDevOpsTimelineOptions options,
CancellationToken cancellationToken)
{
DateTimeOffset latest;
try
{
using (ICslQueryProvider query =
KustoClientFactory.CreateCslQueryProvider(options.KustoQueryConnectionString))
using (IDataReader result = await query.ExecuteQueryAsync(
options.KustoDatabase,
// This isn't use controlled, so I'm not worried about the Kusto injection
$"TimelineBuilds | where Project == '{project}' | summarize max(FinishTime)",
new ClientRequestProperties()
))
{
if (!result.Read())
{
latest = DateTimeOffset.UtcNow.Subtract(TimeSpan.FromDays(30));
_logger.LogWarning($"No previous time found, using {latest.LocalDateTime:O}");
}
else
{
latest = result.GetDateTime(0);
_logger.LogInformation($"... fetched previous time of {latest.LocalDateTime:O}");
}
}
}
catch (SemanticException e) when(e.SemanticErrors == "'where' operator: Failed to resolve column or scalar expression named 'Project'")
{
// The Project column isn't there, we probably reinitalized the tables
latest = DateTimeOffset.UtcNow.Subtract(TimeSpan.FromDays(30));
_logger.LogWarning($"No table column 'Project' found, assumed reinialization: using {latest.LocalDateTime:O}");
}
_logger.LogInformation("Reading project {project}", project);
Build[] builds = await GetBuildsAsync(azureServer, project, latest, buildBatchSize, cancellationToken);
_logger.LogTrace("... found {builds} builds...", builds.Length);
if (builds.Length == 0)
{
_logger.LogTrace("No work to do");
return;
}
List <(int buildId, BuildRequestValidationResult validationResult)> validationResults = builds
.SelectMany(
build => build.ValidationResults,
(build, validationResult) => (build.Id, validationResult))
.ToList();
_logger.LogTrace("Fetching timeline...");
Dictionary <Build, Task <Timeline> > tasks = builds
.ToDictionary(
build => build,
build => azureServer.GetTimelineAsync(project, build.Id, cancellationToken)
);
await Task.WhenAll(tasks.Select(s => s.Value));
_logger.LogTrace("... finished timeline");
var records = new List <AugmentedTimelineRecord>();
var issues = new List <AugmentedTimelineIssue>();
_logger.LogTrace("Aggregating results...");
foreach ((Build build, Task <Timeline> timelineTask) in tasks)
{
Timeline timeline = await timelineTask;
if (timeline?.Records == null)
{
continue;
}
var recordCache =
new Dictionary <string, AugmentedTimelineRecord>();
var issueCache = new List <AugmentedTimelineIssue>();
foreach (TimelineRecord record in timeline.Records)
{
var augRecord = new AugmentedTimelineRecord(build.Id, record);
recordCache.Add(record.Id, augRecord);
records.Add(augRecord);
if (record.Issues == null)
{
continue;
}
for (int iIssue = 0; iIssue < record.Issues.Length; iIssue++)
{
var augIssue =
new AugmentedTimelineIssue(build.Id, record.Id, iIssue, record.Issues[iIssue]);
augIssue.Bucket = GetBucket(augIssue);
issueCache.Add(augIssue);
issues.Add(augIssue);
}
}
foreach (AugmentedTimelineRecord record in recordCache.Values)
{
FillAugmentedOrder(record, recordCache);
}
foreach (AugmentedTimelineIssue issue in issueCache)
{
if (recordCache.TryGetValue(issue.RecordId, out AugmentedTimelineRecord record))
{
issue.AugmentedIndex = record.AugmentedOrder + "." + issue.Index.ToString("D3");
}
else
{
issue.AugmentedIndex = "999." + issue.Index.ToString("D3");
}
}
}
if (string.IsNullOrEmpty(options.KustoIngestConnectionString))
{
_logger.LogError("No KustoIngestConnectionString set");
return;
}
IKustoIngestClient ingest =
KustoIngestFactory.CreateQueuedIngestClient(options.KustoIngestConnectionString);
_logger.LogInformation("Saving TimelineBuilds...");
await KustoHelpers.WriteDataToKustoInMemoryAsync(
ingest,
options.KustoDatabase,
"TimelineBuilds",
_logger,
builds,
b => new[]
{
new KustoValue("BuildId", b.Id.ToString(), KustoDataTypes.Int),
new KustoValue("Status", b.Status, KustoDataTypes.String),
new KustoValue("Result", b.Result, KustoDataTypes.String),
new KustoValue("Repository", b.Repository?.Name ?? b.Repository?.Id, KustoDataTypes.String),
new KustoValue("Reason", b.Reason, KustoDataTypes.String),
new KustoValue("BuildNumber", b.BuildNumber, KustoDataTypes.String),
new KustoValue("QueueTime", b.QueueTime, KustoDataTypes.DateTime),
new KustoValue("StartTime", b.StartTime, KustoDataTypes.DateTime),
new KustoValue("FinishTime", b.FinishTime, KustoDataTypes.DateTime),
new KustoValue("Project", b.Project?.Name, KustoDataTypes.String),
new KustoValue("DefinitionId", b.Definition?.Id.ToString(), KustoDataTypes.String),
new KustoValue("Definition", $"{b.Definition?.Path}\\{b.Definition?.Name}", KustoDataTypes.String),
});
_logger.LogInformation("Saving TimelineValidationMessages...");
await KustoHelpers.WriteDataToKustoInMemoryAsync(
ingest,
options.KustoDatabase,
"TimelineIssues",
_logger,
validationResults,
b => new[]
{
new KustoValue("BuildId", b.buildId.ToString(), KustoDataTypes.Int),
new KustoValue("RecordId", null, KustoDataTypes.String),
new KustoValue("Index", null, KustoDataTypes.Int),
new KustoValue("Path", null, KustoDataTypes.String),
new KustoValue("Type", b.validationResult.Result, KustoDataTypes.String),
new KustoValue("Category", "ValidationResult", KustoDataTypes.String),
new KustoValue("Message", b.validationResult.Message, KustoDataTypes.String),
new KustoValue("Bucket", "ValidationResult", KustoDataTypes.String),
});
_logger.LogInformation("Saving TimelineRecords...");
await KustoHelpers.WriteDataToKustoInMemoryAsync(
ingest,
options.KustoDatabase,
"TimelineRecords",
_logger,
records,
b => new[]
{
new KustoValue("BuildId", b.BuildId.ToString(), KustoDataTypes.Int),
new KustoValue("RecordId", b.Raw.Id, KustoDataTypes.String),
new KustoValue("Order", b.Raw.Order.ToString(), KustoDataTypes.Int),
new KustoValue("Path", b.AugmentedOrder, KustoDataTypes.String),
new KustoValue("ParentId", b.Raw.ParentId, KustoDataTypes.String),
new KustoValue("Name", b.Raw.Name, KustoDataTypes.String),
new KustoValue("StartTime", b.Raw.StartTime, KustoDataTypes.DateTime),
new KustoValue("FinishTime", b.Raw.FinishTime, KustoDataTypes.DateTime),
new KustoValue("Result", b.Raw.Result, KustoDataTypes.String),
new KustoValue("ResultCode", b.Raw.ResultCode, KustoDataTypes.String),
new KustoValue("ChangeId", b.Raw.ChangeId.ToString(), KustoDataTypes.Int),
new KustoValue("LastModified", b.Raw.LastModified, KustoDataTypes.DateTime),
new KustoValue("WorkerName", b.Raw.WorkerName, KustoDataTypes.String),
new KustoValue("Details", b.Raw.Details?.Url, KustoDataTypes.String),
new KustoValue("ErrorCount", b.Raw.ErrorCount.ToString(), KustoDataTypes.Int),
new KustoValue("WarningCount", b.Raw.WarningCount.ToString(), KustoDataTypes.Int),
new KustoValue("Url", b.Raw.Url, KustoDataTypes.String),
new KustoValue("LogId", b.Raw.Log?.Id.ToString(), KustoDataTypes.Int),
new KustoValue("LogUri", b.Raw.Log?.Url, KustoDataTypes.String),
new KustoValue("TaskId", b.Raw.Task?.Id, KustoDataTypes.Int),
new KustoValue("TaskName", b.Raw.Task?.Name, KustoDataTypes.String),
new KustoValue("TaskVersion", b.Raw.Task?.Version, KustoDataTypes.String),
new KustoValue("Attempt", b.Raw.Attempt.ToString(), KustoDataTypes.Int),
});
_logger.LogInformation("Saving TimelineIssues...");
await KustoHelpers.WriteDataToKustoInMemoryAsync(
ingest,
options.KustoDatabase,
"TimelineIssues",
_logger,
issues,
b => new[]
{
new KustoValue("BuildId", b.BuildId.ToString(), KustoDataTypes.Int),
new KustoValue("RecordId", b.RecordId, KustoDataTypes.String),
new KustoValue("Index", b.Index.ToString(), KustoDataTypes.Int),
new KustoValue("Path", b.AugmentedIndex, KustoDataTypes.String),
new KustoValue("Type", b.Raw.Type, KustoDataTypes.String),
new KustoValue("Category", b.Raw.Category, KustoDataTypes.String),
new KustoValue("Message", b.Raw.Message, KustoDataTypes.String),
new KustoValue("Bucket", b.Bucket, KustoDataTypes.String),
});
}
