public Monitor(Configuration configuration)
{
Contract.RequiresNotNull(configuration);
_configuration = configuration;
_logger = CreateLogger();
Contract.RequiresNotNullOrEmpty(_configuration.KustoIngestionClusterUrl);
Contract.RequiresNotNullOrEmpty(_configuration.ApplicationClientId);
Contract.RequiresNotNullOrEmpty(_configuration.ApplicationKey);
Contract.RequiresNotNullOrEmpty(_configuration.Authority);
var kustoIngestConnectionString = new KustoConnectionStringBuilder(_configuration.KustoIngestionClusterUrl)
.WithAadApplicationKeyAuthentication(_configuration.ApplicationClientId, _configuration.ApplicationKey, _configuration.Authority);
_kustoIngestClient = KustoIngestFactory.CreateDirectIngestClient(kustoIngestConnectionString);
var kustoConnectionString = new KustoConnectionStringBuilder(_configuration.KustoClusterUrl)
.WithAadApplicationKeyAuthentication(_configuration.ApplicationClientId, _configuration.ApplicationKey, _configuration.Authority);
_cslQueryProvider = KustoClientFactory.CreateCslQueryProvider(kustoConnectionString);
Contract.RequiresNotNull(_configuration.KustoNotifier);
_alertNotifier = new KustoWriter <Notification>(_configuration.KustoNotifier, _logger, _kustoIngestClient);
Contract.RequiresNotNull(_configuration.SchedulerKustoNotifier);
_schedulerLogWriter = new KustoWriter <Scheduler.LogEntry>(_configuration.SchedulerKustoNotifier, _logger, _kustoIngestClient);
_scheduler = new Scheduler(_configuration.Scheduler, _logger, _clock, _schedulerLogWriter);
}
public BlockingKustoUploader(
KustoConnectionStringBuilder kscb,
string tableName,
int batchSize,
TimeSpan flushDuration)
{
csb = kscb;
TableName = TableName;
BatchSize = batchSize;
_flushDuration = flushDuration;
_lastUploadTime = DateTime.Now;
Completed = new AutoResetEvent(false);
_ingestionProperties = new KustoIngestionProperties(csb.InitialCatalog, tableName);
_fields = typeof(EtwEvent).GetFields().Select(f => f.Name).ToArray();
if (csb.DataSource.StartsWith("https://ingest-"))
{
_ingestClient = KustoIngestFactory.CreateQueuedIngestClient(csb);
}
else
{
_ingestClient = KustoIngestFactory.CreateDirectIngestClient(csb);
}
_nextBatch = new List <T>();
}
public KustoClientLogger(string databaseName, string tableName, List <KustoColumnMapping> _schema, string clientId, string clientKey)
{
var csvColumnMappings = new List <CsvColumnMapping>();
foreach (var mapping in _schema)
{
csvColumnMappings.Add(new CsvColumnMapping {
ColumnName = mapping.ColumnName, CslDataType = mapping.DataType, Ordinal = mapping.ColumnNumber
});
}
_kustoProperties = new KustoIngestionProperties(databaseName, tableName)
{
Format = DataSourceFormat.csv,
CSVMapping = csvColumnMappings
};
var kustoConnectionStringBuilderDM = new KustoConnectionStringBuilder(@"https://wawstest.kusto.windows.net:443")
{
FederatedSecurity = true,
InitialCatalog = databaseName,
ApplicationKey = clientKey,
ApplicationClientId = clientId
};
_kustoClient = KustoIngestFactory.CreateDirectIngestClient(kustoConnectionStringBuilderDM);
}
public Monitor(Configuration configuration)
{
Contract.RequiresNotNull(configuration);
_configuration = configuration;
_logger = CreateLogger();
// TODO(jubayard): use streaming ingestion instead of direct ingestion. There seems to be some assembly
// issues when attempting to do that
Contract.RequiresNotNullOrEmpty(_configuration.KustoIngestionClusterUrl);
Contract.RequiresNotNullOrEmpty(_configuration.ApplicationClientId);
Contract.RequiresNotNullOrEmpty(_configuration.ApplicationKey);
Contract.RequiresNotNullOrEmpty(_configuration.Authority);
var kustoIngestConnectionString = new KustoConnectionStringBuilder(_configuration.KustoIngestionClusterUrl)
.WithAadApplicationKeyAuthentication(_configuration.ApplicationClientId, _configuration.ApplicationKey, _configuration.Authority);
_kustoIngestClient = KustoIngestFactory.CreateDirectIngestClient(kustoIngestConnectionString);
var kustoConnectionString = new KustoConnectionStringBuilder(_configuration.KustoClusterUrl)
.WithAadApplicationKeyAuthentication(_configuration.ApplicationClientId, _configuration.ApplicationKey, _configuration.Authority);
_cslQueryProvider = KustoClientFactory.CreateCslQueryProvider(kustoConnectionString);
Contract.RequiresNotNull(_configuration.KustoNotifier);
_alertNotifier = new KustoWriter <Notification>(_configuration.KustoNotifier, _logger, _kustoIngestClient);
Contract.RequiresNotNull(_configuration.SchedulerKustoNotifier);
_schedulerLogWriter = new KustoWriter <Scheduler.LogEntry>(_configuration.SchedulerKustoNotifier, _logger, _kustoIngestClient);
_scheduler = new Scheduler(_configuration.Scheduler, _logger, _clock, _schedulerLogWriter);
}
public AdxOutput(
BaseLogger logger,
string authority,
string appclientId,
string appKey,
string cluster,
string database,
string table,
bool createOrResetTable = false,
bool directIngest = false)
{
_logger = logger;
_table = table;
_createOrResetTable = createOrResetTable;
_batchSize = 10000;
_flushDuration = TimeSpan.FromMilliseconds(5);
_lastUploadTime = DateTime.UtcNow;
_initializeTable = false;
_nextBatch = new List <IDictionary <string, object> >();
Completed = new AutoResetEvent(false);
// Setting up kusto connection
if (!string.IsNullOrEmpty(authority))
{
if (!string.IsNullOrEmpty(appclientId) && !string.IsNullOrEmpty(appKey))
{
kscbIngest = new KustoConnectionStringBuilder($"https://ingest-{cluster}", database).WithAadApplicationKeyAuthentication(appclientId, appKey, authority);
kscbAdmin = new KustoConnectionStringBuilder($"https://{cluster}", database).WithAadApplicationKeyAuthentication(appclientId, appKey, authority);
}
else
{
kscbIngest = new KustoConnectionStringBuilder($"https://ingest-{cluster}", database).WithAadUserPromptAuthentication(authority);
kscbAdmin = new KustoConnectionStringBuilder($"https://{cluster}", database).WithAadUserPromptAuthentication(authority);
}
}
if (kscbAdmin != null)
{
_ingestionProperties = new KustoIngestionProperties(kscbIngest.InitialCatalog, table);
if (directIngest)
{
_ingestClient = KustoIngestFactory.CreateDirectIngestClient(this.kscbAdmin);
}
else
{
_ingestClient = KustoIngestFactory.CreateQueuedIngestClient(kscbIngest);
}
}
else
{
_logger.Log(LogLevel.ERROR, "ERROR getting ADX connection strings. Please double check the information provided.");
_error = true;
}
}
public static Result <IKustoIngestClient> CreateKustoIngestClient(string kustoIngestionClusterUrl, string azureTenantId, string azureAppId, string azureAppKey)
{
Contract.RequiresNotNullOrEmpty(kustoIngestionClusterUrl);
Contract.RequiresNotNullOrEmpty(azureTenantId);
Contract.RequiresNotNullOrEmpty(azureAppId);
Contract.RequiresNotNullOrEmpty(azureAppKey);
var kustoIngestConnectionString = new KustoConnectionStringBuilder(kustoIngestionClusterUrl)
.WithAadApplicationKeyAuthentication(azureAppId, azureAppKey, azureTenantId);
return(new Result <IKustoIngestClient>(KustoIngestFactory.CreateDirectIngestClient(kustoIngestConnectionString)));
}
/// <summary>
/// Performs a write to the indicated table, sending the appropriately filtered test report to Kusto
/// Returns the number of lines written
/// Skips the final write step when testing
/// </summary>
public bool WriteDataForTable(bool testing, List <AdoBuildResultRow> reports)
{
try
{
// Create Kusto connection string with App Authentication
var kustoConnectionStringBuilderEngine =
new KustoConnectionStringBuilder(config.clusterUri).WithAadApplicationKeyAuthentication(
applicationClientId: config.clientId,
applicationKey: config.secretKey,
authority: config.authority);
using (var ingestClient = KustoIngestFactory.CreateDirectIngestClient(kustoConnectionStringBuilderEngine))
{
var ingestProps = new KustoQueuedIngestionProperties(config.dbName, config.tableName);
// For the sake of getting both failure and success notifications we set this to IngestionReportLevel.FailuresAndSuccesses
// Usually the recommended level is IngestionReportLevel.FailuresOnly
ingestProps.ReportLevel = IngestionReportLevel.FailuresAndSuccesses;
ingestProps.ReportMethod = IngestionReportMethod.Queue;
ingestProps.IngestionMapping = new IngestionMapping {
IngestionMappingReference = config.mappingName
};
ingestProps.Format = DataSourceFormat.json;
// Prepare data for ingestion
using (var memStream = new MemoryStream())
using (var writer = new StreamWriter(memStream))
{
foreach (AdoBuildResultRow eachReport in reports)
{
writer.WriteLine(JsonConvert.SerializeObject(eachReport));
Console.WriteLine($"Writing: {eachReport.jobname}, ({eachReport.passed}/{eachReport.failed}/{eachReport.other}/{eachReport.total}) for {eachReport.timestamp}");
}
writer.Flush();
memStream.Seek(0, SeekOrigin.Begin);
// Post ingestion message
if (!testing)
{
ingestClient.IngestFromStream(memStream, ingestProps, leaveOpen: true);
}
}
}
return(true);
}
catch (Exception e)
{
Console.WriteLine($"ERROR: Could not write to: {config.clusterUri}.{config.dbName}.{config.tableName}, using Mapping: {config.mappingName}");
Console.WriteLine(e.ToString());
return(false);
}
}
internal IBigBrother UseKustoInternal(string kustoEngineName, string kustoEngineLocation, string kustoDb, string tenantId)
{
try
{
KustoDbName = kustoDb;
var kustoUri = $"https://{kustoEngineName}.{kustoEngineLocation}.kusto.windows.net";
var token = new AzureServiceTokenProvider().GetAccessTokenAsync(kustoUri, string.Empty).Result;
KustoAdminClient = KustoClientFactory.CreateCslAdminProvider(
new KustoConnectionStringBuilder(kustoUri)
{
FederatedSecurity = true,
InitialCatalog = KustoDbName,
Authority = tenantId,
ApplicationToken = token
});
var kustoQueuedIngestUri = $"https://ingest-{kustoEngineName}.{kustoEngineLocation}.kusto.windows.net";
var kustoIngestUri = $"https://{kustoEngineName}.{kustoEngineLocation}.kusto.windows.net";
KustoQueuedIngestClient = KustoIngestFactory.CreateQueuedIngestClient(
new KustoConnectionStringBuilder(kustoQueuedIngestUri)
{
FederatedSecurity = true,
InitialCatalog = KustoDbName,
Authority = tenantId,
ApplicationToken = token
});
KustoDirectIngestClient = KustoIngestFactory.CreateDirectIngestClient(
new KustoConnectionStringBuilder(kustoIngestUri)
{
FederatedSecurity = true,
InitialCatalog = KustoDbName,
Authority = tenantId,
ApplicationToken = token
});
SetupKustoSubscription();
KustoIngestionTimeMetric = InternalClient.GetMetric(new MetricIdentifier("", "KustoIngestionTime"));
}
catch (Exception e)
{
InternalStream.OnNext(e.ToExceptionEvent());
}
return(this);
}
public async Task IngestFromCsvAsync(
string csv, ServicePrincipalOptions servicePrincipal, string cluster, string database, string table, bool isDryRun)
{
KustoConnectionStringBuilder connectionBuilder =
new KustoConnectionStringBuilder($"https://{cluster}.kusto.windows.net")
.WithAadApplicationKeyAuthentication(
servicePrincipal.ClientId,
servicePrincipal.Secret,
servicePrincipal.Tenant);
using (IKustoIngestClient client = KustoIngestFactory.CreateDirectIngestClient(connectionBuilder))
{
KustoIngestionProperties properties =
new(database, table) { Format = DataSourceFormat.csv };
StreamSourceOptions sourceOptions = new() { SourceId = Guid.NewGuid() };
if (!isDryRun)
{
AsyncRetryPolicy retryPolicy = Policy
.Handle <Kusto.Data.Exceptions.KustoException>()
.Or <Kusto.Ingest.Exceptions.KustoException>()
.WaitAndRetryAsync(
Backoff.DecorrelatedJitterBackoffV2(TimeSpan.FromSeconds(10), RetryHelper.MaxRetries),
RetryHelper.GetOnRetryDelegate(RetryHelper.MaxRetries, _loggerService));
IKustoIngestionResult result = await retryPolicy.ExecuteAsync(
() => IngestFromStreamAsync(csv, client, properties, sourceOptions));
IngestionStatus ingestionStatus = result.GetIngestionStatusBySourceId(sourceOptions.SourceId);
for (int i = 0; i < 10 && ingestionStatus.Status == Status.Pending; i++)
{
await Task.Delay(TimeSpan.FromSeconds(30));
ingestionStatus = result.GetIngestionStatusBySourceId(sourceOptions.SourceId);
}
if (ingestionStatus.Status == Status.Pending)
{
throw new InvalidOperationException($"Timeout while ingesting Kusto data.");
}
else if (ingestionStatus.Status != Status.Succeeded)
{
throw new InvalidOperationException(
$"Failed to ingest Kusto data.{Environment.NewLine}{ingestionStatus.Details}");
}
}
}
}
/// <summary>
/// Ingest data into Kusto.
/// </summary>
/// <param name="table">Name of table to ingest into.</param>
/// <param name="mappingName">Name of table mapping to ingest with.</param>
/// <param name="stream">input JSON data stream.</param>
/// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
private static async Task <IKustoIngestionResult> KustoIngest(KustoConnectionStringBuilder kusto, string db, string table, string mappingName, Stream stream)
{
// Create a disposable client that will execute the ingestion
using IKustoIngestClient client = KustoIngestFactory.CreateDirectIngestClient(kusto);
var ingestProps = new KustoIngestionProperties(db, table)
{
JSONMappingReference = mappingName,
Format = DataSourceFormat.json,
};
var ssOptions = new StreamSourceOptions
{
CompressionType = DataSourceCompressionType.GZip,
};
return(await client.IngestFromStreamAsync(stream, ingestProps, ssOptions));
}
public BlockingKustoUploader(
string blobConnectionString,
string blobContainerName,
KustoConnectionStringBuilder adminCsb,
KustoConnectionStringBuilder kscb,
bool demoMode,
string tableName,
int batchSize,
TimeSpan flushDuration,
bool resetTable = false)
{
Csb = kscb;
AdminCsb = adminCsb;
TableName = tableName;
BatchSize = batchSize;
_flushDuration = flushDuration;
_lastUploadTime = DateTime.UtcNow;
_resetTable = resetTable;
_initializeTable = false;
Completed = new AutoResetEvent(false);
if (Csb != null)
{
_ingestionProperties = new KustoIngestionProperties(Csb.InitialCatalog, tableName);
if (demoMode)
{
_ingestClient = KustoIngestFactory.CreateDirectIngestClient(this.AdminCsb);
}
else
{
_ingestClient = KustoIngestFactory.CreateQueuedIngestClient(Csb);
}
}
if (!string.IsNullOrEmpty(blobConnectionString) &&
!string.IsNullOrEmpty(blobContainerName))
{
blobServiceClient = new BlobServiceClient(blobConnectionString);
blobContainerClient = blobServiceClient.GetBlobContainerClient(blobContainerName);
blobContainerClient.CreateIfNotExists();
}
_nextBatch = new List <IDictionary <string, object> >();
}
public async Task PublishManualReviews(IEnumerable <SloManualReview> results)
{
var dt = results.ToDataTable();
var kustoConnectionStringBuilderEngine = new KustoConnectionStringBuilder(kustoUrl_).WithAadUserPromptAuthentication();
using (var client = KustoIngestFactory.CreateDirectIngestClient(kustoConnectionStringBuilderEngine))
{
//Ingest from blobs according to the required properties
var kustoIngestionProperties = new KustoIngestionProperties(
databaseName: kustoDb_,
tableName: kustoManualReviewTable_
);
var reader = dt.CreateDataReader();
var result = await client.IngestFromDataReaderAsync(reader, kustoIngestionProperties);
}
}
public async Task IngestFromCsvStreamAsync(Stream csv, IngestKustoImageInfoOptions options)
{
KustoConnectionStringBuilder connectionBuilder =
new KustoConnectionStringBuilder($"https://{options.Cluster}.kusto.windows.net")
.WithAadApplicationKeyAuthentication(
options.ServicePrincipal.ClientId,
options.ServicePrincipal.Secret,
options.ServicePrincipal.Tenant);
using (IKustoIngestClient client = KustoIngestFactory.CreateDirectIngestClient(connectionBuilder))
{
KustoIngestionProperties properties =
new KustoIngestionProperties(options.Database, options.Table)
{
Format = DataSourceFormat.csv
};
StreamSourceOptions sourceOptions = new StreamSourceOptions {
SourceId = Guid.NewGuid()
};
if (!options.IsDryRun)
{
IKustoIngestionResult result = await client.IngestFromStreamAsync(csv, properties, sourceOptions);
IngestionStatus ingestionStatus = result.GetIngestionStatusBySourceId(sourceOptions.SourceId);
for (int i = 0; i < 10 && ingestionStatus.Status == Status.Pending; i++)
{
await Task.Delay(TimeSpan.FromSeconds(30));
ingestionStatus = result.GetIngestionStatusBySourceId(sourceOptions.SourceId);
}
if (ingestionStatus.Status != Status.Succeeded)
{
throw new InvalidOperationException(
$"Failed to ingest Kusto data.{Environment.NewLine}{ingestionStatus.Details}");
}
else if (ingestionStatus.Status == Status.Pending)
{
throw new InvalidOperationException($"Timeout while ingesting Kusto data.");
}
}
}
}
public BlockingKustoUploader(
string outputFileName,
KustoConnectionStringBuilder adminCsb,
KustoConnectionStringBuilder kscb,
bool demoMode,
string tableName,
int batchSize,
TimeSpan flushDuration,
bool resetTable = false)
{
OutputFileName = outputFileName;
Csb = kscb;
AdminCsb = adminCsb;
TableName = tableName;
BatchSize = batchSize;
_flushDuration = flushDuration;
_lastUploadTime = DateTime.UtcNow;
_resetTable = resetTable;
_initializeTable = false;
Completed = new AutoResetEvent(false);
if (Csb != null)
{
_ingestionProperties = new KustoIngestionProperties(Csb.InitialCatalog, tableName);
if (demoMode)
{
_ingestClient = KustoIngestFactory.CreateDirectIngestClient(this.AdminCsb);
}
else
{
_ingestClient = KustoIngestFactory.CreateQueuedIngestClient(Csb);
}
}
if (!string.IsNullOrEmpty(OutputFileName))
{
outputFile = new StreamWriter(this.OutputFileName);
outputFile.Write($"[{Environment.NewLine}");
}
_nextBatch = new List <IDictionary <string, object> >();
}
static void Main(string[] args)
{
// Ingest From Local File using KustoDirectIngestClient (only for test purposes):
// Create Kusto connection string with App Authentication
var kustoConnectionStringBuilderEngine =
new KustoConnectionStringBuilder(@"https://{clusterNameAndRegion}.kusto.windows.net").WithAadApplicationKeyAuthentication(
applicationClientId: "{Application Client ID}",
applicationKey: "{Application Key (secret)}",
authority: "{AAD TenantID or name}");
// Create a disposable client that will execute the ingestion
using (IKustoIngestClient client = KustoIngestFactory.CreateDirectIngestClient(kustoConnectionStringBuilderEngine))
{
//Ingest from blobs according to the required properties
var kustoIngestionProperties = new KustoIngestionProperties(databaseName: "myDB", tableName: "myTable");
client.IngestFromStorageAsync(@"< Path to local file >", ingestionProperties: kustoIngestionProperties).GetAwaiter().GetResult();
}
}
public async Task <bool> UpdateStormEvent(string update)
{
try
{
var kcsb = new KustoConnectionStringBuilder(_options.ADXCluster, _options.ADXDatabase)
.WithAadUserPromptAuthentication();
using (var queryProvider = KustoIngestFactory.CreateDirectIngestClient(kcsb))
{
// Ingest from a file according to the required properties
var kustoIngestionProperties = new KustoQueuedIngestionProperties(databaseName: _options.ADXDatabase, tableName: _options.ADXTable)
{
// Setting the report level to FailuresAndSuccesses will cause both successful and failed ingestions to be reported
// (Rather than the default "FailuresOnly" level - which is demonstrated in the
// 'Ingest From Local File(s) using KustoQueuedIngestClient and Ingestion Validation' section)
ReportLevel = IngestionReportLevel.FailuresAndSuccesses,
// Choose the report method of choice. 'Queue' is the default method.
// For the sake of the example, we will choose it anyway.
ReportMethod = IngestionReportMethod.Queue,
Format = DataSourceFormat.json
};
StreamDescription sd = new StreamDescription
{
SourceId = Guid.NewGuid(),
Stream = GenericHelper.GenerateStreamFromString(update)
};
await queryProvider.IngestFromStreamAsync(sd, kustoIngestionProperties);
}
return(true);
}
catch
{
return(false);
}
}
public KustoClientFactory(IServiceProvider serviceProvider, KustoSettings kustoSettings = null)
{
var configuration = serviceProvider.GetRequiredService <IConfiguration>();
var aadSettings = configuration.GetConfiguredSettings <AadSettings>();
kustoSettings = kustoSettings ?? configuration.GetConfiguredSettings <KustoSettings>();
var vaultSettings = configuration.GetConfiguredSettings <VaultSettings>();
var kvClient = serviceProvider.GetRequiredService <IKeyVaultClient>();
Func <string, string> getSecretFromVault =
secretName => kvClient.GetSecretAsync(vaultSettings.VaultUrl, secretName).GetAwaiter().GetResult().Value;;
Func <string, X509Certificate2> getCertFromVault =
secretName => kvClient.GetX509CertificateAsync(vaultSettings.VaultUrl, secretName).GetAwaiter().GetResult();
var authBuilder = new AadTokenProvider(aadSettings);
var clientSecretCert = authBuilder.GetClientSecretOrCert(getSecretFromVault, getCertFromVault);
KustoConnectionStringBuilder kcsb;
if (clientSecretCert.secret != null)
{
kcsb = new KustoConnectionStringBuilder($"{kustoSettings.ClusterUrl}")
.WithAadApplicationKeyAuthentication(
aadSettings.ClientId,
clientSecretCert.secret,
aadSettings.Authority);
}
else
{
kcsb = new KustoConnectionStringBuilder($"{kustoSettings.ClusterUrl}")
.WithAadApplicationCertificateAuthentication(
aadSettings.ClientId,
clientSecretCert.cert,
aadSettings.Authority);
}
QueryQueryClient = global::Kusto.Data.Net.Client.KustoClientFactory.CreateCslQueryProvider(kcsb);
AdminClient = global::Kusto.Data.Net.Client.KustoClientFactory.CreateCslAdminProvider(kcsb);
IngestClient = KustoIngestFactory.CreateDirectIngestClient(kcsb);
}
static void IngestData(KustoConnectionStringBuilder engineKustoConnectionStringBuilder, KustoConnectionStringBuilder dmKustoConnectionStringBuilderDM,
string databaseName, string tableName)
{
// 1. Ingest by connecting directly to the Kustolab cluster and sending a command
using (IKustoIngestClient directClient = KustoIngestFactory.CreateDirectIngestClient(engineKustoConnectionStringBuilder))
{
var kustoIngestionProperties = new KustoIngestionProperties(databaseName, tableName);
directClient.IngestFromDataReaderAsync(GetDataAsIDataReader(), kustoIngestionProperties);
}
// 2. Ingest by submitting the data to the Kustolab ingestion cluster
// Note that this is an async operation, so data might not appear immediately
using (IKustoIngestClient queuedClient = KustoIngestFactory.CreateQueuedIngestClient(dmKustoConnectionStringBuilderDM))
{
var kustoIngestionProperties = new KustoIngestionProperties(databaseName, tableName);
queuedClient.IngestFromDataReaderAsync(GetDataAsIDataReader(), kustoIngestionProperties);
}
// 3. Ingest by submitting the data to the Kustolab ingestion cluster -
// This time, update the report method and level so you can track the status of your ingestion
// using the IKustoIngestionResult returned from the ingest operation.
IKustoIngestionResult ingestionResult;
using (IKustoIngestClient queuedClient = KustoIngestFactory.CreateQueuedIngestClient(dmKustoConnectionStringBuilderDM))
{
var kustoIngestionProperties = new KustoQueuedIngestionProperties(databaseName, tableName)
{
// The default ReportLevel is set to FailuresOnly.
// In this case we want to check the status of successful ingestions as well.
ReportLevel = IngestionReportLevel.FailuresAndSuccesses,
// You can use either a queue or a table to track the status of your ingestion.
ReportMethod = IngestionReportMethod.Table
};
ingestionResult = queuedClient.IngestFromDataReaderAsync(GetDataAsIDataReader(), kustoIngestionProperties).Result;
}
// Obtain the status of our ingestion
var ingestionStatus = ingestionResult.GetIngestionStatusCollection().First();
var watch = System.Diagnostics.Stopwatch.StartNew();
var shouldContinue = true;
while ((ingestionStatus.Status == Status.Pending) && (shouldContinue))
{
// Wait a minute...
Thread.Sleep(TimeSpan.FromMinutes(1));
// Try again
ingestionStatus = ingestionResult.GetIngestionStatusBySourceId(ingestionStatus.IngestionSourceId);
shouldContinue = watch.ElapsedMilliseconds < TimeOutInMilliSeconds;
}
watch.Stop();
if (ingestionStatus.Status == Status.Pending)
{
// The status of the ingestion did not change.
Console.WriteLine(
"Ingestion with ID:'{0}' did not complete. Timed out after :'{1}' Milliseconds"
.FormatWithInvariantCulture(
ingestionStatus.IngestionSourceId, watch.ElapsedMilliseconds));
}
else
{
// The status of the ingestion has changed
Console.WriteLine(
"Ingestion with ID:'{0}' is complete. Ingestion Status:'{1}'".FormatWithInvariantCulture(
ingestionStatus.IngestionSourceId, ingestionStatus.Status));
}
// 4. Show the contents of the table
using (var client = Kusto.Data.Net.Client.KustoClientFactory.CreateCslQueryProvider(engineKustoConnectionStringBuilder))
{
while (true)
{
var query = string.Format("{0}", tableName);
var reader = client.ExecuteQuery(query);
Kusto.Cloud.Platform.Data.ExtendedDataReader.WriteAsText(reader, "Data ingested into the table:", tabify: true, firstOnly: true);
Console.WriteLine("Press 'r' to retry retrieving data from the table, any other key to quit");
var key = Console.ReadKey();
if (key.KeyChar != 'r' && key.KeyChar != 'R')
{
break;
}
Console.WriteLine();
}
}
}
public static async Task Run(int numMessagesToSend = 100000, bool direct = true)
{
var tenantId = "1a59e398-83d8-4052-aec9-74a7d6461c5e";
//var user = "******";
//var password = "******";
var clientid = "24c6b350-9b47-4e49-b794-7861cf0ce116";
var clientsecret = "=BYPyhmHf0efu7[-zuoOLGtsT=xcO6v3";
var database = "trillsample";
var rand = new Random((int)DateTime.Now.Ticks);
var m1 = new List <MeasureT1>();
var m3 = new List <MeasureT3>();
var m4 = new List <MeasureT4>();
var j1 = new StringBuilder();
var j3 = new StringBuilder();
var j4 = new StringBuilder();
if (direct)
{
var ingestUri = "Data Source=https://trillsample.westeurope.kusto.windows.net;Initial Catalog=trillsample";
var ingestConnectionStringBuilder = new KustoConnectionStringBuilder(ingestUri)
{
FederatedSecurity = true,
InitialCatalog = database,
#if false
UserID = user,
Password = password,
#else
ApplicationClientId = clientid,
ApplicationKey = clientsecret,
#endif
Authority = tenantId
};
using (var ingestClient = KustoIngestFactory.CreateDirectIngestClient(ingestConnectionStringBuilder))
{
for (var i = 1; i < numMessagesToSend; i++)
{
var uri = (1002030 + i).ToString();
var min = 200 + ((float)rand.Next(100, 500) / 10.0f);
var now = DateTime.Now;
var value = 23f + (float)rand.Next(10) / 10f;
j1.Append(JsonConvert.SerializeObject(Program.CreateMeasureT1(rand, i, uri, min, now, value))).Append("\n");
j3.Append(JsonConvert.SerializeObject(Program.CreateMeasureT3(i, uri, now, value))).Append("\n");
j4.Append(JsonConvert.SerializeObject(Program.CreateMeasureT4(rand, uri, now))).Append("\n");
if (i % 5000 == 0)
{
Console.Title = $"{i}/{numMessagesToSend}";
m1 = new List <MeasureT1>();
m3 = new List <MeasureT3>();
m4 = new List <MeasureT4>();
using (var s1 = new MemoryStream(Encoding.ASCII.GetBytes(j1.ToString())))
using (var s3 = new MemoryStream(Encoding.ASCII.GetBytes(j1.ToString())))
using (var s4 = new MemoryStream(Encoding.ASCII.GetBytes(j1.ToString())))
{
Console.Write("+");
var props = new KustoQueuedIngestionProperties(database, "MeasuresT1")
{
Format = DataSourceFormat.json, JSONMappingReference = "MapMeasureT1"
};
var result = Task.Run(() => ingestClient.IngestFromStream(s1, props, leaveOpen: true));
props = new KustoQueuedIngestionProperties(database, "MeasuresT3")
{
Format = DataSourceFormat.json, JSONMappingReference = "MapMeasureT3"
};
result = ingestClient.IngestFromStreamAsync(s3, props, leaveOpen: true);
props = new KustoQueuedIngestionProperties(database, "MeasuresT4")
{
Format = DataSourceFormat.json, JSONMappingReference = "MapMeasureT4"
};
result = ingestClient.IngestFromStreamAsync(s4, props, leaveOpen: true);
Console.Write("-");
}
}
}
}
}
else
{
var storageAccount = CloudStorageAccount.Parse(StorageConnectionString);
var blobClient = storageAccount.CreateCloudBlobClient();
var blobContainer = blobClient.GetContainerReference("uploads");
blobContainer.CreateIfNotExistsAsync().Wait();
var ingestUri = "https://ingest-trillsample.westeurope.kusto.windows.net/";
var kustoConnectionStringBuilder = new KustoConnectionStringBuilder(ingestUri)
{
FederatedSecurity = true,
InitialCatalog = database,
#if false
UserID = user,
Password = password,
#else
ApplicationClientId = clientid,
ApplicationKey = clientsecret,
#endif
Authority = tenantId
};
for (var i = 1; i < numMessagesToSend; i++)
{
var uri = (1002030 + 10000000 + i).ToString();
var min = 200 + ((float)rand.Next(100, 500) / 10.0f);
var now = DateTime.Now;
var value = 23f + (float)rand.Next(10) / 10f;
j1.Append(JsonConvert.SerializeObject(Program.CreateMeasureT1(rand, i, uri, min, now, value))).Append("\n");
j3.Append(JsonConvert.SerializeObject(Program.CreateMeasureT3(i, uri, now, value))).Append("\n");
j4.Append(JsonConvert.SerializeObject(Program.CreateMeasureT4(rand, uri, now))).Append("\n");
if (i % 5000 == 0)
{
Console.Title = $"{i}/{numMessagesToSend}";
var blockBlob1 = blobContainer.GetBlockBlobReference($"MeasuresT1-{i}");
using (var stream = new StreamWriter(await blockBlob1.OpenWriteAsync()))
await stream.WriteAsync(j1);
var blockBlob3 = blobContainer.GetBlockBlobReference($"MeasuresT3-{i}");
using (var stream = new StreamWriter(await blockBlob3.OpenWriteAsync()))
await stream.WriteAsync(j3);
var blockBlob4 = blobContainer.GetBlockBlobReference($"MeasuresT4-{i}");
using (var stream = new StreamWriter(await blockBlob4.OpenWriteAsync()))
await stream.WriteAsync(j4);
var adHocPolicy = new SharedAccessBlobPolicy()
{
SharedAccessExpiryTime = DateTime.UtcNow.AddHours(24),
Permissions = SharedAccessBlobPermissions.Read | SharedAccessBlobPermissions.Delete
};
Console.WriteLine("+");
using (var ingestClient = KustoIngestFactory.CreateQueuedIngestClient(kustoConnectionStringBuilder))
{
var props = new KustoQueuedIngestionProperties(database, "MeasuresT1")
{
Format = DataSourceFormat.json, JSONMappingReference = "MapMeasureT1"
};
await ingestClient.IngestFromStorageAsync($"{blockBlob1.Uri.AbsoluteUri}{blockBlob1.GetSharedAccessSignature(adHocPolicy).ToString()}", props, new StorageSourceOptions { DeleteSourceOnSuccess = true });
props = new KustoQueuedIngestionProperties(database, "MeasuresT3")
{
Format = DataSourceFormat.json, JSONMappingReference = "MapMeasureT3"
};
await ingestClient.IngestFromStorageAsync($"{blockBlob3.Uri.AbsoluteUri}{blockBlob3.GetSharedAccessSignature(adHocPolicy).ToString()}", props, new StorageSourceOptions { DeleteSourceOnSuccess = true });
props = new KustoQueuedIngestionProperties(database, "MeasuresT4")
{
Format = DataSourceFormat.json, JSONMappingReference = "MapMeasureT4"
};
await ingestClient.IngestFromStorageAsync($"{blockBlob4.Uri.AbsoluteUri}{blockBlob4.GetSharedAccessSignature(adHocPolicy).ToString()}", props, new StorageSourceOptions { DeleteSourceOnSuccess = true });
}
Console.WriteLine("-");
}
}
}
}
public ActionResult Upload(string instanceUrl, string databaseName, string tenantId, string clientId, string secret)
{
ADXModel model = new ADXModel
{
InstanceUrl = instanceUrl,
DatabaseName = databaseName,
TenantId = tenantId,
ClientId = clientId,
Secret = secret,
};
try
{
var kustoConnectionStringBuilder = new KustoConnectionStringBuilder(instanceUrl).WithAadApplicationKeyAuthentication(clientId, secret, tenantId);
string tableName = DTDL._nodesetNamespaceURI.Replace("http://", "").Replace('/', '_').Replace('.', '_').TrimEnd('_');
using (var kustoClient = KustoClientFactory.CreateCslAdminProvider(kustoConnectionStringBuilder))
{
kustoClient.ExecuteControlCommand(databaseName, ".create table " + tableName + " (ExpandedNodeID: string, DisplayName: string, Type: string, ParentNodeID: string)");
string command = ".create table " + tableName + " ingestion json mapping 'Nodeset_Mapping' '["
+ "{ \"properties\": { \"path\": \"$.Key\" }, \"column\": \"ExpandedNodeID\", \"datatype\": \"string\" },"
+ "{ \"properties\": { \"path\": \"$.Value.Item1\" }, \"column\": \"DisplayName\", \"datatype\": \"string\" } ,"
+ "{ \"properties\": { \"path\": \"$.Value.Item2\" }, \"column\": \"Type\", \"datatype\": \"string\" },"
+ "{ \"properties\": { \"path\": \"$.Value.Item3\" }, \"column\": \"ParentNodeID\", \"datatype\": \"string\" } ]'";
kustoClient.ExecuteControlCommand(databaseName, command);
command = ".alter table " + tableName + " policy ingestionbatching @'{"
+ "\"MaximumBatchingTimeSpan\": \"00:00:05\","
+ "\"MaximumNumberOfItems\": 100,"
+ "\"MaximumRawDataSizeMB\": 1024 }'";
kustoClient.ExecuteControlCommand(databaseName, command);
}
using (IKustoIngestClient client = KustoIngestFactory.CreateDirectIngestClient(kustoConnectionStringBuilder))
{
var kustoIngestionProperties = new KustoIngestionProperties(databaseName, tableName);
kustoIngestionProperties.Format = DataSourceFormat.multijson;
kustoIngestionProperties.IngestionMapping = new IngestionMapping()
{
IngestionMappingReference = "Nodeset_Mapping",
IngestionMappingKind = IngestionMappingKind.Json
};
foreach (KeyValuePair <string, Tuple <string, string, string> > entry in DTDL._nodeList.ToArray(100))
{
string content = JsonConvert.SerializeObject(entry, Formatting.Indented);
MemoryStream stream = new MemoryStream(Encoding.UTF8.GetBytes(content));
client.IngestFromStream(stream, kustoIngestionProperties);
stream.Dispose();
}
}
using (var cslQueryProvider = KustoClientFactory.CreateCslQueryProvider(kustoConnectionStringBuilder))
{
string query = $"{tableName} | count";
var results = cslQueryProvider.ExecuteQuery <long>(databaseName, query);
foreach (long result in results)
{
model.StatusMessage = "ADX Ingestion succeeded.";
return(View("Index", model));
}
}
model.StatusMessage = "ADX Ingestion failed!";
return(View("Error", model));
}
catch (Exception ex)
{
model.StatusMessage = ex.Message;
return(View("Index", model));
}
}