public KustoClientLogger(string databaseName, string tableName, List <KustoColumnMapping> _schema, string clientId, string clientKey)
{
var csvColumnMappings = new List <CsvColumnMapping>();
foreach (var mapping in _schema)
{
csvColumnMappings.Add(new CsvColumnMapping {
ColumnName = mapping.ColumnName, CslDataType = mapping.DataType, Ordinal = mapping.ColumnNumber
});
}
_kustoProperties = new KustoIngestionProperties(databaseName, tableName)
{
Format = DataSourceFormat.csv,
CSVMapping = csvColumnMappings
};
var kustoConnectionStringBuilderDM = new KustoConnectionStringBuilder(@"https://wawstest.kusto.windows.net:443")
{
FederatedSecurity = true,
InitialCatalog = databaseName,
ApplicationKey = clientKey,
ApplicationClientId = clientId
};
_kustoClient = KustoIngestFactory.CreateDirectIngestClient(kustoConnectionStringBuilderDM);
}
/// <summary>
/// Constructor. Initializes this object and does nothing else.
/// </summary>
/// <param name="connectionString">Kusto connection string.</param>
/// <param name="database">Database into which to ingest.</param>
/// <param name="table">Table into which to ingest.</param>
/// <param name="csvMapping">Csv mapping that applies to all CSV files to be uploaded via <see cref="UploadSingleCsvFile"/></param>
/// <param name="deleteFilesOnSuccess">Whether to delete files upon successful upload.</param>
/// <param name="checkForIngestionErrors">
/// Whether to check for ingestion errors before disposing this object.
/// Note that at this time not all uploaded files have necessarily been ingested; this class
/// does not wait for ingestions to complete, it only checks for failures of those that have completed.
/// </param>
/// <param name="log">Optional log to which to write some debug information.</param>
public KustoUploader
(
string connectionString,
string database,
string table,
IEnumerable <CsvColumnMapping> csvMapping,
bool deleteFilesOnSuccess,
bool checkForIngestionErrors,
ILog log = null
)
{
_log = log;
_deleteFilesOnSuccess = deleteFilesOnSuccess;
_checkForIngestionErrors = checkForIngestionErrors;
_client = KustoIngestFactory.CreateQueuedIngestClient(connectionString);
_hasUploadErrors = false;
_ingestionProperties = new KustoQueuedIngestionProperties(database, table)
{
CSVMapping = csvMapping,
ReportLevel = IngestionReportLevel.FailuresOnly,
ReportMethod = IngestionReportMethod.Queue,
};
_block = new ActionBlock <FileDescription>
(
UploadSingleCsvFile,
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = 1,
BoundedCapacity = DataflowBlockOptions.Unbounded
}
);
}
static async Task Main(string[] args)
{
// Async Ingestion From a Single Azure Blob using KustoQueuedIngestClient with (optional) RetryPolicy:
//Create Kusto connection string with App Authentication
var kustoConnectionStringBuilderDM =
new KustoConnectionStringBuilder(@"https://ingest-{clusterNameAndRegion}.kusto.windows.net").WithAadApplicationKeyAuthentication(
applicationClientId: "{Application Client ID}",
applicationKey: "{Application Key (secret)}",
authority: "{AAD TenantID or name}");
// Create an ingest client
// Note, that creating a separate instance per ingestion operation is an anti-pattern.
// IngestClient classes are thread-safe and intended for reuse
IKustoIngestClient client = KustoIngestFactory.CreateQueuedIngestClient(kustoConnectionStringBuilderDM);
// Ingest from blobs according to the required properties
var kustoIngestionProperties = new KustoIngestionProperties(databaseName: "myDB", tableName: "myTable");
var sourceOptions = new StorageSourceOptions()
{
DeleteSourceOnSuccess = true
};
//// Create your custom implementation of IRetryPolicy, which will affect how the ingest client handles retrying on transient failures
IRetryPolicy retryPolicy = new NoRetry();
//// This line sets the retry policy on the ingest client that will be enforced on every ingest call from here on
((IKustoQueuedIngestClient)client).QueueRetryPolicy = retryPolicy;
await client.IngestFromStorageAsync(uri : @"BLOB-URI-WITH-SAS-KEY", ingestionProperties : kustoIngestionProperties, sourceOptions);
client.Dispose();
}
public BlockingKustoUploader(
KustoConnectionStringBuilder kscb,
string tableName,
int batchSize,
TimeSpan flushDuration)
{
csb = kscb;
TableName = TableName;
BatchSize = batchSize;
_flushDuration = flushDuration;
_lastUploadTime = DateTime.Now;
Completed = new AutoResetEvent(false);
_ingestionProperties = new KustoIngestionProperties(csb.InitialCatalog, tableName);
_fields = typeof(EtwEvent).GetFields().Select(f => f.Name).ToArray();
if (csb.DataSource.StartsWith("https://ingest-"))
{
_ingestClient = KustoIngestFactory.CreateQueuedIngestClient(csb);
}
else
{
_ingestClient = KustoIngestFactory.CreateDirectIngestClient(csb);
}
_nextBatch = new List <T>();
}
public Monitor(Configuration configuration)
{
Contract.RequiresNotNull(configuration);
_configuration = configuration;
_logger = CreateLogger();
Contract.RequiresNotNullOrEmpty(_configuration.KustoIngestionClusterUrl);
Contract.RequiresNotNullOrEmpty(_configuration.ApplicationClientId);
Contract.RequiresNotNullOrEmpty(_configuration.ApplicationKey);
Contract.RequiresNotNullOrEmpty(_configuration.Authority);
var kustoIngestConnectionString = new KustoConnectionStringBuilder(_configuration.KustoIngestionClusterUrl)
.WithAadApplicationKeyAuthentication(_configuration.ApplicationClientId, _configuration.ApplicationKey, _configuration.Authority);
_kustoIngestClient = KustoIngestFactory.CreateDirectIngestClient(kustoIngestConnectionString);
var kustoConnectionString = new KustoConnectionStringBuilder(_configuration.KustoClusterUrl)
.WithAadApplicationKeyAuthentication(_configuration.ApplicationClientId, _configuration.ApplicationKey, _configuration.Authority);
_cslQueryProvider = KustoClientFactory.CreateCslQueryProvider(kustoConnectionString);
Contract.RequiresNotNull(_configuration.KustoNotifier);
_alertNotifier = new KustoWriter <Notification>(_configuration.KustoNotifier, _logger, _kustoIngestClient);
Contract.RequiresNotNull(_configuration.SchedulerKustoNotifier);
_schedulerLogWriter = new KustoWriter <Scheduler.LogEntry>(_configuration.SchedulerKustoNotifier, _logger, _kustoIngestClient);
_scheduler = new Scheduler(_configuration.Scheduler, _logger, _clock, _schedulerLogWriter);
}
/// <inheritdoc />
public IBigBrother UseKusto(string kustoEngineName, string kustoEngineLocation, string kustoDb, string tenantId)
{
KustoDbName = kustoDb;
var kustoUri = $"https://{kustoEngineName}.{kustoEngineLocation}.kusto.windows.net";
var kustoIngestUri = $"https://ingest-{kustoEngineName}.{kustoEngineLocation}.kusto.windows.net";
var token = new AzureServiceTokenProvider().GetAccessTokenAsync(kustoUri, string.Empty).Result;
KustoAdminClient = KustoClientFactory.CreateCslAdminProvider(
new KustoConnectionStringBuilder(kustoUri)
{
FederatedSecurity = true,
InitialCatalog = KustoDbName,
AuthorityId = tenantId,
ApplicationToken = token
});
KustoIngestClient = KustoIngestFactory.CreateQueuedIngestClient(
new KustoConnectionStringBuilder(kustoIngestUri)
{
FederatedSecurity = true,
InitialCatalog = KustoDbName,
AuthorityId = tenantId,
ApplicationToken = token
});
SetupKustoSubscription();
return(this);
}
static void Main(string[] args)
{
// Ingest From Local Files using KustoQueuedIngestClient and Ingestion Validation
// Create Kusto connection string with App Authentication
var kustoConnectionStringBuilderDM =
new KustoConnectionStringBuilder(@"https://ingest-{clusterNameAndRegion}.kusto.windows.net").WithAadApplicationKeyAuthentication(
applicationClientId: "{Application Client ID}",
applicationKey: "{Application Key (secret)}",
authority: "{AAD TenantID or name}");
// Create a disposable client that will execute the ingestion
IKustoQueuedIngestClient client = KustoIngestFactory.CreateQueuedIngestClient(kustoConnectionStringBuilderDM);
// Ingest from files according to the required properties
var kustoIngestionProperties = new KustoIngestionProperties(databaseName: "myDB", tableName: "myTable");
client.IngestFromStorageAsync(@"ValidTestFile.csv", kustoIngestionProperties);
client.IngestFromStorageAsync(@"InvalidTestFile.csv", kustoIngestionProperties);
// Waiting for the aggregation
Thread.Sleep(TimeSpan.FromMinutes(8));
// Retrieve and validate failures
var ingestionFailures = client.PeekTopIngestionFailures().GetAwaiter().GetResult();
Ensure.IsTrue((ingestionFailures.Count() > 0), "Failures expected");
// Retrieve, delete and validate failures
ingestionFailures = client.GetAndDiscardTopIngestionFailures().GetAwaiter().GetResult();
Ensure.IsTrue((ingestionFailures.Count() > 0), "Failures expected");
// Dispose of the client
client.Dispose();
}
public Monitor(Configuration configuration)
{
Contract.RequiresNotNull(configuration);
_configuration = configuration;
_logger = CreateLogger();
// TODO(jubayard): use streaming ingestion instead of direct ingestion. There seems to be some assembly
// issues when attempting to do that
Contract.RequiresNotNullOrEmpty(_configuration.KustoIngestionClusterUrl);
Contract.RequiresNotNullOrEmpty(_configuration.ApplicationClientId);
Contract.RequiresNotNullOrEmpty(_configuration.ApplicationKey);
Contract.RequiresNotNullOrEmpty(_configuration.Authority);
var kustoIngestConnectionString = new KustoConnectionStringBuilder(_configuration.KustoIngestionClusterUrl)
.WithAadApplicationKeyAuthentication(_configuration.ApplicationClientId, _configuration.ApplicationKey, _configuration.Authority);
_kustoIngestClient = KustoIngestFactory.CreateDirectIngestClient(kustoIngestConnectionString);
var kustoConnectionString = new KustoConnectionStringBuilder(_configuration.KustoClusterUrl)
.WithAadApplicationKeyAuthentication(_configuration.ApplicationClientId, _configuration.ApplicationKey, _configuration.Authority);
_cslQueryProvider = KustoClientFactory.CreateCslQueryProvider(kustoConnectionString);
Contract.RequiresNotNull(_configuration.KustoNotifier);
_alertNotifier = new KustoWriter <Notification>(_configuration.KustoNotifier, _logger, _kustoIngestClient);
Contract.RequiresNotNull(_configuration.SchedulerKustoNotifier);
_schedulerLogWriter = new KustoWriter <Scheduler.LogEntry>(_configuration.SchedulerKustoNotifier, _logger, _kustoIngestClient);
_scheduler = new Scheduler(_configuration.Scheduler, _logger, _clock, _schedulerLogWriter);
}
public static IServiceCollection AddKustoIngest(this IServiceCollection services, Action <IOptions <KustoOptions> > configure)
{
services.Configure(configure);
services.AddSingleton <IKustoIngestClient>(provider =>
KustoIngestFactory.CreateQueuedIngestClient(
provider.GetRequiredService <IOptions <KustoOptions> >().Value.IngestConnectionString));
return(services);
}
public AdxOutput(
BaseLogger logger,
string authority,
string appclientId,
string appKey,
string cluster,
string database,
string table,
bool createOrResetTable = false,
bool directIngest = false)
{
_logger = logger;
_table = table;
_createOrResetTable = createOrResetTable;
_batchSize = 10000;
_flushDuration = TimeSpan.FromMilliseconds(5);
_lastUploadTime = DateTime.UtcNow;
_initializeTable = false;
_nextBatch = new List <IDictionary <string, object> >();
Completed = new AutoResetEvent(false);
// Setting up kusto connection
if (!string.IsNullOrEmpty(authority))
{
if (!string.IsNullOrEmpty(appclientId) && !string.IsNullOrEmpty(appKey))
{
kscbIngest = new KustoConnectionStringBuilder($"https://ingest-{cluster}", database).WithAadApplicationKeyAuthentication(appclientId, appKey, authority);
kscbAdmin = new KustoConnectionStringBuilder($"https://{cluster}", database).WithAadApplicationKeyAuthentication(appclientId, appKey, authority);
}
else
{
kscbIngest = new KustoConnectionStringBuilder($"https://ingest-{cluster}", database).WithAadUserPromptAuthentication(authority);
kscbAdmin = new KustoConnectionStringBuilder($"https://{cluster}", database).WithAadUserPromptAuthentication(authority);
}
}
if (kscbAdmin != null)
{
_ingestionProperties = new KustoIngestionProperties(kscbIngest.InitialCatalog, table);
if (directIngest)
{
_ingestClient = KustoIngestFactory.CreateDirectIngestClient(this.kscbAdmin);
}
else
{
_ingestClient = KustoIngestFactory.CreateQueuedIngestClient(kscbIngest);
}
}
else
{
_logger.Log(LogLevel.ERROR, "ERROR getting ADX connection strings. Please double check the information provided.");
_error = true;
}
}
public static void Run([CosmosDBTrigger(
databaseName: "data",
collectionName: "device",
ConnectionStringSetting = "rbdemocosmosdb_DOCUMENTDB",
LeaseCollectionName = "leases", CreateLeaseCollectionIfNotExists = true)] IReadOnlyList <Document> input, ILogger log)
{
if (input != null && input.Count > 0)
{
log.LogInformation("Documents modified " + input.Count);
log.LogInformation("First document Id " + input[0].Id);
//input[0].ToString() is the data that needs to be ingested.
var tenantId = System.Environment.GetEnvironmentVariable($"tenantId");
var applicationClientId = System.Environment.GetEnvironmentVariable($"applicationClientId");
var applicationKey = System.Environment.GetEnvironmentVariable($"applicationKey");
var serviceNameAndRegion = System.Environment.GetEnvironmentVariable($"serviceNameAndRegion");
var kustoDatabase = "device";
var kustoTableName = "Staging";
var mappingName = "deviceMapping";
var kustoIngestConnectionStringBuilder =
new KustoConnectionStringBuilder($"https://ingest-{serviceNameAndRegion}.kusto.windows.net")
{
FederatedSecurity = true,
InitialCatalog = kustoDatabase,
ApplicationClientId = applicationClientId,
ApplicationKey = applicationKey,
Authority = tenantId
};
using (var ingestClient = KustoIngestFactory.CreateQueuedIngestClient(kustoIngestConnectionStringBuilder))
{
var properties = new KustoQueuedIngestionProperties(kustoDatabase, kustoTableName)
{
Format = DataSourceFormat.json,
JSONMappingReference = mappingName,
ReportLevel = IngestionReportLevel.FailuresAndSuccesses,
ReportMethod = IngestionReportMethod.Queue
};
string inputValue = input[0].ToString().Replace("\r\n", "");
//string inputValue = "{ \"Id\":\"1\", \"Timestamp\":\"2\", \"Message\":\"This is a test message\" }";
//object val = Newtonsoft.Json.JsonConvert.DeserializeObject(inputValue);
var stream = new System.IO.MemoryStream(System.Text.Encoding.UTF8.GetBytes(inputValue));
stream.Seek(0, SeekOrigin.Begin);
// Post ingestion message
ingestClient.IngestFromStream(stream, properties, leaveOpen: true);
// Wait and retrieve all notifications
// - Actual duration should be decided based on the effective Ingestion Batching Policy set on the table/database
Thread.Sleep(10000);
var errors = ingestClient.GetAndDiscardTopIngestionFailures().GetAwaiter().GetResult();
var successes = ingestClient.GetAndDiscardTopIngestionSuccesses().GetAwaiter().GetResult();
errors.ForEach((f) => { Console.WriteLine($"Ingestion error: {f.Info.Details}"); });
successes.ForEach((s) => { Console.WriteLine($"Ingested: {s.Info.IngestionSourcePath}"); });
}
}
}
static async Task Main(string[] args)
{
// Ingest From a Local File using KustoQueuedIngestClient and report status to a table
// Create Kusto connection string with App Authentication
var kustoConnectionStringBuilderDM =
new KustoConnectionStringBuilder(@"https://ingest-{clusterNameAndRegion}.kusto.windows.net").WithAadApplicationKeyAuthentication(
applicationClientId: "{Application Client ID}",
applicationKey: "{Application Key (secret)}",
authority: "{AAD TenantID or name}");
// Create a disposable client that will execute the ingestion
IKustoQueuedIngestClient client = KustoIngestFactory.CreateQueuedIngestClient(kustoConnectionStringBuilderDM);
// Ingest from a file according to the required properties
var kustoIngestionProperties = new KustoQueuedIngestionProperties(databaseName: "myDB", tableName: "myDB")
{
// Setting the report level to FailuresAndSuccesses will cause both successful and failed ingestions to be reported
// (Rather than the default "FailuresOnly" level)
ReportLevel = IngestionReportLevel.FailuresAndSuccesses,
// Choose the report method of choice
ReportMethod = IngestionReportMethod.Table
};
var filePath = @"< Path to file >";
var fileIdentifier = Guid.NewGuid();
var fileDescription = new FileDescription()
{
FilePath = filePath, SourceId = fileIdentifier
};
var sourceOptions = new StorageSourceOptions()
{
SourceId = fileDescription.SourceId.Value
};
// Execute the ingest operation and save the result.
var clientResult = await client.IngestFromStorageAsync(fileDescription.FilePath,
ingestionProperties : kustoIngestionProperties, sourceOptions);
// Use the fileIdentifier you supplied to get the status of your ingestion
var ingestionStatus = clientResult.GetIngestionStatusBySourceId(fileIdentifier);
while (ingestionStatus.Status == Status.Pending)
{
// Wait a minute...
Thread.Sleep(TimeSpan.FromMinutes(1));
// Try again
ingestionStatus = clientResult.GetIngestionStatusBySourceId(fileIdentifier);
}
// Verify the results of the ingestion
Ensure.ConditionIsMet(ingestionStatus.Status == Status.Succeeded,
"The file should have been ingested successfully");
// Dispose of the client
client.Dispose();
}
protected FI(ILogger iLog, Microsoft.Azure.Management.Fluent.IAzure iAzure, string iRGName, string kustoConn, string kustoDBName, string kustoTableName)
{
curRGName = iRGName;
log = iLog;
curSubName = iAzure.SubscriptionId;
ingestClient = KustoIngestFactory.CreateQueuedIngestClient(kustoConn);
ingestProps = new KustoIngestionProperties(kustoDBName, kustoTableName);
ingestProps.Format = Kusto.Data.Common.DataSourceFormat.csv;
}
public IKustoIngestClient GetClient()
{
string ingestConnectionString = _kustoOptions.Value.IngestConnectionString;
if (string.IsNullOrWhiteSpace(ingestConnectionString))
{
throw new InvalidCastException($"Kusto {nameof(_kustoOptions.Value.IngestConnectionString)} is not configured in settings or related KeyVault");
}
return(_clients.GetOrAdd(ingestConnectionString, _ => KustoIngestFactory.CreateQueuedIngestClient(ingestConnectionString)));
}
public static Result <IKustoIngestClient> CreateKustoIngestClient(string kustoIngestionClusterUrl, string azureTenantId, string azureAppId, string azureAppKey)
{
Contract.RequiresNotNullOrEmpty(kustoIngestionClusterUrl);
Contract.RequiresNotNullOrEmpty(azureTenantId);
Contract.RequiresNotNullOrEmpty(azureAppId);
Contract.RequiresNotNullOrEmpty(azureAppKey);
var kustoIngestConnectionString = new KustoConnectionStringBuilder(kustoIngestionClusterUrl)
.WithAadApplicationKeyAuthentication(azureAppId, azureAppKey, azureTenantId);
return(new Result <IKustoIngestClient>(KustoIngestFactory.CreateDirectIngestClient(kustoIngestConnectionString)));
}
/// <summary>
/// Performs a write to the indicated table, sending the appropriately filtered test report to Kusto
/// Returns the number of lines written
/// Skips the final write step when testing
/// </summary>
public bool WriteDataForTable(bool testing, List <AdoBuildResultRow> reports)
{
try
{
// Create Kusto connection string with App Authentication
var kustoConnectionStringBuilderEngine =
new KustoConnectionStringBuilder(config.clusterUri).WithAadApplicationKeyAuthentication(
applicationClientId: config.clientId,
applicationKey: config.secretKey,
authority: config.authority);
using (var ingestClient = KustoIngestFactory.CreateDirectIngestClient(kustoConnectionStringBuilderEngine))
{
var ingestProps = new KustoQueuedIngestionProperties(config.dbName, config.tableName);
// For the sake of getting both failure and success notifications we set this to IngestionReportLevel.FailuresAndSuccesses
// Usually the recommended level is IngestionReportLevel.FailuresOnly
ingestProps.ReportLevel = IngestionReportLevel.FailuresAndSuccesses;
ingestProps.ReportMethod = IngestionReportMethod.Queue;
ingestProps.IngestionMapping = new IngestionMapping {
IngestionMappingReference = config.mappingName
};
ingestProps.Format = DataSourceFormat.json;
// Prepare data for ingestion
using (var memStream = new MemoryStream())
using (var writer = new StreamWriter(memStream))
{
foreach (AdoBuildResultRow eachReport in reports)
{
writer.WriteLine(JsonConvert.SerializeObject(eachReport));
Console.WriteLine($"Writing: {eachReport.jobname}, ({eachReport.passed}/{eachReport.failed}/{eachReport.other}/{eachReport.total}) for {eachReport.timestamp}");
}
writer.Flush();
memStream.Seek(0, SeekOrigin.Begin);
// Post ingestion message
if (!testing)
{
ingestClient.IngestFromStream(memStream, ingestProps, leaveOpen: true);
}
}
}
return(true);
}
catch (Exception e)
{
Console.WriteLine($"ERROR: Could not write to: {config.clusterUri}.{config.dbName}.{config.tableName}, using Mapping: {config.mappingName}");
Console.WriteLine(e.ToString());
return(false);
}
}
public async Task InsertRowAsync(T row, CancellationToken cancellationToken = default)
{
using IKustoQueuedIngestClient ingestClient = KustoIngestFactory.CreateQueuedIngestClient(_connectionStringBuilder);
string serializedData = JsonConvert.SerializeObject(row);
byte[] serializedBytes = Encoding.UTF8.GetBytes(serializedData);
using MemoryStream dataStream = new MemoryStream(serializedBytes);
// IKustoQueuedIngestClient doesn't support cancellation at the moment. Update the line below if it does in the future.
await ingestClient.IngestFromStreamAsync(dataStream, _ingestionProperties, leaveOpen : true);
}
private static bool TryInitialize(ILogger log)
{
lock (_lock)
{
if (!_initialized)
{
string kustoIngestUrl = GetEnvVariable("KustoIngestUrl");
string clientId = GetEnvVariable("ClientId");
string clientSecret = GetEnvVariable("ClientSecret");
string tenantId = GetEnvVariable("TenantId");
if (String.IsNullOrWhiteSpace(kustoIngestUrl) ||
String.IsNullOrWhiteSpace(clientId) ||
String.IsNullOrWhiteSpace(clientSecret) ||
String.IsNullOrWhiteSpace(tenantId))
{
log.LogError($"Could not initialize the Kusto client because the connection parameters are wrong (url: {kustoIngestUrl} clientId: {clientId}, tenant: {tenantId}");
return(false);
}
//Initialize adx
connection =
new KustoConnectionStringBuilder(GetEnvVariable("KustoIngestUrl")).WithAadApplicationKeyAuthentication(
applicationClientId: GetEnvVariable("ClientId"),
applicationKey: GetEnvVariable("ClientSecret"),
authority: GetEnvVariable("TenantId"));
adx = KustoIngestFactory.CreateQueuedIngestClient(connection);
if (adx != null)
{
_initialized = true;
log.LogInformation("Function successfully initialized");
return(true);
}
else
{
log.LogWarning("Function not successfully initialized");
}
}
else
{
return(true);
}
}
return(false);
}
internal IBigBrother UseKustoInternal(string kustoEngineName, string kustoEngineLocation, string kustoDb, string tenantId)
{
try
{
KustoDbName = kustoDb;
var kustoUri = $"https://{kustoEngineName}.{kustoEngineLocation}.kusto.windows.net";
var token = new AzureServiceTokenProvider().GetAccessTokenAsync(kustoUri, string.Empty).Result;
KustoAdminClient = KustoClientFactory.CreateCslAdminProvider(
new KustoConnectionStringBuilder(kustoUri)
{
FederatedSecurity = true,
InitialCatalog = KustoDbName,
Authority = tenantId,
ApplicationToken = token
});
var kustoQueuedIngestUri = $"https://ingest-{kustoEngineName}.{kustoEngineLocation}.kusto.windows.net";
var kustoIngestUri = $"https://{kustoEngineName}.{kustoEngineLocation}.kusto.windows.net";
KustoQueuedIngestClient = KustoIngestFactory.CreateQueuedIngestClient(
new KustoConnectionStringBuilder(kustoQueuedIngestUri)
{
FederatedSecurity = true,
InitialCatalog = KustoDbName,
Authority = tenantId,
ApplicationToken = token
});
KustoDirectIngestClient = KustoIngestFactory.CreateDirectIngestClient(
new KustoConnectionStringBuilder(kustoIngestUri)
{
FederatedSecurity = true,
InitialCatalog = KustoDbName,
Authority = tenantId,
ApplicationToken = token
});
SetupKustoSubscription();
KustoIngestionTimeMetric = InternalClient.GetMetric(new MetricIdentifier("", "KustoIngestionTime"));
}
catch (Exception e)
{
InternalStream.OnNext(e.ToExceptionEvent());
}
return(this);
}
static void Main(string[] args)
{
// Ingest From a Local Files using KustoQueuedIngestClient and report status to a queue
// Create Kusto connection string with App Authentication
var kustoConnectionStringBuilderDM =
new KustoConnectionStringBuilder(@"https://ingest-{clusterNameAndRegion}.kusto.windows.net").WithAadApplicationKeyAuthentication(
applicationClientId: "{Application Client ID}",
applicationKey: "{Application Key (secret)}",
authority: "{AAD TenantID or name}");
// Create a disposable client that will execute the ingestion
IKustoQueuedIngestClient client = KustoIngestFactory.CreateQueuedIngestClient(kustoConnectionStringBuilderDM);
// Ingest from a file according to the required properties
var kustoIngestionProperties = new KustoQueuedIngestionProperties(databaseName: "myDB", tableName: "myTable")
{
// Setting the report level to FailuresAndSuccesses will cause both successful and failed ingestions to be reported
// (Rather than the default "FailuresOnly" level - which is demonstrated in the
// 'Ingest From Local File(s) using KustoQueuedIngestClient and Ingestion Validation' section)
ReportLevel = IngestionReportLevel.FailuresAndSuccesses,
// Choose the report method of choice. 'Queue' is the default method.
// For the sake of the example, we will choose it anyway.
ReportMethod = IngestionReportMethod.Queue
};
client.IngestFromStorageAsync("ValidTestFile.csv", kustoIngestionProperties);
client.IngestFromStorageAsync("InvalidTestFile.csv", kustoIngestionProperties);
// Waiting for the aggregation
Thread.Sleep(TimeSpan.FromMinutes(8));
// Retrieve and validate failures
var ingestionFailures = client.PeekTopIngestionFailures().GetAwaiter().GetResult();
Ensure.IsTrue((ingestionFailures.Count() > 0), "The failed ingestion should have been reported to the failed ingestions queue");
// Retrieve, delete and validate failures
ingestionFailures = client.GetAndDiscardTopIngestionFailures().GetAwaiter().GetResult();
Ensure.IsTrue((ingestionFailures.Count() > 0), "The failed ingestion should have been reported to the failed ingestions queue");
// Verify the success has also been reported to the queue
var ingestionSuccesses = client.GetAndDiscardTopIngestionSuccesses().GetAwaiter().GetResult();
Ensure.ConditionIsMet((ingestionSuccesses.Count() > 0),
"The successful ingestion should have been reported to the successful ingestions queue");
// Dispose of the client
client.Dispose();
}
public async Task IngestFromCsvAsync(
string csv, ServicePrincipalOptions servicePrincipal, string cluster, string database, string table, bool isDryRun)
{
KustoConnectionStringBuilder connectionBuilder =
new KustoConnectionStringBuilder($"https://{cluster}.kusto.windows.net")
.WithAadApplicationKeyAuthentication(
servicePrincipal.ClientId,
servicePrincipal.Secret,
servicePrincipal.Tenant);
using (IKustoIngestClient client = KustoIngestFactory.CreateDirectIngestClient(connectionBuilder))
{
KustoIngestionProperties properties =
new(database, table) { Format = DataSourceFormat.csv };
StreamSourceOptions sourceOptions = new() { SourceId = Guid.NewGuid() };
if (!isDryRun)
{
AsyncRetryPolicy retryPolicy = Policy
.Handle <Kusto.Data.Exceptions.KustoException>()
.Or <Kusto.Ingest.Exceptions.KustoException>()
.WaitAndRetryAsync(
Backoff.DecorrelatedJitterBackoffV2(TimeSpan.FromSeconds(10), RetryHelper.MaxRetries),
RetryHelper.GetOnRetryDelegate(RetryHelper.MaxRetries, _loggerService));
IKustoIngestionResult result = await retryPolicy.ExecuteAsync(
() => IngestFromStreamAsync(csv, client, properties, sourceOptions));
IngestionStatus ingestionStatus = result.GetIngestionStatusBySourceId(sourceOptions.SourceId);
for (int i = 0; i < 10 && ingestionStatus.Status == Status.Pending; i++)
{
await Task.Delay(TimeSpan.FromSeconds(30));
ingestionStatus = result.GetIngestionStatusBySourceId(sourceOptions.SourceId);
}
if (ingestionStatus.Status == Status.Pending)
{
throw new InvalidOperationException($"Timeout while ingesting Kusto data.");
}
else if (ingestionStatus.Status != Status.Succeeded)
{
throw new InvalidOperationException(
$"Failed to ingest Kusto data.{Environment.NewLine}{ingestionStatus.Details}");
}
}
}
}
/// <summary>
/// Ingest data into Kusto.
/// </summary>
/// <param name="table">Name of table to ingest into.</param>
/// <param name="mappingName">Name of table mapping to ingest with.</param>
/// <param name="stream">input JSON data stream.</param>
/// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
private static async Task <IKustoIngestionResult> KustoIngest(KustoConnectionStringBuilder kusto, string db, string table, string mappingName, Stream stream)
{
// Create a disposable client that will execute the ingestion
using IKustoIngestClient client = KustoIngestFactory.CreateDirectIngestClient(kusto);
var ingestProps = new KustoIngestionProperties(db, table)
{
JSONMappingReference = mappingName,
Format = DataSourceFormat.json,
};
var ssOptions = new StreamSourceOptions
{
CompressionType = DataSourceCompressionType.GZip,
};
return(await client.IngestFromStreamAsync(stream, ingestProps, ssOptions));
}
public BlockingKustoUploader(
string blobConnectionString,
string blobContainerName,
KustoConnectionStringBuilder adminCsb,
KustoConnectionStringBuilder kscb,
bool demoMode,
string tableName,
int batchSize,
TimeSpan flushDuration,
bool resetTable = false)
{
Csb = kscb;
AdminCsb = adminCsb;
TableName = tableName;
BatchSize = batchSize;
_flushDuration = flushDuration;
_lastUploadTime = DateTime.UtcNow;
_resetTable = resetTable;
_initializeTable = false;
Completed = new AutoResetEvent(false);
if (Csb != null)
{
_ingestionProperties = new KustoIngestionProperties(Csb.InitialCatalog, tableName);
if (demoMode)
{
_ingestClient = KustoIngestFactory.CreateDirectIngestClient(this.AdminCsb);
}
else
{
_ingestClient = KustoIngestFactory.CreateQueuedIngestClient(Csb);
}
}
if (!string.IsNullOrEmpty(blobConnectionString) &&
!string.IsNullOrEmpty(blobContainerName))
{
blobServiceClient = new BlobServiceClient(blobConnectionString);
blobContainerClient = blobServiceClient.GetBlobContainerClient(blobContainerName);
blobContainerClient.CreateIfNotExists();
}
_nextBatch = new List <IDictionary <string, object> >();
}
public KustoTimelineTelemetryRepository(ILogger <KustoTimelineTelemetryRepository> logger, IOptionsSnapshot <KustoTimelineTelemetryOptions> options)
{
_logger = logger;
// removing the IngestConnectionString was a default setup in local debugging
if (string.IsNullOrEmpty(options.Value.IngestConnectionString))
{
_logger.LogDebug("No ingest connection string provided; will ignore ingest operations");
_ingest = new NullKustoIngestClient();
}
else
{
_ingest = KustoIngestFactory.CreateQueuedIngestClient(options.Value.IngestConnectionString);
}
_query = KustoClientFactory.CreateCslQueryProvider(options.Value.QueryConnectionString);
_database = options.Value.Database;
}
public async Task PublishManualReviews(IEnumerable <SloManualReview> results)
{
var dt = results.ToDataTable();
var kustoConnectionStringBuilderEngine = new KustoConnectionStringBuilder(kustoUrl_).WithAadUserPromptAuthentication();
using (var client = KustoIngestFactory.CreateDirectIngestClient(kustoConnectionStringBuilderEngine))
{
//Ingest from blobs according to the required properties
var kustoIngestionProperties = new KustoIngestionProperties(
databaseName: kustoDb_,
tableName: kustoManualReviewTable_
);
var reader = dt.CreateDataReader();
var result = await client.IngestFromDataReaderAsync(reader, kustoIngestionProperties);
}
}
public BlockingKustoUploader(
string outputFileName,
KustoConnectionStringBuilder adminCsb,
KustoConnectionStringBuilder kscb,
bool demoMode,
string tableName,
int batchSize,
TimeSpan flushDuration,
bool resetTable = false)
{
OutputFileName = outputFileName;
Csb = kscb;
AdminCsb = adminCsb;
TableName = tableName;
BatchSize = batchSize;
_flushDuration = flushDuration;
_lastUploadTime = DateTime.UtcNow;
_resetTable = resetTable;
_initializeTable = false;
Completed = new AutoResetEvent(false);
if (Csb != null)
{
_ingestionProperties = new KustoIngestionProperties(Csb.InitialCatalog, tableName);
if (demoMode)
{
_ingestClient = KustoIngestFactory.CreateDirectIngestClient(this.AdminCsb);
}
else
{
_ingestClient = KustoIngestFactory.CreateQueuedIngestClient(Csb);
}
}
if (!string.IsNullOrEmpty(OutputFileName))
{
outputFile = new StreamWriter(this.OutputFileName);
outputFile.Write($"[{Environment.NewLine}");
}
_nextBatch = new List <IDictionary <string, object> >();
}
public async Task IngestFromCsvStreamAsync(Stream csv, IngestKustoImageInfoOptions options)
{
KustoConnectionStringBuilder connectionBuilder =
new KustoConnectionStringBuilder($"https://{options.Cluster}.kusto.windows.net")
.WithAadApplicationKeyAuthentication(
options.ServicePrincipal.ClientId,
options.ServicePrincipal.Secret,
options.ServicePrincipal.Tenant);
using (IKustoIngestClient client = KustoIngestFactory.CreateDirectIngestClient(connectionBuilder))
{
KustoIngestionProperties properties =
new KustoIngestionProperties(options.Database, options.Table)
{
Format = DataSourceFormat.csv
};
StreamSourceOptions sourceOptions = new StreamSourceOptions {
SourceId = Guid.NewGuid()
};
if (!options.IsDryRun)
{
IKustoIngestionResult result = await client.IngestFromStreamAsync(csv, properties, sourceOptions);
IngestionStatus ingestionStatus = result.GetIngestionStatusBySourceId(sourceOptions.SourceId);
for (int i = 0; i < 10 && ingestionStatus.Status == Status.Pending; i++)
{
await Task.Delay(TimeSpan.FromSeconds(30));
ingestionStatus = result.GetIngestionStatusBySourceId(sourceOptions.SourceId);
}
if (ingestionStatus.Status != Status.Succeeded)
{
throw new InvalidOperationException(
$"Failed to ingest Kusto data.{Environment.NewLine}{ingestionStatus.Details}");
}
else if (ingestionStatus.Status == Status.Pending)
{
throw new InvalidOperationException($"Timeout while ingesting Kusto data.");
}
}
}
}
static void Main(string[] args)
{
// Ingest From Local File using KustoDirectIngestClient (only for test purposes):
// Create Kusto connection string with App Authentication
var kustoConnectionStringBuilderEngine =
new KustoConnectionStringBuilder(@"https://{clusterNameAndRegion}.kusto.windows.net").WithAadApplicationKeyAuthentication(
applicationClientId: "{Application Client ID}",
applicationKey: "{Application Key (secret)}",
authority: "{AAD TenantID or name}");
// Create a disposable client that will execute the ingestion
using (IKustoIngestClient client = KustoIngestFactory.CreateDirectIngestClient(kustoConnectionStringBuilderEngine))
{
//Ingest from blobs according to the required properties
var kustoIngestionProperties = new KustoIngestionProperties(databaseName: "myDB", tableName: "myTable");
client.IngestFromStorageAsync(@"< Path to local file >", ingestionProperties: kustoIngestionProperties).GetAwaiter().GetResult();
}
}
private static async Task IngestTriageItemsIntoKusto(TriageItem[] triageItems, ILogger log)
{
log.LogInformation("Entering IngestTriageItemIntoKusto");
string kustoIngestConnectionString = System.Environment.GetEnvironmentVariable("KustoIngestConnectionString");
string databaseName = System.Environment.GetEnvironmentVariable("KustoDatabaseName");
IKustoIngestClient ingestClient = KustoIngestFactory.CreateQueuedIngestClient(kustoIngestConnectionString);
await KustoHelpers.WriteDataToKustoInMemoryAsync(
ingestClient,
databaseName,
"TimelineIssuesTriage",
log,
triageItems,
b => new[]
{
new KustoValue("ModifiedDateTime", b.ModifiedDateTime.ToString(), KustoDataTypes.DateTime),
new KustoValue("BuildId", b.BuildId.ToString(), KustoDataTypes.Int),
new KustoValue("RecordId", b.RecordId.ToString(), KustoDataTypes.Guid),
new KustoValue("Index", b.Index.ToString(), KustoDataTypes.Int),
new KustoValue("UpdatedCategory", b?.UpdatedCategory, KustoDataTypes.String),
new KustoValue("Url", b?.Url, KustoDataTypes.String)
});
}
private IKustoIngestClient GetKustoIngestClient(string appId, string appKey, EventSinkInfo eventSinkInfo)
{
IKustoIngestClient _client;
try
{
var kcsb = new KustoConnectionStringBuilder(eventSinkInfo.Settings.EndpointUrl)
{
InitialCatalog = eventSinkInfo.Settings.Database,
FederatedSecurity = true
};
kcsb = kcsb.WithAadApplicationKeyAuthentication(appId, appKey, eventSinkInfo.Settings.Authority);
_client = KustoIngestFactory.CreateQueuedIngestClient(kcsb);
}
catch (Exception e)
{
Console.WriteLine(e);
throw;
}
return(_client);
}