/// <inheritdoc />
public IBigBrother UseKusto(string kustoEngineName, string kustoEngineLocation, string kustoDb, string tenantId)
{
KustoDbName = kustoDb;
var kustoUri = $"https://{kustoEngineName}.{kustoEngineLocation}.kusto.windows.net";
var kustoIngestUri = $"https://ingest-{kustoEngineName}.{kustoEngineLocation}.kusto.windows.net";
var token = new AzureServiceTokenProvider().GetAccessTokenAsync(kustoUri, string.Empty).Result;
KustoAdminClient = KustoClientFactory.CreateCslAdminProvider(
new KustoConnectionStringBuilder(kustoUri)
{
FederatedSecurity = true,
InitialCatalog = KustoDbName,
AuthorityId = tenantId,
ApplicationToken = token
});
KustoIngestClient = KustoIngestFactory.CreateQueuedIngestClient(
new KustoConnectionStringBuilder(kustoIngestUri)
{
FederatedSecurity = true,
InitialCatalog = KustoDbName,
AuthorityId = tenantId,
ApplicationToken = token
});
SetupKustoSubscription();
return(this);
}
public BlockingKustoUploader(
KustoConnectionStringBuilder kscb,
string tableName,
int batchSize,
TimeSpan flushDuration)
{
csb = kscb;
TableName = TableName;
BatchSize = batchSize;
_flushDuration = flushDuration;
_lastUploadTime = DateTime.Now;
Completed = new AutoResetEvent(false);
_ingestionProperties = new KustoIngestionProperties(csb.InitialCatalog, tableName);
_fields = typeof(EtwEvent).GetFields().Select(f => f.Name).ToArray();
if (csb.DataSource.StartsWith("https://ingest-"))
{
_ingestClient = KustoIngestFactory.CreateQueuedIngestClient(csb);
}
else
{
_ingestClient = KustoIngestFactory.CreateDirectIngestClient(csb);
}
_nextBatch = new List <T>();
}
protected FI(ILogger iLog, Microsoft.Azure.Management.Fluent.IAzure iAzure, string iRGName, string kustoConn, string kustoDBName, string kustoTableName)
{
curRGName = iRGName;
log = iLog;
curSubName = iAzure.SubscriptionId;
ingestClient = KustoIngestFactory.CreateQueuedIngestClient(kustoConn);
ingestProps = new KustoIngestionProperties(kustoDBName, kustoTableName);
ingestProps.Format = Kusto.Data.Common.DataSourceFormat.csv;
}
public async Task InsertRowAsync(T row, CancellationToken cancellationToken = default)
{
using IKustoQueuedIngestClient ingestClient = KustoIngestFactory.CreateQueuedIngestClient(_connectionStringBuilder);
string serializedData = JsonConvert.SerializeObject(row);
byte[] serializedBytes = Encoding.UTF8.GetBytes(serializedData);
using MemoryStream dataStream = new MemoryStream(serializedBytes);
// IKustoQueuedIngestClient doesn't support cancellation at the moment. Update the line below if it does in the future.
await ingestClient.IngestFromStreamAsync(dataStream, _ingestionProperties, leaveOpen : true);
}
internal IBigBrother UseKustoInternal(string kustoEngineName, string kustoEngineLocation, string kustoDb, string tenantId)
{
try
{
KustoDbName = kustoDb;
var kustoUri = $"https://{kustoEngineName}.{kustoEngineLocation}.kusto.windows.net";
var token = new AzureServiceTokenProvider().GetAccessTokenAsync(kustoUri, string.Empty).Result;
KustoAdminClient = KustoClientFactory.CreateCslAdminProvider(
new KustoConnectionStringBuilder(kustoUri)
{
FederatedSecurity = true,
InitialCatalog = KustoDbName,
Authority = tenantId,
ApplicationToken = token
});
var kustoQueuedIngestUri = $"https://ingest-{kustoEngineName}.{kustoEngineLocation}.kusto.windows.net";
var kustoIngestUri = $"https://{kustoEngineName}.{kustoEngineLocation}.kusto.windows.net";
KustoQueuedIngestClient = KustoIngestFactory.CreateQueuedIngestClient(
new KustoConnectionStringBuilder(kustoQueuedIngestUri)
{
FederatedSecurity = true,
InitialCatalog = KustoDbName,
Authority = tenantId,
ApplicationToken = token
});
KustoDirectIngestClient = KustoIngestFactory.CreateDirectIngestClient(
new KustoConnectionStringBuilder(kustoIngestUri)
{
FederatedSecurity = true,
InitialCatalog = KustoDbName,
Authority = tenantId,
ApplicationToken = token
});
SetupKustoSubscription();
KustoIngestionTimeMetric = InternalClient.GetMetric(new MetricIdentifier("", "KustoIngestionTime"));
}
catch (Exception e)
{
InternalStream.OnNext(e.ToExceptionEvent());
}
return(this);
}
public BlockingKustoUploader(
string blobConnectionString,
string blobContainerName,
KustoConnectionStringBuilder adminCsb,
KustoConnectionStringBuilder kscb,
bool demoMode,
string tableName,
int batchSize,
TimeSpan flushDuration,
bool resetTable = false)
{
Csb = kscb;
AdminCsb = adminCsb;
TableName = tableName;
BatchSize = batchSize;
_flushDuration = flushDuration;
_lastUploadTime = DateTime.UtcNow;
_resetTable = resetTable;
_initializeTable = false;
Completed = new AutoResetEvent(false);
if (Csb != null)
{
_ingestionProperties = new KustoIngestionProperties(Csb.InitialCatalog, tableName);
if (demoMode)
{
_ingestClient = KustoIngestFactory.CreateDirectIngestClient(this.AdminCsb);
}
else
{
_ingestClient = KustoIngestFactory.CreateQueuedIngestClient(Csb);
}
}
if (!string.IsNullOrEmpty(blobConnectionString) &&
!string.IsNullOrEmpty(blobContainerName))
{
blobServiceClient = new BlobServiceClient(blobConnectionString);
blobContainerClient = blobServiceClient.GetBlobContainerClient(blobContainerName);
blobContainerClient.CreateIfNotExists();
}
_nextBatch = new List <IDictionary <string, object> >();
}
public KustoTimelineTelemetryRepository(ILogger <KustoTimelineTelemetryRepository> logger, IOptionsSnapshot <KustoTimelineTelemetryOptions> options)
{
_logger = logger;
// removing the IngestConnectionString was a default setup in local debugging
if (string.IsNullOrEmpty(options.Value.IngestConnectionString))
{
_logger.LogDebug("No ingest connection string provided; will ignore ingest operations");
_ingest = new NullKustoIngestClient();
}
else
{
_ingest = KustoIngestFactory.CreateQueuedIngestClient(options.Value.IngestConnectionString);
}
_query = KustoClientFactory.CreateCslQueryProvider(options.Value.QueryConnectionString);
_database = options.Value.Database;
}
/// <param name="engineConnectionString">Indicates the connection to the Kusto engine service.</param>
/// <param name="dmConnectionString">Indicates the connection to the Kusto data management service.</param>
public static void KustoIngest(string ingestURI, string kustoConnectURI, string engineURI, string dmsURI, string database, string adxtable,
string tableMapping, Stream ingestionStream,
string AppclientID, string AppKey, string key, bool isBatch,
bool isMulti, bool createTables, bool getFromStorage, string blobPath)
{
//var adxtenantId = "<TenantId>";
// var kustoUri = "https://<ClusterName>.<Region>.kusto.windows.net/";
var adxingestUri = ingestURI;// "https://ingest-<ClusterName>.<Region>.kusto.windows.net";
var adxingestConnectionStringBuilder = new KustoConnectionStringBuilder(adxingestUri)
.WithAadApplicationKeyAuthentication(AppclientID, AppKey, key);
var ConnectionStringBuilder = new KustoConnectionStringBuilder(kustoConnectURI)
.WithAadApplicationKeyAuthentication(AppclientID, AppKey, key);
var streamAdxEngineConnectionStringBuilder = new KustoConnectionStringBuilder(kustoConnectURI)
.WithAadApplicationKeyAuthentication(AppclientID, AppKey, key);
var streamAdxDmsURIConnectionStringBuilder = new KustoConnectionStringBuilder(ingestURI)
.WithAadApplicationKeyAuthentication(AppclientID, AppKey, key);
if (createTables)
{
Console.WriteLine("Creating ADX table: " + adxtable);
var createCommand = CreateADXTable(database, adxtable, ConnectionStringBuilder);
Console.WriteLine("Successfully created ADX table: " + adxtable);
CreateTableMapping(database, adxtable, tableMapping, ConnectionStringBuilder);
Console.WriteLine("Successfully created ADX table mapping: " + tableMapping);
}
var kustoClient = KustoClientFactory.CreateCslAdminProvider(ConnectionStringBuilder);
var ingestClient = KustoIngestFactory.CreateQueuedIngestClient(adxingestConnectionStringBuilder);
var streamIngestClient = KustoIngestFactory.
CreateManagedStreamingIngestClient(streamAdxEngineConnectionStringBuilder, streamAdxDmsURIConnectionStringBuilder);
if (isMulti)
{
using (kustoClient)
{
var tablePolicyAlterCommand =
CslCommandGenerator.GenerateTableAlterStreamingIngestionPolicyCommand(adxtable, isEnabled: true);
kustoClient.ExecuteControlCommand(database, tablePolicyAlterCommand);
var command =
CslCommandGenerator.GenerateTableAlterIngestionBatchingPolicyCommand(
database,
adxtable,
new IngestionBatchingPolicy(maximumBatchingTimeSpan: TimeSpan.FromSeconds(2.0),
maximumNumberOfItems: 10000, maximumRawDataSizeMB: 1024));
kustoClient.ExecuteControlCommand(command);
}
}
if (isBatch)
{
using (kustoClient)
{
var command =
CslCommandGenerator.GenerateTableAlterIngestionBatchingPolicyCommand(
database,
adxtable,
new IngestionBatchingPolicy(maximumBatchingTimeSpan: TimeSpan.FromSeconds(2.0),
maximumNumberOfItems: 10000, maximumRawDataSizeMB: 1024));
kustoClient.ExecuteControlCommand(command);
}
}
else
{
if (!isMulti)
{
var tablePolicyAlterCommand =
CslCommandGenerator.GenerateTableAlterStreamingIngestionPolicyCommand(adxtable, isEnabled: true);
kustoClient.ExecuteControlCommand(database, tablePolicyAlterCommand);
}
}
var properties =
new KustoQueuedIngestionProperties(database, adxtable)
{
Format = DataSourceFormat.csv,
IngestionMapping = new IngestionMapping()
{
IngestionMappingReference = tableMapping,
IngestionMappingKind = Kusto.Data.Ingestion.IngestionMappingKind.Csv
},
IgnoreFirstRecord = true
};
//ingestClient.IngestFromStreamAsync(ingestionStream, ingestionProperties: properties).GetAwaiter().GetResult();
if (isBatch)
{
if (getFromStorage)
{
//can occour in stream still!!
ingestClient.IngestFromStorageAsync(blobPath, ingestionProperties: properties).Wait();
Console.WriteLine("Ingestion from blob completed successfully!!");
}
else
{
ingestClient.IngestFromStream(ingestionStream, ingestionProperties: properties).GetIngestionStatusCollection();
Console.WriteLine("Ingestion from stream completed successfully!!");
}
}
else
{
if (getFromStorage)
{
//can occour in stream still!!
streamIngestClient.IngestFromStorageAsync(blobPath, ingestionProperties: properties).Wait();
Console.WriteLine("Ingestion from blob completed successfully!!");
}
else
{
streamIngestClient.IngestFromStream(ingestionStream, ingestionProperties: properties).GetIngestionStatusCollection();
Console.WriteLine("Ingestion from stream completed successfully!!");
}
}
}
static void Main(string[] args)
{
var clusterName = "KustoLab";
var db = "KustoIngestClientDemo";
var table = "Table1";
var mappingName = "Table1_mapping_1";
var serviceNameAndRegion = "clusterNameAndRegion"; // For example, "mycluster.westus"
var authority = "AAD Tenant or name"; // For example, "microsoft.com"
// Set up table
var kcsbEngine =
new KustoConnectionStringBuilder($"https://{serviceNameAndRegion}.kusto.windows.net").WithAadUserPromptAuthentication(authority: $"{authority}");
using (var kustoAdminClient = KustoClientFactory.CreateCslAdminProvider(kcsbEngine))
{
var columns = new List <Tuple <string, string> >()
{
new Tuple <string, string>("Column1", "System.Int64"),
new Tuple <string, string>("Column2", "System.DateTime"),
new Tuple <string, string>("Column3", "System.String"),
};
var command = CslCommandGenerator.GenerateTableCreateCommand(table, columns);
kustoAdminClient.ExecuteControlCommand(databaseName: db, command: command);
// Set up mapping
var columnMappings = new List <JsonColumnMapping>();
columnMappings.Add(new JsonColumnMapping()
{
ColumnName = "Column1", JsonPath = "$.Id"
});
columnMappings.Add(new JsonColumnMapping()
{
ColumnName = "Column2", JsonPath = "$.Timestamp"
});
columnMappings.Add(new JsonColumnMapping()
{
ColumnName = "Column3", JsonPath = "$.Message"
});
command = CslCommandGenerator.GenerateTableJsonMappingCreateCommand(
table, mappingName, columnMappings);
kustoAdminClient.ExecuteControlCommand(databaseName: db, command: command);
}
// Create Ingest Client
var kcsbDM =
new KustoConnectionStringBuilder($"https://ingest-{serviceNameAndRegion}.kusto.windows.net").WithAadUserPromptAuthentication(authority: $"{authority}");
using (var ingestClient = KustoIngestFactory.CreateQueuedIngestClient(kcsbDM))
{
var ingestProps = new KustoQueuedIngestionProperties(db, table);
// For the sake of getting both failure and success notifications we set this to IngestionReportLevel.FailuresAndSuccesses
// Usually the recommended level is IngestionReportLevel.FailuresOnly
ingestProps.ReportLevel = IngestionReportLevel.FailuresAndSuccesses;
ingestProps.ReportMethod = IngestionReportMethod.Queue;
// Setting FlushImmediately to 'true' overrides any aggregation preceding the ingestion.
// Not recommended unless you are certain you know what you are doing
ingestProps.FlushImmediately = true;
ingestProps.JSONMappingReference = mappingName;
ingestProps.Format = DataSourceFormat.json;
// Prepare data for ingestion
using (var memStream = new MemoryStream())
using (var writer = new StreamWriter(memStream))
{
for (int counter = 1; counter <= 10; ++counter)
{
writer.WriteLine(
"{{ \"Id\":\"{0}\", \"Timestamp\":\"{1}\", \"Message\":\"{2}\" }}",
counter, DateTime.UtcNow.AddSeconds(100 * counter),
$"This is a dummy message number {counter}");
}
writer.Flush();
memStream.Seek(0, SeekOrigin.Begin);
// Post ingestion message
var res = ingestClient.IngestFromStreamAsync(memStream, ingestProps, leaveOpen: true);
}
// Wait a bit (20s) and retrieve all notifications:
Thread.Sleep(20000);
var errors = ingestClient.GetAndDiscardTopIngestionFailures().GetAwaiter().GetResult();
var successes = ingestClient.GetAndDiscardTopIngestionSuccesses().GetAwaiter().GetResult();
errors.ForEach((f) => { Console.WriteLine($"Ingestion error: {f.Info.Details}"); });
successes.ForEach((s) => { Console.WriteLine($"Ingested: {s.Info.IngestionSourcePath}"); });
}
}