/// <inheritdoc />
public IBigBrother UseKusto(string kustoEngineName, string kustoEngineLocation, string kustoDb, string tenantId)
{
KustoDbName = kustoDb;
var kustoUri = $"https://{kustoEngineName}.{kustoEngineLocation}.kusto.windows.net";
var kustoIngestUri = $"https://ingest-{kustoEngineName}.{kustoEngineLocation}.kusto.windows.net";
var token = new AzureServiceTokenProvider().GetAccessTokenAsync(kustoUri, string.Empty).Result;
KustoAdminClient = KustoClientFactory.CreateCslAdminProvider(
new KustoConnectionStringBuilder(kustoUri)
{
FederatedSecurity = true,
InitialCatalog = KustoDbName,
AuthorityId = tenantId,
ApplicationToken = token
});
KustoIngestClient = KustoIngestFactory.CreateQueuedIngestClient(
new KustoConnectionStringBuilder(kustoIngestUri)
{
FederatedSecurity = true,
InitialCatalog = KustoDbName,
AuthorityId = tenantId,
ApplicationToken = token
});
SetupKustoSubscription();
return(this);
}
static void Main(string[] args)
{
// Ingest From Local Files using KustoQueuedIngestClient and Ingestion Validation
// Create Kusto connection string with App Authentication
var kustoConnectionStringBuilderDM =
new KustoConnectionStringBuilder(@"https://ingest-{clusterNameAndRegion}.kusto.windows.net").WithAadApplicationKeyAuthentication(
applicationClientId: "{Application Client ID}",
applicationKey: "{Application Key (secret)}",
authority: "{AAD TenantID or name}");
// Create a disposable client that will execute the ingestion
IKustoQueuedIngestClient client = KustoIngestFactory.CreateQueuedIngestClient(kustoConnectionStringBuilderDM);
// Ingest from files according to the required properties
var kustoIngestionProperties = new KustoIngestionProperties(databaseName: "myDB", tableName: "myTable");
client.IngestFromStorageAsync(@"ValidTestFile.csv", kustoIngestionProperties);
client.IngestFromStorageAsync(@"InvalidTestFile.csv", kustoIngestionProperties);
// Waiting for the aggregation
Thread.Sleep(TimeSpan.FromMinutes(8));
// Retrieve and validate failures
var ingestionFailures = client.PeekTopIngestionFailures().GetAwaiter().GetResult();
Ensure.IsTrue((ingestionFailures.Count() > 0), "Failures expected");
// Retrieve, delete and validate failures
ingestionFailures = client.GetAndDiscardTopIngestionFailures().GetAwaiter().GetResult();
Ensure.IsTrue((ingestionFailures.Count() > 0), "Failures expected");
// Dispose of the client
client.Dispose();
}
/// <summary>
/// Constructor. Initializes this object and does nothing else.
/// </summary>
/// <param name="connectionString">Kusto connection string.</param>
/// <param name="database">Database into which to ingest.</param>
/// <param name="table">Table into which to ingest.</param>
/// <param name="csvMapping">Csv mapping that applies to all CSV files to be uploaded via <see cref="UploadSingleCsvFile"/></param>
/// <param name="deleteFilesOnSuccess">Whether to delete files upon successful upload.</param>
/// <param name="checkForIngestionErrors">
/// Whether to check for ingestion errors before disposing this object.
/// Note that at this time not all uploaded files have necessarily been ingested; this class
/// does not wait for ingestions to complete, it only checks for failures of those that have completed.
/// </param>
/// <param name="log">Optional log to which to write some debug information.</param>
public KustoUploader
(
string connectionString,
string database,
string table,
IEnumerable <CsvColumnMapping> csvMapping,
bool deleteFilesOnSuccess,
bool checkForIngestionErrors,
ILog log = null
)
{
_log = log;
_deleteFilesOnSuccess = deleteFilesOnSuccess;
_checkForIngestionErrors = checkForIngestionErrors;
_client = KustoIngestFactory.CreateQueuedIngestClient(connectionString);
_hasUploadErrors = false;
_ingestionProperties = new KustoQueuedIngestionProperties(database, table)
{
CSVMapping = csvMapping,
ReportLevel = IngestionReportLevel.FailuresOnly,
ReportMethod = IngestionReportMethod.Queue,
};
_block = new ActionBlock <FileDescription>
(
UploadSingleCsvFile,
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = 1,
BoundedCapacity = DataflowBlockOptions.Unbounded
}
);
}
static async Task Main(string[] args)
{
// Ingest From a Local File using KustoQueuedIngestClient and report status to a table
// Create Kusto connection string with App Authentication
var kustoConnectionStringBuilderDM =
new KustoConnectionStringBuilder(@"https://ingest-{clusterNameAndRegion}.kusto.windows.net").WithAadApplicationKeyAuthentication(
applicationClientId: "{Application Client ID}",
applicationKey: "{Application Key (secret)}",
authority: "{AAD TenantID or name}");
// Create a disposable client that will execute the ingestion
IKustoQueuedIngestClient client = KustoIngestFactory.CreateQueuedIngestClient(kustoConnectionStringBuilderDM);
// Ingest from a file according to the required properties
var kustoIngestionProperties = new KustoQueuedIngestionProperties(databaseName: "myDB", tableName: "myDB")
{
// Setting the report level to FailuresAndSuccesses will cause both successful and failed ingestions to be reported
// (Rather than the default "FailuresOnly" level)
ReportLevel = IngestionReportLevel.FailuresAndSuccesses,
// Choose the report method of choice
ReportMethod = IngestionReportMethod.Table
};
var filePath = @"< Path to file >";
var fileIdentifier = Guid.NewGuid();
var fileDescription = new FileDescription()
{
FilePath = filePath, SourceId = fileIdentifier
};
var sourceOptions = new StorageSourceOptions()
{
SourceId = fileDescription.SourceId.Value
};
// Execute the ingest operation and save the result.
var clientResult = await client.IngestFromStorageAsync(fileDescription.FilePath,
ingestionProperties : kustoIngestionProperties, sourceOptions);
// Use the fileIdentifier you supplied to get the status of your ingestion
var ingestionStatus = clientResult.GetIngestionStatusBySourceId(fileIdentifier);
while (ingestionStatus.Status == Status.Pending)
{
// Wait a minute...
Thread.Sleep(TimeSpan.FromMinutes(1));
// Try again
ingestionStatus = clientResult.GetIngestionStatusBySourceId(fileIdentifier);
}
// Verify the results of the ingestion
Ensure.ConditionIsMet(ingestionStatus.Status == Status.Succeeded,
"The file should have been ingested successfully");
// Dispose of the client
client.Dispose();
}
protected FI(ILogger iLog, Microsoft.Azure.Management.Fluent.IAzure iAzure, string iRGName, string kustoConn, string kustoDBName, string kustoTableName)
{
curRGName = iRGName;
log = iLog;
curSubName = iAzure.SubscriptionId;
ingestClient = KustoIngestFactory.CreateQueuedIngestClient(kustoConn);
ingestProps = new KustoIngestionProperties(kustoDBName, kustoTableName);
ingestProps.Format = Kusto.Data.Common.DataSourceFormat.csv;
}
public async Task InsertRowAsync(T row, CancellationToken cancellationToken = default)
{
using IKustoQueuedIngestClient ingestClient = KustoIngestFactory.CreateQueuedIngestClient(_connectionStringBuilder);
string serializedData = JsonConvert.SerializeObject(row);
byte[] serializedBytes = Encoding.UTF8.GetBytes(serializedData);
using MemoryStream dataStream = new MemoryStream(serializedBytes);
// IKustoQueuedIngestClient doesn't support cancellation at the moment. Update the line below if it does in the future.
await ingestClient.IngestFromStreamAsync(dataStream, _ingestionProperties, leaveOpen : true);
}
private static bool TryInitialize(ILogger log)
{
lock (_lock)
{
if (!_initialized)
{
string kustoIngestUrl = GetEnvVariable("KustoIngestUrl");
string clientId = GetEnvVariable("ClientId");
string clientSecret = GetEnvVariable("ClientSecret");
string tenantId = GetEnvVariable("TenantId");
if (String.IsNullOrWhiteSpace(kustoIngestUrl) ||
String.IsNullOrWhiteSpace(clientId) ||
String.IsNullOrWhiteSpace(clientSecret) ||
String.IsNullOrWhiteSpace(tenantId))
{
log.LogError($"Could not initialize the Kusto client because the connection parameters are wrong (url: {kustoIngestUrl} clientId: {clientId}, tenant: {tenantId}");
return(false);
}
//Initialize adx
connection =
new KustoConnectionStringBuilder(GetEnvVariable("KustoIngestUrl")).WithAadApplicationKeyAuthentication(
applicationClientId: GetEnvVariable("ClientId"),
applicationKey: GetEnvVariable("ClientSecret"),
authority: GetEnvVariable("TenantId"));
adx = KustoIngestFactory.CreateQueuedIngestClient(connection);
if (adx != null)
{
_initialized = true;
log.LogInformation("Function successfully initialized");
return(true);
}
else
{
log.LogWarning("Function not successfully initialized");
}
}
else
{
return(true);
}
}
return(false);
}
static void Main(string[] args)
{
// Ingest From a Local Files using KustoQueuedIngestClient and report status to a queue
// Create Kusto connection string with App Authentication
var kustoConnectionStringBuilderDM =
new KustoConnectionStringBuilder(@"https://ingest-{clusterNameAndRegion}.kusto.windows.net").WithAadApplicationKeyAuthentication(
applicationClientId: "{Application Client ID}",
applicationKey: "{Application Key (secret)}",
authority: "{AAD TenantID or name}");
// Create a disposable client that will execute the ingestion
IKustoQueuedIngestClient client = KustoIngestFactory.CreateQueuedIngestClient(kustoConnectionStringBuilderDM);
// Ingest from a file according to the required properties
var kustoIngestionProperties = new KustoQueuedIngestionProperties(databaseName: "myDB", tableName: "myTable")
{
// Setting the report level to FailuresAndSuccesses will cause both successful and failed ingestions to be reported
// (Rather than the default "FailuresOnly" level - which is demonstrated in the
// 'Ingest From Local File(s) using KustoQueuedIngestClient and Ingestion Validation' section)
ReportLevel = IngestionReportLevel.FailuresAndSuccesses,
// Choose the report method of choice. 'Queue' is the default method.
// For the sake of the example, we will choose it anyway.
ReportMethod = IngestionReportMethod.Queue
};
client.IngestFromStorageAsync("ValidTestFile.csv", kustoIngestionProperties);
client.IngestFromStorageAsync("InvalidTestFile.csv", kustoIngestionProperties);
// Waiting for the aggregation
Thread.Sleep(TimeSpan.FromMinutes(8));
// Retrieve and validate failures
var ingestionFailures = client.PeekTopIngestionFailures().GetAwaiter().GetResult();
Ensure.IsTrue((ingestionFailures.Count() > 0), "The failed ingestion should have been reported to the failed ingestions queue");
// Retrieve, delete and validate failures
ingestionFailures = client.GetAndDiscardTopIngestionFailures().GetAwaiter().GetResult();
Ensure.IsTrue((ingestionFailures.Count() > 0), "The failed ingestion should have been reported to the failed ingestions queue");
// Verify the success has also been reported to the queue
var ingestionSuccesses = client.GetAndDiscardTopIngestionSuccesses().GetAwaiter().GetResult();
Ensure.ConditionIsMet((ingestionSuccesses.Count() > 0),
"The successful ingestion should have been reported to the successful ingestions queue");
// Dispose of the client
client.Dispose();
}
/// <summary>
/// Saves the details of the <see cref="HttpResponseMessage"/> to
/// </summary>
/// <param name="monitorName"></param>
/// <param name="httpResponse"></param>
/// <returns></returns>
public async Task ReportUrlAccessAsync(string monitorName, HttpResponseMessage httpResponse, CancellationToken cancellationToken = default)
{
HttpRequestLogEntry logEntry = new HttpRequestLogEntry()
{
MonitorName = monitorName,
EventTime = DateTime.UtcNow,
RequestedUrl = httpResponse.RequestMessage.RequestUri.AbsoluteUri,
HttpStatusCode = (int)httpResponse.StatusCode
};
KustoConnectionStringBuilder kcsb = new KustoConnectionStringBuilder($"https://ingest-{ServiceNameAndRegion}.kusto.windows.net")
.WithAadManagedIdentity("system");
using IKustoQueuedIngestClient ingestClient = KustoIngestFactory.CreateQueuedIngestClient(kcsb);
KustoQueuedIngestionProperties ingestProps = new KustoQueuedIngestionProperties(DatabaseName, TableName)
{
ReportLevel = IngestionReportLevel.FailuresOnly,
ReportMethod = IngestionReportMethod.Queue,
IngestionMapping = new IngestionMapping()
{
IngestionMappingKind = Kusto.Data.Ingestion.IngestionMappingKind.Json,
IngestionMappings = HttpRequestLogColumnMapping
},
Format = DataSourceFormat.json
};
using MemoryStream memStream = new MemoryStream();
using StreamWriter writer = new StreamWriter(memStream);
writer.WriteLine(JsonConvert.SerializeObject(logEntry));
writer.Flush();
memStream.Seek(0, SeekOrigin.Begin);
// IKustoQueuedIngestClient doesn't support cancellation at the moment. Update the line below if it does in the future.
await ingestClient.IngestFromStreamAsync(memStream, ingestProps, leaveOpen : true);
}
protected AzureDataExplorerService(IKustoClientFactory kustoClientFactory)
{
this._ingestionClient = kustoClientFactory.GetQueuedIngestClient();
this._kustoConnectionStringBuilder = kustoClientFactory.KustoConnectionStringBuilder;
}
