public static async Task <string> AdxExportStatusCheck(
[ActivityTrigger] string operationId, ILogger log)
{
using (var client = KustoClientFactory.CreateCslAdminProvider(await GetKustoConnectionStringBuilder()))
{
var operationQuery = CslCommandGenerator.GenerateOperationsShowCommand(Guid.Parse(operationId));
var resultReader = new ObjectReader <OperationsShowCommandResult>(client.ExecuteControlCommand(adxDatabaseName, operationQuery));
var res = resultReader?.FirstOrDefault();
var state = res?.State;
if (state == "Completed")
{
// When the state is completed, we can query the export details which contains the path to the file on blob storage
var operationDetailsQuery = CslCommandGenerator.GenerateOperationDetailsShowCommand(Guid.Parse(operationId));
var resultReader2 = new ObjectReader <DataExportToBlobCommandResult>(client.ExecuteControlCommand(adxDatabaseName, operationDetailsQuery));
var res2 = resultReader2?.FirstOrDefault();
var path = res2?.Path;
return(path);
}
else if (state == "Cancelled")
{
return("Error");
}
else
{
return(null);
}
}
}
/// <summary>
/// Export all records from <paramref name="tableName"/> in database <paramref name="databaseName"/> to external table
/// <paramref name="externalTableName"/>. Split by intervals of <paramref name="step"/> (of ingestion time)
/// </summary>
private static void ExportTableToExternalTable(string databaseName, string tableName, string externalTableName, TimeSpan step)
{
// This authenticates based on user credentials. Can replace with AAD app if needed (see KustoConnectionStringBuilder API)
var connection = new KustoConnectionStringBuilder($"{KustoClusterUri};Fed=true");
var queryClient = KustoClientFactory.CreateCslQueryProvider(connection);
var adminClient = KustoClientFactory.CreateCslAdminProvider(connection);
var minMaxIngestionTime = queryClient.ExecuteQuery <MinMaxDateTime>(databaseName, $"{tableName} | summarize min(ingestion_time()), max(ingestion_time())").Single();
var start = minMaxIngestionTime.Min;
var end = minMaxIngestionTime.Max;
while (start < end)
{
DateTime curEnd = start + step;
var exportCommand = CslCommandGenerator.GenerateExportToExternalTableCommand(externalTableName,
query: $"{tableName} | where ingestion_time() >= {CslDateTimeLiteral.AsCslString(start)} and ingestion_time() < {CslDateTimeLiteral.AsCslString(curEnd)}",
sizeLimitInBytes: MemoryConstants._1GB,
persistDetails: true,
isAsync: true);
var crp = new ClientRequestProperties();
crp.ClientRequestId = $"ExportApp;{Guid.NewGuid().ToString()}";
ExtendedConsole.WriteLine(ConsoleColor.Cyan, $"Executing export command from {start.FastToString()} to {curEnd.FastToString()} with request id {crp.ClientRequestId}");
var operationDetails = adminClient.ExecuteAsyncControlCommand(databaseName, exportCommand, TimeSpan.FromMinutes(60), TimeSpan.FromSeconds(1), crp);
var state = operationDetails.State;
if (state == "Completed")
{
// check num records exported
var command = $"{CslCommandGenerator.GenerateOperationDetailsShowCommand(operationDetails.OperationId)} | summarize sum(NumRecords)";
long exportedRecords = adminClient.ExecuteControlCommand <long>(command).Single();
ExtendedConsole.WriteLine(ConsoleColor.Green, $"Operation {operationDetails.OperationId} completed with state {operationDetails.State} and exported {exportedRecords} records");
}
else
{
// TODO: retry same scope again (or abort and check root cause for failure). Operation may still be running on server side, so retrying
// without checking status can lead to duplicates
ExtendedConsole.WriteLine(ConsoleColor.Red, $"Operation {operationDetails.OperationId} completed with state {operationDetails.State}. Status: {operationDetails.Status}");
}
start = curEnd;
}
}