private static void PopulateTypesIndexData()
{
// Build list of columns and mappings to provision Kusto
var kustoColumns = new List <string>();
var columnMappings = new List <JsonColumnMapping>();
foreach (var entry in KustoColumnType)
{
var name = entry.Key;
kustoColumns.Add($"{name}:{entry.Value}");
columnMappings.Add(new JsonColumnMapping()
{
ColumnName = name, JsonPath = $"$.{name}"
});
}
// Send drop table ifexists command to Kusto
var command = CslCommandGenerator.GenerateTableDropCommand(TypesIndex, true);
KustoExecute(command);
// Send create table command to Kusto
command = $".create table {TypesIndex} ({string.Join(", ", kustoColumns)})";
Console.WriteLine(command);
KustoExecute(command);
// Send create table mapping command to Kusto
command = CslCommandGenerator.GenerateTableJsonMappingCreateCommand(
TypesIndex, TypesMapping, columnMappings, true);
KustoExecute(command);
command = ".append types_index <|" +
"print x = true, datetime('2020-02-23T07:22:29.1990163Z'), guid(74be27de-1e4e-49d9-b579-fe0b331d3642), int(17), long(17), real(0.3), 'string type', 30m, decimal(0.3), dynamic({'a':123, 'b':'hello'})";
KustoExecute(command);
}
private void CreateOrResetTable(IDictionary <string, object> value)
{
using (var admin = KustoClientFactory.CreateCslAdminProvider(kscbAdmin))
{
string dropTable = CslCommandGenerator.GenerateTableDropCommand(_table, true);
admin.ExecuteControlCommand(dropTable);
CreateMergeKustoTable(admin, value);
}
}
/// <summary>
/// Populate the Kusto backend with test data.
/// </summary>
/// <param name="kusto"></param>
/// <param name="db"></param>
/// <param name="table"></param>
/// <param name="mapping"></param>
/// <param name="structure">JSON file containing the Elasticsearch index structure. Elasticsearch types will be converted to Kusto types. Note that the method only supported a small set of Elasticsearch types.</param>
/// <param name="dataFile">Gzipped JSON file containing the data to be loaded.</param>
/// <returns>Bulk Insert operation result.</returns>
public static async Task <IKustoIngestionResult> Populate(KustoConnectionStringBuilder kusto, string db, string table, string mapping, string structure, string dataFile)
{
var struc = JObject.Parse(structure);
var properties = struc["mappings"]["_doc"]["properties"] as JObject;
// Build list of columns and mappings to provision Kusto
var kustoColumns = new List <string>();
var columnMappings = new List <ColumnMapping>();
foreach (var prop in properties)
{
string name = prop.Key;
JObject value = prop.Value as JObject;
string type = (string)value["type"];
if (ES2KUSTOTYPE.ContainsKey(type))
{
type = ES2KUSTOTYPE[type];
}
kustoColumns.Add($"{name}:{type}");
columnMappings.Add(new ColumnMapping()
{
ColumnName = name,
Properties = new Dictionary <string, string>
{
["Path"] = $"$.{name}",
},
});
}
using (var kustoAdminClient = KustoClientFactory.CreateCslAdminProvider(kusto))
{
// Send drop table ifexists command to Kusto
var command = CslCommandGenerator.GenerateTableDropCommand(table, true);
kustoAdminClient.ExecuteControlCommand(command);
// Send create table command to Kusto
command = $".create table {table} ({string.Join(", ", kustoColumns)})";
Console.WriteLine(command);
kustoAdminClient.ExecuteControlCommand(command);
// Send create table mapping command to Kusto
command = CslCommandGenerator.GenerateTableMappingCreateCommand(IngestionMappingKind.Json, table, mapping, columnMappings);
kustoAdminClient.ExecuteControlCommand(command);
}
// Log information to console.
// Can't use Console.WriteLine here: https://github.com/nunit/nunit3-vs-adapter/issues/266
TestContext.Progress.WriteLine($"Ingesting {dataFile} as compressed data into Kusto");
// Populate Kusto
using Stream fs = File.OpenRead(dataFile);
return(await KustoIngest(kusto, db, table, mapping, fs));
}
static void ResetTable(KustoConnectionStringBuilder kscb, string tableName, Type type)
{
using (var admin = KustoClientFactory.CreateCslAdminProvider(kscb))
{
string dropTable = CslCommandGenerator.GenerateTableDropCommand(tableName, true);
admin.ExecuteControlCommand(dropTable);
string createTable = CslCommandGenerator.GenerateTableCreateCommand(tableName, type);
admin.ExecuteControlCommand(createTable);
string enableIngestTime = CslCommandGenerator.GenerateIngestionTimePolicyAlterCommand(tableName, true);
admin.ExecuteControlCommand(enableIngestTime);
}
}
/// <summary>
/// Remove any functions and tables that are present in the database. Note that this should only be called when
/// connecting to the temporary database
/// </summary>
public void CleanDatabase()
{
if (!_tempDatabaseUsed)
{
throw new Exception("CleanDatabase() was called on something other than the temporary database. This method will wipe out the entire database schema and data.");
}
var schema = GetDatabaseSchema();
foreach (var function in schema.Functions)
{
string command = CslCommandGenerator.GenerateFunctionDropCommand(function.Value.Name, true);
_adminClient.ExecuteControlCommand(command);
}
foreach (var table in schema.Tables)
{
string command = CslCommandGenerator.GenerateTableDropCommand(table.Value.Name, true);
_adminClient.ExecuteControlCommand(command);
}
}
/// <summary>
/// Populate the Kusto backend with test data.
/// </summary>
/// <param name="kusto">Kusto connection string builder.</param>
/// <param name="db">Kusto database.</param>
/// <param name="table">Kusto table within database.</param>
/// <returns>Bulk Insert operation result.</returns>
public static async Task <IDataReader> PopulateTypesIndex(KustoConnectionStringBuilder kusto, string db, string table)
{
// Build list of columns and mappings to provision Kusto
var kustoColumns = new List <string>();
foreach (var entry in KustoColumnType)
{
kustoColumns.Add($"{entry.Key}:{entry.Value}");
}
using var kustoAdminClient = KustoClientFactory.CreateCslAdminProvider(kusto);
// Send drop table ifexists command to Kusto
var command = CslCommandGenerator.GenerateTableDropCommand(table, true);
await kustoAdminClient.ExecuteControlCommandAsync(db, command);
// Send create table command to Kusto
command = $".create table {table} ({string.Join(", ", kustoColumns)})";
Console.WriteLine(command);
return(await kustoAdminClient.ExecuteControlCommandAsync(db, command));
}