private async Task <IKustoIngestionResult> IngestFromStreamAsync(
string csv, IKustoIngestClient client, KustoIngestionProperties properties, StreamSourceOptions sourceOptions)
{
using MemoryStream stream = new();
using StreamWriter writer = new(stream);
writer.Write(csv);
writer.Flush();
stream.Seek(0, SeekOrigin.Begin);
return(await client.IngestFromStreamAsync(stream, properties, sourceOptions));
}
/// <summary>
/// Ingest data into Kusto.
/// </summary>
/// <param name="table">Name of table to ingest into.</param>
/// <param name="mappingName">Name of table mapping to ingest with.</param>
/// <param name="stream">input JSON data stream.</param>
/// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
private static async Task <IKustoIngestionResult> KustoIngest(KustoConnectionStringBuilder kusto, string db, string table, string mappingName, Stream stream)
{
// Create a disposable client that will execute the ingestion
using IKustoIngestClient client = KustoIngestFactory.CreateDirectIngestClient(kusto);
var ingestProps = new KustoIngestionProperties(db, table)
{
JSONMappingReference = mappingName,
Format = DataSourceFormat.json,
};
var ssOptions = new StreamSourceOptions
{
CompressionType = DataSourceCompressionType.GZip,
};
return(await client.IngestFromStreamAsync(stream, ingestProps, ssOptions));
}
public async Task IngestFromCsvStreamAsync(Stream csv, IngestKustoImageInfoOptions options)
{
KustoConnectionStringBuilder connectionBuilder =
new KustoConnectionStringBuilder($"https://{options.Cluster}.kusto.windows.net")
.WithAadApplicationKeyAuthentication(
options.ServicePrincipal.ClientId,
options.ServicePrincipal.Secret,
options.ServicePrincipal.Tenant);
using (IKustoIngestClient client = KustoIngestFactory.CreateDirectIngestClient(connectionBuilder))
{
KustoIngestionProperties properties =
new KustoIngestionProperties(options.Database, options.Table)
{
Format = DataSourceFormat.csv
};
StreamSourceOptions sourceOptions = new StreamSourceOptions {
SourceId = Guid.NewGuid()
};
if (!options.IsDryRun)
{
IKustoIngestionResult result = await client.IngestFromStreamAsync(csv, properties, sourceOptions);
IngestionStatus ingestionStatus = result.GetIngestionStatusBySourceId(sourceOptions.SourceId);
for (int i = 0; i < 10 && ingestionStatus.Status == Status.Pending; i++)
{
await Task.Delay(TimeSpan.FromSeconds(30));
ingestionStatus = result.GetIngestionStatusBySourceId(sourceOptions.SourceId);
}
if (ingestionStatus.Status != Status.Succeeded)
{
throw new InvalidOperationException(
$"Failed to ingest Kusto data.{Environment.NewLine}{ingestionStatus.Details}");
}
else if (ingestionStatus.Status == Status.Pending)
{
throw new InvalidOperationException($"Timeout while ingesting Kusto data.");
}
}
}
}
private async Task <IEnumerable <IngestionStatus> > KustoIngestAsync(IReadOnlyList <T> rows)
{
Contract.Requires(rows.Count > 0);
using var stream = new MemoryStream();
using var writer = new StreamWriter(stream, encoding: Encoding.UTF8);
foreach (var row in rows)
{
await writer.WriteLineAsync(JsonConvert.SerializeObject(row));
}
await writer.FlushAsync();
stream.Seek(0, SeekOrigin.Begin);
var ingestion = await _kustoIngestClient.IngestFromStreamAsync(stream, _kustoIngestionProperties, new StreamSourceOptions()
{
LeaveOpen = true,
});
return(ingestion.GetIngestionStatusCollection());
}
public static async Task WriteDataToKustoInMemoryAsync <T>(
IKustoIngestClient client,
string databaseName,
string tableName,
ILogger logger,
IEnumerable <T> data,
Func <T, IList <KustoValue> > mapFunc)
{
CsvColumnMapping[] mappings = null;
int size = 5;
using (var stream = new MemoryStream())
{
using (var writer = new StreamWriter(stream, new UTF8Encoding(false), 1024, leaveOpen: true))
{
foreach (T d in data)
{
var dataList = new List <string>(size);
if (mappings == null)
{
var mapList = new List <CsvColumnMapping>();
foreach (KustoValue p in mapFunc(d))
{
mapList.Add(new CsvColumnMapping {
ColumnName = p.Column, CslDataType = p.DataType
});
dataList.Add(p.StringValue);
}
mappings = mapList.ToArray();
size = mappings.Length;
}
else
{
dataList.AddRange(mapFunc(d).Select(p => p.StringValue));
}
await writer.WriteCsvLineAsync(dataList);
}
}
if (mappings == null)
{
logger.LogInformation("No rows to upload.");
return;
}
for (int i = 0; i < mappings.Length; i++)
{
mappings[i].Ordinal = i;
}
stream.Seek(0, SeekOrigin.Begin);
logger.LogInformation($"Ingesting {mappings.Length} columns at {stream.Length} bytes...");
await client.IngestFromStreamAsync(
stream,
new KustoQueuedIngestionProperties(databaseName, tableName)
{
Format = DataSourceFormat.csv,
ReportLevel = IngestionReportLevel.FailuresOnly,
ReportMethod = IngestionReportMethod.Queue,
CSVMapping = mappings
});
logger.LogTrace("Ingest complete");
}
}
public async Task WriteRow(string message)
{
await _kustoClient.IngestFromStreamAsync(GenerateStreamFromString(message), _kustoProperties).ConfigureAwait(false);
}