private async Task <IKustoIngestionResult> IngestFromStreamAsync( string csv, IKustoIngestClient client, KustoIngestionProperties properties, StreamSourceOptions sourceOptions) { using MemoryStream stream = new(); using StreamWriter writer = new(stream); writer.Write(csv); writer.Flush(); stream.Seek(0, SeekOrigin.Begin); return(await client.IngestFromStreamAsync(stream, properties, sourceOptions)); }
/// <summary> /// Ingest data into Kusto. /// </summary> /// <param name="table">Name of table to ingest into.</param> /// <param name="mappingName">Name of table mapping to ingest with.</param> /// <param name="stream">input JSON data stream.</param> /// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns> private static async Task <IKustoIngestionResult> KustoIngest(KustoConnectionStringBuilder kusto, string db, string table, string mappingName, Stream stream) { // Create a disposable client that will execute the ingestion using IKustoIngestClient client = KustoIngestFactory.CreateDirectIngestClient(kusto); var ingestProps = new KustoIngestionProperties(db, table) { JSONMappingReference = mappingName, Format = DataSourceFormat.json, }; var ssOptions = new StreamSourceOptions { CompressionType = DataSourceCompressionType.GZip, }; return(await client.IngestFromStreamAsync(stream, ingestProps, ssOptions)); }
public async Task IngestFromCsvStreamAsync(Stream csv, IngestKustoImageInfoOptions options) { KustoConnectionStringBuilder connectionBuilder = new KustoConnectionStringBuilder($"https://{options.Cluster}.kusto.windows.net") .WithAadApplicationKeyAuthentication( options.ServicePrincipal.ClientId, options.ServicePrincipal.Secret, options.ServicePrincipal.Tenant); using (IKustoIngestClient client = KustoIngestFactory.CreateDirectIngestClient(connectionBuilder)) { KustoIngestionProperties properties = new KustoIngestionProperties(options.Database, options.Table) { Format = DataSourceFormat.csv }; StreamSourceOptions sourceOptions = new StreamSourceOptions { SourceId = Guid.NewGuid() }; if (!options.IsDryRun) { IKustoIngestionResult result = await client.IngestFromStreamAsync(csv, properties, sourceOptions); IngestionStatus ingestionStatus = result.GetIngestionStatusBySourceId(sourceOptions.SourceId); for (int i = 0; i < 10 && ingestionStatus.Status == Status.Pending; i++) { await Task.Delay(TimeSpan.FromSeconds(30)); ingestionStatus = result.GetIngestionStatusBySourceId(sourceOptions.SourceId); } if (ingestionStatus.Status != Status.Succeeded) { throw new InvalidOperationException( $"Failed to ingest Kusto data.{Environment.NewLine}{ingestionStatus.Details}"); } else if (ingestionStatus.Status == Status.Pending) { throw new InvalidOperationException($"Timeout while ingesting Kusto data."); } } } }
private async Task <IEnumerable <IngestionStatus> > KustoIngestAsync(IReadOnlyList <T> rows) { Contract.Requires(rows.Count > 0); using var stream = new MemoryStream(); using var writer = new StreamWriter(stream, encoding: Encoding.UTF8); foreach (var row in rows) { await writer.WriteLineAsync(JsonConvert.SerializeObject(row)); } await writer.FlushAsync(); stream.Seek(0, SeekOrigin.Begin); var ingestion = await _kustoIngestClient.IngestFromStreamAsync(stream, _kustoIngestionProperties, new StreamSourceOptions() { LeaveOpen = true, }); return(ingestion.GetIngestionStatusCollection()); }
public static async Task WriteDataToKustoInMemoryAsync <T>( IKustoIngestClient client, string databaseName, string tableName, ILogger logger, IEnumerable <T> data, Func <T, IList <KustoValue> > mapFunc) { CsvColumnMapping[] mappings = null; int size = 5; using (var stream = new MemoryStream()) { using (var writer = new StreamWriter(stream, new UTF8Encoding(false), 1024, leaveOpen: true)) { foreach (T d in data) { var dataList = new List <string>(size); if (mappings == null) { var mapList = new List <CsvColumnMapping>(); foreach (KustoValue p in mapFunc(d)) { mapList.Add(new CsvColumnMapping { ColumnName = p.Column, CslDataType = p.DataType }); dataList.Add(p.StringValue); } mappings = mapList.ToArray(); size = mappings.Length; } else { dataList.AddRange(mapFunc(d).Select(p => p.StringValue)); } await writer.WriteCsvLineAsync(dataList); } } if (mappings == null) { logger.LogInformation("No rows to upload."); return; } for (int i = 0; i < mappings.Length; i++) { mappings[i].Ordinal = i; } stream.Seek(0, SeekOrigin.Begin); logger.LogInformation($"Ingesting {mappings.Length} columns at {stream.Length} bytes..."); await client.IngestFromStreamAsync( stream, new KustoQueuedIngestionProperties(databaseName, tableName) { Format = DataSourceFormat.csv, ReportLevel = IngestionReportLevel.FailuresOnly, ReportMethod = IngestionReportMethod.Queue, CSVMapping = mappings }); logger.LogTrace("Ingest complete"); } }
public async Task WriteRow(string message) { await _kustoClient.IngestFromStreamAsync(GenerateStreamFromString(message), _kustoProperties).ConfigureAwait(false); }