public async Task AssertPropertiesConfigured() { var ingest = new Mock <IKustoIngestClient>(); KustoIngestionProperties props = null; ingest.Setup(i => i.IngestFromStreamAsync( It.IsAny <Stream>(), It.IsAny <KustoIngestionProperties>(), It.IsAny <StreamSourceOptions>())) .Callback((Stream s, KustoIngestionProperties p, StreamSourceOptions o) => { props = p; }) .Returns(Task.FromResult(Mock.Of <IKustoIngestionResult>())); await KustoHelpers.WriteDataToKustoInMemoryAsync(ingest.Object, "TEST-DATABASE", "TEST-TABLE", new NUnitLogger(), new[] { 1 }, v => new[] { new KustoValue("columnName", 1, KustoDataType.Int), } ); props.Should().NotBeNull(); props.DatabaseName.Should().Be("TEST-DATABASE"); props.TableName.Should().Be("TEST-TABLE"); }
static void Main(string[] args) { // Ingest From Local Files using KustoQueuedIngestClient and Ingestion Validation // Create Kusto connection string with App Authentication var kustoConnectionStringBuilderDM = new KustoConnectionStringBuilder(@"https://ingest-{clusterNameAndRegion}.kusto.windows.net").WithAadApplicationKeyAuthentication( applicationClientId: "{Application Client ID}", applicationKey: "{Application Key (secret)}", authority: "{AAD TenantID or name}"); // Create a disposable client that will execute the ingestion IKustoQueuedIngestClient client = KustoIngestFactory.CreateQueuedIngestClient(kustoConnectionStringBuilderDM); // Ingest from files according to the required properties var kustoIngestionProperties = new KustoIngestionProperties(databaseName: "myDB", tableName: "myTable"); client.IngestFromStorageAsync(@"ValidTestFile.csv", kustoIngestionProperties); client.IngestFromStorageAsync(@"InvalidTestFile.csv", kustoIngestionProperties); // Waiting for the aggregation Thread.Sleep(TimeSpan.FromMinutes(8)); // Retrieve and validate failures var ingestionFailures = client.PeekTopIngestionFailures().GetAwaiter().GetResult(); Ensure.IsTrue((ingestionFailures.Count() > 0), "Failures expected"); // Retrieve, delete and validate failures ingestionFailures = client.GetAndDiscardTopIngestionFailures().GetAwaiter().GetResult(); Ensure.IsTrue((ingestionFailures.Count() > 0), "Failures expected"); // Dispose of the client client.Dispose(); }
private KustoIngestionProperties GetDirectModelProperties(Type eventType) { lock (_gate) { KustoIngestionProperties ingestProps; if (!KustoDirectMappings.ContainsKey(eventType)) { ingestProps = new KustoIngestionProperties(KustoDbName, "Unknown") { TableName = KustoAdminClient.GenerateTableFromType(eventType).Result, JSONMappingReference = KustoAdminClient.GenerateTableJsonMappingFromType(eventType).Result, IgnoreSizeLimit = true, ValidationPolicy = null, Format = DataSourceFormat.json }; KustoDirectMappings.Add(eventType, ingestProps); } else { ingestProps = KustoDirectMappings[eventType]; } return(ingestProps); } }
static async Task Main(string[] args) { // Async Ingestion From a Single Azure Blob using KustoQueuedIngestClient with (optional) RetryPolicy: //Create Kusto connection string with App Authentication var kustoConnectionStringBuilderDM = new KustoConnectionStringBuilder(@"https://ingest-{clusterNameAndRegion}.kusto.windows.net").WithAadApplicationKeyAuthentication( applicationClientId: "{Application Client ID}", applicationKey: "{Application Key (secret)}", authority: "{AAD TenantID or name}"); // Create an ingest client // Note, that creating a separate instance per ingestion operation is an anti-pattern. // IngestClient classes are thread-safe and intended for reuse IKustoIngestClient client = KustoIngestFactory.CreateQueuedIngestClient(kustoConnectionStringBuilderDM); // Ingest from blobs according to the required properties var kustoIngestionProperties = new KustoIngestionProperties(databaseName: "myDB", tableName: "myTable"); var sourceOptions = new StorageSourceOptions() { DeleteSourceOnSuccess = true }; //// Create your custom implementation of IRetryPolicy, which will affect how the ingest client handles retrying on transient failures IRetryPolicy retryPolicy = new NoRetry(); //// This line sets the retry policy on the ingest client that will be enforced on every ingest call from here on ((IKustoQueuedIngestClient)client).QueueRetryPolicy = retryPolicy; await client.IngestFromStorageAsync(uri : @"BLOB-URI-WITH-SAS-KEY", ingestionProperties : kustoIngestionProperties, sourceOptions); client.Dispose(); }
public KustoClientLogger(string databaseName, string tableName, List <KustoColumnMapping> _schema, string clientId, string clientKey) { var csvColumnMappings = new List <CsvColumnMapping>(); foreach (var mapping in _schema) { csvColumnMappings.Add(new CsvColumnMapping { ColumnName = mapping.ColumnName, CslDataType = mapping.DataType, Ordinal = mapping.ColumnNumber }); } _kustoProperties = new KustoIngestionProperties(databaseName, tableName) { Format = DataSourceFormat.csv, CSVMapping = csvColumnMappings }; var kustoConnectionStringBuilderDM = new KustoConnectionStringBuilder(@"https://wawstest.kusto.windows.net:443") { FederatedSecurity = true, InitialCatalog = databaseName, ApplicationKey = clientKey, ApplicationClientId = clientId }; _kustoClient = KustoIngestFactory.CreateDirectIngestClient(kustoConnectionStringBuilderDM); }
public BlockingKustoUploader( KustoConnectionStringBuilder kscb, string tableName, int batchSize, TimeSpan flushDuration) { csb = kscb; TableName = TableName; BatchSize = batchSize; _flushDuration = flushDuration; _lastUploadTime = DateTime.Now; Completed = new AutoResetEvent(false); _ingestionProperties = new KustoIngestionProperties(csb.InitialCatalog, tableName); _fields = typeof(EtwEvent).GetFields().Select(f => f.Name).ToArray(); if (csb.DataSource.StartsWith("https://ingest-")) { _ingestClient = KustoIngestFactory.CreateQueuedIngestClient(csb); } else { _ingestClient = KustoIngestFactory.CreateDirectIngestClient(csb); } _nextBatch = new List <T>(); }
private static Task TriggerIngestCommand(DateTime insertDate, string urlOfToBeInsertedBlob, long rawDatasize) { var ingestProperties = new KustoIngestionProperties(GetEnvVariable("KustoDatabase"), GetEnvVariable("KustoTableName")); switch (GetEnvVariable("KustoMappingType")) { case "json": ingestProperties.JSONMappingReference = GetEnvVariable("KustoMappingRef"); break; case "csv": ingestProperties.CSVMappingReference = GetEnvVariable("KustoMappingRef"); break; case "avro": ingestProperties.AvroMappingReference = GetEnvVariable("KustoMappingRef"); break; default: ingestProperties.JSONMappingReference = GetEnvVariable("KustoMappingRef"); break; } ingestProperties.AdditionalProperties.Add(CREATIONDATEKEY, insertDate.ToString()); ingestProperties.AdditionalTags = new List <String> { insertDate.ToString() }; StorageSourceOptions sourceOptions = new StorageSourceOptions(); sourceOptions.Size = rawDatasize; sourceOptions.DeleteSourceOnSuccess = Boolean.Parse(GetEnvVariable("DeleteAfterInsert")); return(adx.IngestFromStorageAsync(urlOfToBeInsertedBlob + Environment.GetEnvironmentVariable("SasToken"), ingestProperties, sourceOptions)); }
public AdxOutput( BaseLogger logger, string authority, string appclientId, string appKey, string cluster, string database, string table, bool createOrResetTable = false, bool directIngest = false) { _logger = logger; _table = table; _createOrResetTable = createOrResetTable; _batchSize = 10000; _flushDuration = TimeSpan.FromMilliseconds(5); _lastUploadTime = DateTime.UtcNow; _initializeTable = false; _nextBatch = new List <IDictionary <string, object> >(); Completed = new AutoResetEvent(false); // Setting up kusto connection if (!string.IsNullOrEmpty(authority)) { if (!string.IsNullOrEmpty(appclientId) && !string.IsNullOrEmpty(appKey)) { kscbIngest = new KustoConnectionStringBuilder($"https://ingest-{cluster}", database).WithAadApplicationKeyAuthentication(appclientId, appKey, authority); kscbAdmin = new KustoConnectionStringBuilder($"https://{cluster}", database).WithAadApplicationKeyAuthentication(appclientId, appKey, authority); } else { kscbIngest = new KustoConnectionStringBuilder($"https://ingest-{cluster}", database).WithAadUserPromptAuthentication(authority); kscbAdmin = new KustoConnectionStringBuilder($"https://{cluster}", database).WithAadUserPromptAuthentication(authority); } } if (kscbAdmin != null) { _ingestionProperties = new KustoIngestionProperties(kscbIngest.InitialCatalog, table); if (directIngest) { _ingestClient = KustoIngestFactory.CreateDirectIngestClient(this.kscbAdmin); } else { _ingestClient = KustoIngestFactory.CreateQueuedIngestClient(kscbIngest); } } else { _logger.Log(LogLevel.ERROR, "ERROR getting ADX connection strings. Please double check the information provided."); _error = true; } }
protected FI(ILogger iLog, Microsoft.Azure.Management.Fluent.IAzure iAzure, string iRGName, string kustoConn, string kustoDBName, string kustoTableName) { curRGName = iRGName; log = iLog; curSubName = iAzure.SubscriptionId; ingestClient = KustoIngestFactory.CreateQueuedIngestClient(kustoConn); ingestProps = new KustoIngestionProperties(kustoDBName, kustoTableName); ingestProps.Format = Kusto.Data.Common.DataSourceFormat.csv; }
private async Task <IKustoIngestionResult> IngestFromStreamAsync( string csv, IKustoIngestClient client, KustoIngestionProperties properties, StreamSourceOptions sourceOptions) { using MemoryStream stream = new(); using StreamWriter writer = new(stream); writer.Write(csv); writer.Flush(); stream.Seek(0, SeekOrigin.Begin); return(await client.IngestFromStreamAsync(stream, properties, sourceOptions)); }
public IKustoClusterBuilder UseKusto() { return(new KustoOptionsBuilder(builder => { _kustoOptionsBuilder = builder; UseKustoInternal(builder.DbProperties.Engine, builder.DbProperties.Region, builder.DbProperties.DbName, builder.DbProperties.ClientId); foreach (var type in _kustoOptionsBuilder.RegisteredDirectTypes) { if (KustoDirectMappings.ContainsKey(type)) { continue; } var tableName = KustoAdminClient.GenerateTableFromType(type).Result; var ingestProps = new KustoIngestionProperties(KustoDbName, tableName) { TableName = tableName, JSONMappingReference = KustoAdminClient.GenerateTableJsonMappingFromType(type).Result, IgnoreSizeLimit = true, ValidationPolicy = null, Format = DataSourceFormat.json }; KustoDirectMappings.Add(type, ingestProps); } foreach (var type in _kustoOptionsBuilder.RegisteredQueuedTypes) { if (KustoQueuedMappings.ContainsKey(type)) { continue; } var tableName = KustoAdminClient.GenerateTableFromType(type).Result; var ingestProps = new KustoQueuedIngestionProperties(KustoDbName, tableName) { TableName = tableName, JSONMappingReference = KustoAdminClient.GenerateTableJsonMappingFromType(type).Result, ReportLevel = IngestionReportLevel.FailuresOnly, ReportMethod = IngestionReportMethod.Queue, FlushImmediately = _kustoOptionsBuilder?.BufferOptions.FlushImmediately ?? true, IgnoreSizeLimit = true, ValidationPolicy = null, Format = DataSourceFormat.json }; KustoQueuedMappings.Add(type, ingestProps); } })); }
/// <summary> /// Ingest data into Kusto. /// </summary> /// <param name="table">Name of table to ingest into.</param> /// <param name="mappingName">Name of table mapping to ingest with.</param> /// <param name="stream">input JSON data stream.</param> /// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns> private static async Task <IKustoIngestionResult> KustoIngest(KustoConnectionStringBuilder kusto, string db, string table, string mappingName, Stream stream) { // Create a disposable client that will execute the ingestion using IKustoIngestClient client = KustoIngestFactory.CreateDirectIngestClient(kusto); var ingestProps = new KustoIngestionProperties(db, table) { JSONMappingReference = mappingName, Format = DataSourceFormat.json, }; var ssOptions = new StreamSourceOptions { CompressionType = DataSourceCompressionType.GZip, }; return(await client.IngestFromStreamAsync(stream, ingestProps, ssOptions)); }
public BlockingKustoUploader( string blobConnectionString, string blobContainerName, KustoConnectionStringBuilder adminCsb, KustoConnectionStringBuilder kscb, bool demoMode, string tableName, int batchSize, TimeSpan flushDuration, bool resetTable = false) { Csb = kscb; AdminCsb = adminCsb; TableName = tableName; BatchSize = batchSize; _flushDuration = flushDuration; _lastUploadTime = DateTime.UtcNow; _resetTable = resetTable; _initializeTable = false; Completed = new AutoResetEvent(false); if (Csb != null) { _ingestionProperties = new KustoIngestionProperties(Csb.InitialCatalog, tableName); if (demoMode) { _ingestClient = KustoIngestFactory.CreateDirectIngestClient(this.AdminCsb); } else { _ingestClient = KustoIngestFactory.CreateQueuedIngestClient(Csb); } } if (!string.IsNullOrEmpty(blobConnectionString) && !string.IsNullOrEmpty(blobContainerName)) { blobServiceClient = new BlobServiceClient(blobConnectionString); blobContainerClient = blobServiceClient.GetBlobContainerClient(blobContainerName); blobContainerClient.CreateIfNotExists(); } _nextBatch = new List <IDictionary <string, object> >(); }
public async Task PublishManualReviews(IEnumerable <SloManualReview> results) { var dt = results.ToDataTable(); var kustoConnectionStringBuilderEngine = new KustoConnectionStringBuilder(kustoUrl_).WithAadUserPromptAuthentication(); using (var client = KustoIngestFactory.CreateDirectIngestClient(kustoConnectionStringBuilderEngine)) { //Ingest from blobs according to the required properties var kustoIngestionProperties = new KustoIngestionProperties( databaseName: kustoDb_, tableName: kustoManualReviewTable_ ); var reader = dt.CreateDataReader(); var result = await client.IngestFromDataReaderAsync(reader, kustoIngestionProperties); } }
public async Task IngestFromCsvStreamAsync(Stream csv, IngestKustoImageInfoOptions options) { KustoConnectionStringBuilder connectionBuilder = new KustoConnectionStringBuilder($"https://{options.Cluster}.kusto.windows.net") .WithAadApplicationKeyAuthentication( options.ServicePrincipal.ClientId, options.ServicePrincipal.Secret, options.ServicePrincipal.Tenant); using (IKustoIngestClient client = KustoIngestFactory.CreateDirectIngestClient(connectionBuilder)) { KustoIngestionProperties properties = new KustoIngestionProperties(options.Database, options.Table) { Format = DataSourceFormat.csv }; StreamSourceOptions sourceOptions = new StreamSourceOptions { SourceId = Guid.NewGuid() }; if (!options.IsDryRun) { IKustoIngestionResult result = await client.IngestFromStreamAsync(csv, properties, sourceOptions); IngestionStatus ingestionStatus = result.GetIngestionStatusBySourceId(sourceOptions.SourceId); for (int i = 0; i < 10 && ingestionStatus.Status == Status.Pending; i++) { await Task.Delay(TimeSpan.FromSeconds(30)); ingestionStatus = result.GetIngestionStatusBySourceId(sourceOptions.SourceId); } if (ingestionStatus.Status != Status.Succeeded) { throw new InvalidOperationException( $"Failed to ingest Kusto data.{Environment.NewLine}{ingestionStatus.Details}"); } else if (ingestionStatus.Status == Status.Pending) { throw new InvalidOperationException($"Timeout while ingesting Kusto data."); } } } }
public BlockingKustoUploader( string outputFileName, KustoConnectionStringBuilder adminCsb, KustoConnectionStringBuilder kscb, bool demoMode, string tableName, int batchSize, TimeSpan flushDuration, bool resetTable = false) { OutputFileName = outputFileName; Csb = kscb; AdminCsb = adminCsb; TableName = tableName; BatchSize = batchSize; _flushDuration = flushDuration; _lastUploadTime = DateTime.UtcNow; _resetTable = resetTable; _initializeTable = false; Completed = new AutoResetEvent(false); if (Csb != null) { _ingestionProperties = new KustoIngestionProperties(Csb.InitialCatalog, tableName); if (demoMode) { _ingestClient = KustoIngestFactory.CreateDirectIngestClient(this.AdminCsb); } else { _ingestClient = KustoIngestFactory.CreateQueuedIngestClient(Csb); } } if (!string.IsNullOrEmpty(OutputFileName)) { outputFile = new StreamWriter(this.OutputFileName); outputFile.Write($"[{Environment.NewLine}"); } _nextBatch = new List <IDictionary <string, object> >(); }
static void Main(string[] args) { // Ingest From Local File using KustoDirectIngestClient (only for test purposes): // Create Kusto connection string with App Authentication var kustoConnectionStringBuilderEngine = new KustoConnectionStringBuilder(@"https://{clusterNameAndRegion}.kusto.windows.net").WithAadApplicationKeyAuthentication( applicationClientId: "{Application Client ID}", applicationKey: "{Application Key (secret)}", authority: "{AAD TenantID or name}"); // Create a disposable client that will execute the ingestion using (IKustoIngestClient client = KustoIngestFactory.CreateDirectIngestClient(kustoConnectionStringBuilderEngine)) { //Ingest from blobs according to the required properties var kustoIngestionProperties = new KustoIngestionProperties(databaseName: "myDB", tableName: "myTable"); client.IngestFromStorageAsync(@"< Path to local file >", ingestionProperties: kustoIngestionProperties).GetAwaiter().GetResult(); } }
public KustoNotifier(Configuration configuration, ILogger logger, IKustoIngestClient kustoIngestClient) { _configuration = configuration; _logger = logger; _kustoIngestClient = kustoIngestClient; _kustoIngestionProperties = new KustoIngestionProperties(_configuration.KustoDatabaseName, _configuration.KustoTableName) { Format = DataSourceFormat.json, }; Contract.RequiresNotNullOrEmpty(_configuration.KustoTableIngestionMappingName, "Kusto ingestion will fail to authenticate without a proper ingestion mapping."); _kustoIngestionProperties.JSONMappingReference = _configuration.KustoTableIngestionMappingName; _queue = NagleQueue <T> .Create(FlushAsync, _configuration.MaxDegreeOfParallelism, _configuration.FlushInterval, _configuration.BatchSize); }
static void Main(string[] args) { if (!ParseCommandLine(args, out var cluster, out var database, out var table)) { Usage(); return; } var kcsb = new KustoConnectionStringBuilder { DataSource = cluster }; kcsb = kcsb.WithAadUserPromptAuthentication(); CreateJsonMappingIfNotExists(kcsb, database, table); // Do ingestion using Kusto.Data client library using (var siClient = KustoClientFactory.CreateCslStreamIngestClient(kcsb)) { using (var data = CreateSampleEventLogCsvStream(10)) { siClient.ExecuteStreamIngest(database, table, data); } using (var data = CreateSampleEventLogJsonStream(10)) { siClient.ExecuteStreamIngestAsync( database, table, data, null, Kusto.Data.Common.DataSourceFormat.json, compressStream: false, mappingName: s_jsonMappingName); } } // Do ingestion using Kusto.Ingest client library. The data still goes directly to the engine cluster // Just a convenience for applications already using IKustoIngest interface using (var ingestClient = KustoIngestFactory.CreateStreamingIngestClient(kcsb)) { using (var data = CreateSampleEventLogCsvStream(10)) { var ingestProperties = new KustoIngestionProperties(database, table) { Format = DataSourceFormat.csv, }; ingestClient.IngestFromStreamAsync(data, ingestProperties); } using (var data = CreateSampleEventLogJsonStream(10)) { var ingestProperties = new KustoIngestionProperties(database, table) { Format = DataSourceFormat.json, JSONMappingReference = s_jsonMappingName }; ingestClient.IngestFromStreamAsync(data, ingestProperties); } } }
public Task <IKustoIngestionResult> IngestFromStreamAsync(Stream stream, KustoIngestionProperties ingestionProperties, StreamSourceOptions sourceOptions = null) { return(_successfulResultInstance); }
static void Main(string[] args) { var kcsb = new KustoConnectionStringBuilder(); kcsb.DataSource = "https://kustolab.kusto.windows.net"; kcsb.FederatedSecurity = true; string databaseName = "StreamingIngestionSample"; string tableName = "EventLog"; CreateJsonMappingIfNotExists(kcsb, databaseName, tableName); // Do ingestion using Kusto.Data client library using (var siClient = KustoClientFactory.CreateCslStreamIngestClient(kcsb)) { using (var data = CreateSampleEventLogCsvStream(10)) { siClient.ExecuteStreamIngestAsync( databaseName, tableName, data, null, DataSourceFormat.csv); } using (var data = CreateSampleEventLogJsonStream(10)) { siClient.ExecuteStreamIngestAsync( databaseName, tableName, data, null, DataSourceFormat.json, compressStream: false, mappingName: s_jsonMappingName).ResultEx(); } } // Do ingestion using Kusto.Ingest client library. The data still goes directly to the engine cluster // Just a convenience for applications already using IKustoIngest interface using (var ingestClient = KustoIngestFactory.CreateStreamingIngestClient(kcsb)) { using (var data = CreateSampleEventLogCsvStream(10)) { var ingestProperties = new KustoIngestionProperties(databaseName, tableName) { Format = DataSourceFormat.csv, }; ingestClient.IngestFromStreamAsync(data, ingestProperties).ResultEx(); } using (var data = CreateSampleEventLogJsonStream(10)) { var ingestProperties = new KustoIngestionProperties(databaseName, tableName) { Format = DataSourceFormat.json, IngestionMapping = new IngestionMapping { IngestionMappingKind = Kusto.Data.Ingestion.IngestionMappingKind.Json, IngestionMappingReference = s_jsonMappingName } }; ingestClient.IngestFromStream(data, ingestProperties); } } }
public Task <IKustoIngestionResult> IngestFromStorageAsync(string uri, KustoIngestionProperties ingestionProperties, StorageSourceOptions sourceOptions = null) { return(_successfulResultInstance); }
public ActionResult Upload(string instanceUrl, string databaseName, string tenantId, string clientId, string secret) { ADXModel model = new ADXModel { InstanceUrl = instanceUrl, DatabaseName = databaseName, TenantId = tenantId, ClientId = clientId, Secret = secret, }; try { var kustoConnectionStringBuilder = new KustoConnectionStringBuilder(instanceUrl).WithAadApplicationKeyAuthentication(clientId, secret, tenantId); string tableName = DTDL._nodesetNamespaceURI.Replace("http://", "").Replace('/', '_').Replace('.', '_').TrimEnd('_'); using (var kustoClient = KustoClientFactory.CreateCslAdminProvider(kustoConnectionStringBuilder)) { kustoClient.ExecuteControlCommand(databaseName, ".create table " + tableName + " (ExpandedNodeID: string, DisplayName: string, Type: string, ParentNodeID: string)"); string command = ".create table " + tableName + " ingestion json mapping 'Nodeset_Mapping' '[" + "{ \"properties\": { \"path\": \"$.Key\" }, \"column\": \"ExpandedNodeID\", \"datatype\": \"string\" }," + "{ \"properties\": { \"path\": \"$.Value.Item1\" }, \"column\": \"DisplayName\", \"datatype\": \"string\" } ," + "{ \"properties\": { \"path\": \"$.Value.Item2\" }, \"column\": \"Type\", \"datatype\": \"string\" }," + "{ \"properties\": { \"path\": \"$.Value.Item3\" }, \"column\": \"ParentNodeID\", \"datatype\": \"string\" } ]'"; kustoClient.ExecuteControlCommand(databaseName, command); command = ".alter table " + tableName + " policy ingestionbatching @'{" + "\"MaximumBatchingTimeSpan\": \"00:00:05\"," + "\"MaximumNumberOfItems\": 100," + "\"MaximumRawDataSizeMB\": 1024 }'"; kustoClient.ExecuteControlCommand(databaseName, command); } using (IKustoIngestClient client = KustoIngestFactory.CreateDirectIngestClient(kustoConnectionStringBuilder)) { var kustoIngestionProperties = new KustoIngestionProperties(databaseName, tableName); kustoIngestionProperties.Format = DataSourceFormat.multijson; kustoIngestionProperties.IngestionMapping = new IngestionMapping() { IngestionMappingReference = "Nodeset_Mapping", IngestionMappingKind = IngestionMappingKind.Json }; foreach (KeyValuePair <string, Tuple <string, string, string> > entry in DTDL._nodeList.ToArray(100)) { string content = JsonConvert.SerializeObject(entry, Formatting.Indented); MemoryStream stream = new MemoryStream(Encoding.UTF8.GetBytes(content)); client.IngestFromStream(stream, kustoIngestionProperties); stream.Dispose(); } } using (var cslQueryProvider = KustoClientFactory.CreateCslQueryProvider(kustoConnectionStringBuilder)) { string query = $"{tableName} | count"; var results = cslQueryProvider.ExecuteQuery <long>(databaseName, query); foreach (long result in results) { model.StatusMessage = "ADX Ingestion succeeded."; return(View("Index", model)); } } model.StatusMessage = "ADX Ingestion failed!"; return(View("Error", model)); } catch (Exception ex) { model.StatusMessage = ex.Message; return(View("Index", model)); } }
static void IngestData(KustoConnectionStringBuilder engineKustoConnectionStringBuilder, KustoConnectionStringBuilder dmKustoConnectionStringBuilderDM, string databaseName, string tableName) { // 1. Ingest by connecting directly to the Kustolab cluster and sending a command using (IKustoIngestClient directClient = KustoIngestFactory.CreateDirectIngestClient(engineKustoConnectionStringBuilder)) { var kustoIngestionProperties = new KustoIngestionProperties(databaseName, tableName); directClient.IngestFromDataReaderAsync(GetDataAsIDataReader(), kustoIngestionProperties); } // 2. Ingest by submitting the data to the Kustolab ingestion cluster // Note that this is an async operation, so data might not appear immediately using (IKustoIngestClient queuedClient = KustoIngestFactory.CreateQueuedIngestClient(dmKustoConnectionStringBuilderDM)) { var kustoIngestionProperties = new KustoIngestionProperties(databaseName, tableName); queuedClient.IngestFromDataReaderAsync(GetDataAsIDataReader(), kustoIngestionProperties); } // 3. Ingest by submitting the data to the Kustolab ingestion cluster - // This time, update the report method and level so you can track the status of your ingestion // using the IKustoIngestionResult returned from the ingest operation. IKustoIngestionResult ingestionResult; using (IKustoIngestClient queuedClient = KustoIngestFactory.CreateQueuedIngestClient(dmKustoConnectionStringBuilderDM)) { var kustoIngestionProperties = new KustoQueuedIngestionProperties(databaseName, tableName) { // The default ReportLevel is set to FailuresOnly. // In this case we want to check the status of successful ingestions as well. ReportLevel = IngestionReportLevel.FailuresAndSuccesses, // You can use either a queue or a table to track the status of your ingestion. ReportMethod = IngestionReportMethod.Table }; ingestionResult = queuedClient.IngestFromDataReaderAsync(GetDataAsIDataReader(), kustoIngestionProperties).Result; } // Obtain the status of our ingestion var ingestionStatus = ingestionResult.GetIngestionStatusCollection().First(); var watch = System.Diagnostics.Stopwatch.StartNew(); var shouldContinue = true; while ((ingestionStatus.Status == Status.Pending) && (shouldContinue)) { // Wait a minute... Thread.Sleep(TimeSpan.FromMinutes(1)); // Try again ingestionStatus = ingestionResult.GetIngestionStatusBySourceId(ingestionStatus.IngestionSourceId); shouldContinue = watch.ElapsedMilliseconds < TimeOutInMilliSeconds; } watch.Stop(); if (ingestionStatus.Status == Status.Pending) { // The status of the ingestion did not change. Console.WriteLine( "Ingestion with ID:'{0}' did not complete. Timed out after :'{1}' Milliseconds" .FormatWithInvariantCulture( ingestionStatus.IngestionSourceId, watch.ElapsedMilliseconds)); } else { // The status of the ingestion has changed Console.WriteLine( "Ingestion with ID:'{0}' is complete. Ingestion Status:'{1}'".FormatWithInvariantCulture( ingestionStatus.IngestionSourceId, ingestionStatus.Status)); } // 4. Show the contents of the table using (var client = Kusto.Data.Net.Client.KustoClientFactory.CreateCslQueryProvider(engineKustoConnectionStringBuilder)) { while (true) { var query = string.Format("{0}", tableName); var reader = client.ExecuteQuery(query); Kusto.Cloud.Platform.Data.ExtendedDataReader.WriteAsText(reader, "Data ingested into the table:", tabify: true, firstOnly: true); Console.WriteLine("Press 'r' to retry retrieving data from the table, any other key to quit"); var key = Console.ReadKey(); if (key.KeyChar != 'r' && key.KeyChar != 'R') { break; } Console.WriteLine(); } } }
public Task <IKustoIngestionResult> IngestFromDataReaderAsync(IDataReader dataReader, KustoIngestionProperties ingestionProperties, DataReaderSourceOptions sourceOptions = null) { return(_successfulResultInstance); }