public async Task AssertPropertiesConfigured()
{
var ingest = new Mock <IKustoIngestClient>();
KustoIngestionProperties props = null;
ingest.Setup(i => i.IngestFromStreamAsync(
It.IsAny <Stream>(),
It.IsAny <KustoIngestionProperties>(),
It.IsAny <StreamSourceOptions>()))
.Callback((Stream s, KustoIngestionProperties p, StreamSourceOptions o) => { props = p; })
.Returns(Task.FromResult(Mock.Of <IKustoIngestionResult>()));
await KustoHelpers.WriteDataToKustoInMemoryAsync(ingest.Object,
"TEST-DATABASE",
"TEST-TABLE",
new NUnitLogger(),
new[] { 1 },
v => new[]
{
new KustoValue("columnName", 1, KustoDataType.Int),
}
);
props.Should().NotBeNull();
props.DatabaseName.Should().Be("TEST-DATABASE");
props.TableName.Should().Be("TEST-TABLE");
}
static void Main(string[] args)
{
// Ingest From Local Files using KustoQueuedIngestClient and Ingestion Validation
// Create Kusto connection string with App Authentication
var kustoConnectionStringBuilderDM =
new KustoConnectionStringBuilder(@"https://ingest-{clusterNameAndRegion}.kusto.windows.net").WithAadApplicationKeyAuthentication(
applicationClientId: "{Application Client ID}",
applicationKey: "{Application Key (secret)}",
authority: "{AAD TenantID or name}");
// Create a disposable client that will execute the ingestion
IKustoQueuedIngestClient client = KustoIngestFactory.CreateQueuedIngestClient(kustoConnectionStringBuilderDM);
// Ingest from files according to the required properties
var kustoIngestionProperties = new KustoIngestionProperties(databaseName: "myDB", tableName: "myTable");
client.IngestFromStorageAsync(@"ValidTestFile.csv", kustoIngestionProperties);
client.IngestFromStorageAsync(@"InvalidTestFile.csv", kustoIngestionProperties);
// Waiting for the aggregation
Thread.Sleep(TimeSpan.FromMinutes(8));
// Retrieve and validate failures
var ingestionFailures = client.PeekTopIngestionFailures().GetAwaiter().GetResult();
Ensure.IsTrue((ingestionFailures.Count() > 0), "Failures expected");
// Retrieve, delete and validate failures
ingestionFailures = client.GetAndDiscardTopIngestionFailures().GetAwaiter().GetResult();
Ensure.IsTrue((ingestionFailures.Count() > 0), "Failures expected");
// Dispose of the client
client.Dispose();
}
private KustoIngestionProperties GetDirectModelProperties(Type eventType)
{
lock (_gate)
{
KustoIngestionProperties ingestProps;
if (!KustoDirectMappings.ContainsKey(eventType))
{
ingestProps = new KustoIngestionProperties(KustoDbName, "Unknown")
{
TableName = KustoAdminClient.GenerateTableFromType(eventType).Result,
JSONMappingReference = KustoAdminClient.GenerateTableJsonMappingFromType(eventType).Result,
IgnoreSizeLimit = true,
ValidationPolicy = null,
Format = DataSourceFormat.json
};
KustoDirectMappings.Add(eventType, ingestProps);
}
else
{
ingestProps = KustoDirectMappings[eventType];
}
return(ingestProps);
}
}
static async Task Main(string[] args)
{
// Async Ingestion From a Single Azure Blob using KustoQueuedIngestClient with (optional) RetryPolicy:
//Create Kusto connection string with App Authentication
var kustoConnectionStringBuilderDM =
new KustoConnectionStringBuilder(@"https://ingest-{clusterNameAndRegion}.kusto.windows.net").WithAadApplicationKeyAuthentication(
applicationClientId: "{Application Client ID}",
applicationKey: "{Application Key (secret)}",
authority: "{AAD TenantID or name}");
// Create an ingest client
// Note, that creating a separate instance per ingestion operation is an anti-pattern.
// IngestClient classes are thread-safe and intended for reuse
IKustoIngestClient client = KustoIngestFactory.CreateQueuedIngestClient(kustoConnectionStringBuilderDM);
// Ingest from blobs according to the required properties
var kustoIngestionProperties = new KustoIngestionProperties(databaseName: "myDB", tableName: "myTable");
var sourceOptions = new StorageSourceOptions()
{
DeleteSourceOnSuccess = true
};
//// Create your custom implementation of IRetryPolicy, which will affect how the ingest client handles retrying on transient failures
IRetryPolicy retryPolicy = new NoRetry();
//// This line sets the retry policy on the ingest client that will be enforced on every ingest call from here on
((IKustoQueuedIngestClient)client).QueueRetryPolicy = retryPolicy;
await client.IngestFromStorageAsync(uri : @"BLOB-URI-WITH-SAS-KEY", ingestionProperties : kustoIngestionProperties, sourceOptions);
client.Dispose();
}
public KustoClientLogger(string databaseName, string tableName, List <KustoColumnMapping> _schema, string clientId, string clientKey)
{
var csvColumnMappings = new List <CsvColumnMapping>();
foreach (var mapping in _schema)
{
csvColumnMappings.Add(new CsvColumnMapping {
ColumnName = mapping.ColumnName, CslDataType = mapping.DataType, Ordinal = mapping.ColumnNumber
});
}
_kustoProperties = new KustoIngestionProperties(databaseName, tableName)
{
Format = DataSourceFormat.csv,
CSVMapping = csvColumnMappings
};
var kustoConnectionStringBuilderDM = new KustoConnectionStringBuilder(@"https://wawstest.kusto.windows.net:443")
{
FederatedSecurity = true,
InitialCatalog = databaseName,
ApplicationKey = clientKey,
ApplicationClientId = clientId
};
_kustoClient = KustoIngestFactory.CreateDirectIngestClient(kustoConnectionStringBuilderDM);
}
public BlockingKustoUploader(
KustoConnectionStringBuilder kscb,
string tableName,
int batchSize,
TimeSpan flushDuration)
{
csb = kscb;
TableName = TableName;
BatchSize = batchSize;
_flushDuration = flushDuration;
_lastUploadTime = DateTime.Now;
Completed = new AutoResetEvent(false);
_ingestionProperties = new KustoIngestionProperties(csb.InitialCatalog, tableName);
_fields = typeof(EtwEvent).GetFields().Select(f => f.Name).ToArray();
if (csb.DataSource.StartsWith("https://ingest-"))
{
_ingestClient = KustoIngestFactory.CreateQueuedIngestClient(csb);
}
else
{
_ingestClient = KustoIngestFactory.CreateDirectIngestClient(csb);
}
_nextBatch = new List <T>();
}
private static Task TriggerIngestCommand(DateTime insertDate, string urlOfToBeInsertedBlob, long rawDatasize)
{
var ingestProperties = new KustoIngestionProperties(GetEnvVariable("KustoDatabase"), GetEnvVariable("KustoTableName"));
switch (GetEnvVariable("KustoMappingType"))
{
case "json":
ingestProperties.JSONMappingReference = GetEnvVariable("KustoMappingRef");
break;
case "csv":
ingestProperties.CSVMappingReference = GetEnvVariable("KustoMappingRef");
break;
case "avro":
ingestProperties.AvroMappingReference = GetEnvVariable("KustoMappingRef");
break;
default:
ingestProperties.JSONMappingReference = GetEnvVariable("KustoMappingRef");
break;
}
ingestProperties.AdditionalProperties.Add(CREATIONDATEKEY, insertDate.ToString());
ingestProperties.AdditionalTags = new List <String> {
insertDate.ToString()
};
StorageSourceOptions sourceOptions = new StorageSourceOptions();
sourceOptions.Size = rawDatasize;
sourceOptions.DeleteSourceOnSuccess = Boolean.Parse(GetEnvVariable("DeleteAfterInsert"));
return(adx.IngestFromStorageAsync(urlOfToBeInsertedBlob + Environment.GetEnvironmentVariable("SasToken"), ingestProperties, sourceOptions));
}
public AdxOutput(
BaseLogger logger,
string authority,
string appclientId,
string appKey,
string cluster,
string database,
string table,
bool createOrResetTable = false,
bool directIngest = false)
{
_logger = logger;
_table = table;
_createOrResetTable = createOrResetTable;
_batchSize = 10000;
_flushDuration = TimeSpan.FromMilliseconds(5);
_lastUploadTime = DateTime.UtcNow;
_initializeTable = false;
_nextBatch = new List <IDictionary <string, object> >();
Completed = new AutoResetEvent(false);
// Setting up kusto connection
if (!string.IsNullOrEmpty(authority))
{
if (!string.IsNullOrEmpty(appclientId) && !string.IsNullOrEmpty(appKey))
{
kscbIngest = new KustoConnectionStringBuilder($"https://ingest-{cluster}", database).WithAadApplicationKeyAuthentication(appclientId, appKey, authority);
kscbAdmin = new KustoConnectionStringBuilder($"https://{cluster}", database).WithAadApplicationKeyAuthentication(appclientId, appKey, authority);
}
else
{
kscbIngest = new KustoConnectionStringBuilder($"https://ingest-{cluster}", database).WithAadUserPromptAuthentication(authority);
kscbAdmin = new KustoConnectionStringBuilder($"https://{cluster}", database).WithAadUserPromptAuthentication(authority);
}
}
if (kscbAdmin != null)
{
_ingestionProperties = new KustoIngestionProperties(kscbIngest.InitialCatalog, table);
if (directIngest)
{
_ingestClient = KustoIngestFactory.CreateDirectIngestClient(this.kscbAdmin);
}
else
{
_ingestClient = KustoIngestFactory.CreateQueuedIngestClient(kscbIngest);
}
}
else
{
_logger.Log(LogLevel.ERROR, "ERROR getting ADX connection strings. Please double check the information provided.");
_error = true;
}
}
protected FI(ILogger iLog, Microsoft.Azure.Management.Fluent.IAzure iAzure, string iRGName, string kustoConn, string kustoDBName, string kustoTableName)
{
curRGName = iRGName;
log = iLog;
curSubName = iAzure.SubscriptionId;
ingestClient = KustoIngestFactory.CreateQueuedIngestClient(kustoConn);
ingestProps = new KustoIngestionProperties(kustoDBName, kustoTableName);
ingestProps.Format = Kusto.Data.Common.DataSourceFormat.csv;
}
private async Task <IKustoIngestionResult> IngestFromStreamAsync(
string csv, IKustoIngestClient client, KustoIngestionProperties properties, StreamSourceOptions sourceOptions)
{
using MemoryStream stream = new();
using StreamWriter writer = new(stream);
writer.Write(csv);
writer.Flush();
stream.Seek(0, SeekOrigin.Begin);
return(await client.IngestFromStreamAsync(stream, properties, sourceOptions));
}
public IKustoClusterBuilder UseKusto()
{
return(new KustoOptionsBuilder(builder =>
{
_kustoOptionsBuilder = builder;
UseKustoInternal(builder.DbProperties.Engine, builder.DbProperties.Region, builder.DbProperties.DbName, builder.DbProperties.ClientId);
foreach (var type in _kustoOptionsBuilder.RegisteredDirectTypes)
{
if (KustoDirectMappings.ContainsKey(type))
{
continue;
}
var tableName = KustoAdminClient.GenerateTableFromType(type).Result;
var ingestProps = new KustoIngestionProperties(KustoDbName, tableName)
{
TableName = tableName,
JSONMappingReference = KustoAdminClient.GenerateTableJsonMappingFromType(type).Result,
IgnoreSizeLimit = true,
ValidationPolicy = null,
Format = DataSourceFormat.json
};
KustoDirectMappings.Add(type, ingestProps);
}
foreach (var type in _kustoOptionsBuilder.RegisteredQueuedTypes)
{
if (KustoQueuedMappings.ContainsKey(type))
{
continue;
}
var tableName = KustoAdminClient.GenerateTableFromType(type).Result;
var ingestProps = new KustoQueuedIngestionProperties(KustoDbName, tableName)
{
TableName = tableName,
JSONMappingReference = KustoAdminClient.GenerateTableJsonMappingFromType(type).Result,
ReportLevel = IngestionReportLevel.FailuresOnly,
ReportMethod = IngestionReportMethod.Queue,
FlushImmediately = _kustoOptionsBuilder?.BufferOptions.FlushImmediately ?? true,
IgnoreSizeLimit = true,
ValidationPolicy = null,
Format = DataSourceFormat.json
};
KustoQueuedMappings.Add(type, ingestProps);
}
}));
}
/// <summary>
/// Ingest data into Kusto.
/// </summary>
/// <param name="table">Name of table to ingest into.</param>
/// <param name="mappingName">Name of table mapping to ingest with.</param>
/// <param name="stream">input JSON data stream.</param>
/// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
private static async Task <IKustoIngestionResult> KustoIngest(KustoConnectionStringBuilder kusto, string db, string table, string mappingName, Stream stream)
{
// Create a disposable client that will execute the ingestion
using IKustoIngestClient client = KustoIngestFactory.CreateDirectIngestClient(kusto);
var ingestProps = new KustoIngestionProperties(db, table)
{
JSONMappingReference = mappingName,
Format = DataSourceFormat.json,
};
var ssOptions = new StreamSourceOptions
{
CompressionType = DataSourceCompressionType.GZip,
};
return(await client.IngestFromStreamAsync(stream, ingestProps, ssOptions));
}
public BlockingKustoUploader(
string blobConnectionString,
string blobContainerName,
KustoConnectionStringBuilder adminCsb,
KustoConnectionStringBuilder kscb,
bool demoMode,
string tableName,
int batchSize,
TimeSpan flushDuration,
bool resetTable = false)
{
Csb = kscb;
AdminCsb = adminCsb;
TableName = tableName;
BatchSize = batchSize;
_flushDuration = flushDuration;
_lastUploadTime = DateTime.UtcNow;
_resetTable = resetTable;
_initializeTable = false;
Completed = new AutoResetEvent(false);
if (Csb != null)
{
_ingestionProperties = new KustoIngestionProperties(Csb.InitialCatalog, tableName);
if (demoMode)
{
_ingestClient = KustoIngestFactory.CreateDirectIngestClient(this.AdminCsb);
}
else
{
_ingestClient = KustoIngestFactory.CreateQueuedIngestClient(Csb);
}
}
if (!string.IsNullOrEmpty(blobConnectionString) &&
!string.IsNullOrEmpty(blobContainerName))
{
blobServiceClient = new BlobServiceClient(blobConnectionString);
blobContainerClient = blobServiceClient.GetBlobContainerClient(blobContainerName);
blobContainerClient.CreateIfNotExists();
}
_nextBatch = new List <IDictionary <string, object> >();
}
public async Task PublishManualReviews(IEnumerable <SloManualReview> results)
{
var dt = results.ToDataTable();
var kustoConnectionStringBuilderEngine = new KustoConnectionStringBuilder(kustoUrl_).WithAadUserPromptAuthentication();
using (var client = KustoIngestFactory.CreateDirectIngestClient(kustoConnectionStringBuilderEngine))
{
//Ingest from blobs according to the required properties
var kustoIngestionProperties = new KustoIngestionProperties(
databaseName: kustoDb_,
tableName: kustoManualReviewTable_
);
var reader = dt.CreateDataReader();
var result = await client.IngestFromDataReaderAsync(reader, kustoIngestionProperties);
}
}
public async Task IngestFromCsvStreamAsync(Stream csv, IngestKustoImageInfoOptions options)
{
KustoConnectionStringBuilder connectionBuilder =
new KustoConnectionStringBuilder($"https://{options.Cluster}.kusto.windows.net")
.WithAadApplicationKeyAuthentication(
options.ServicePrincipal.ClientId,
options.ServicePrincipal.Secret,
options.ServicePrincipal.Tenant);
using (IKustoIngestClient client = KustoIngestFactory.CreateDirectIngestClient(connectionBuilder))
{
KustoIngestionProperties properties =
new KustoIngestionProperties(options.Database, options.Table)
{
Format = DataSourceFormat.csv
};
StreamSourceOptions sourceOptions = new StreamSourceOptions {
SourceId = Guid.NewGuid()
};
if (!options.IsDryRun)
{
IKustoIngestionResult result = await client.IngestFromStreamAsync(csv, properties, sourceOptions);
IngestionStatus ingestionStatus = result.GetIngestionStatusBySourceId(sourceOptions.SourceId);
for (int i = 0; i < 10 && ingestionStatus.Status == Status.Pending; i++)
{
await Task.Delay(TimeSpan.FromSeconds(30));
ingestionStatus = result.GetIngestionStatusBySourceId(sourceOptions.SourceId);
}
if (ingestionStatus.Status != Status.Succeeded)
{
throw new InvalidOperationException(
$"Failed to ingest Kusto data.{Environment.NewLine}{ingestionStatus.Details}");
}
else if (ingestionStatus.Status == Status.Pending)
{
throw new InvalidOperationException($"Timeout while ingesting Kusto data.");
}
}
}
}
public BlockingKustoUploader(
string outputFileName,
KustoConnectionStringBuilder adminCsb,
KustoConnectionStringBuilder kscb,
bool demoMode,
string tableName,
int batchSize,
TimeSpan flushDuration,
bool resetTable = false)
{
OutputFileName = outputFileName;
Csb = kscb;
AdminCsb = adminCsb;
TableName = tableName;
BatchSize = batchSize;
_flushDuration = flushDuration;
_lastUploadTime = DateTime.UtcNow;
_resetTable = resetTable;
_initializeTable = false;
Completed = new AutoResetEvent(false);
if (Csb != null)
{
_ingestionProperties = new KustoIngestionProperties(Csb.InitialCatalog, tableName);
if (demoMode)
{
_ingestClient = KustoIngestFactory.CreateDirectIngestClient(this.AdminCsb);
}
else
{
_ingestClient = KustoIngestFactory.CreateQueuedIngestClient(Csb);
}
}
if (!string.IsNullOrEmpty(OutputFileName))
{
outputFile = new StreamWriter(this.OutputFileName);
outputFile.Write($"[{Environment.NewLine}");
}
_nextBatch = new List <IDictionary <string, object> >();
}
static void Main(string[] args)
{
// Ingest From Local File using KustoDirectIngestClient (only for test purposes):
// Create Kusto connection string with App Authentication
var kustoConnectionStringBuilderEngine =
new KustoConnectionStringBuilder(@"https://{clusterNameAndRegion}.kusto.windows.net").WithAadApplicationKeyAuthentication(
applicationClientId: "{Application Client ID}",
applicationKey: "{Application Key (secret)}",
authority: "{AAD TenantID or name}");
// Create a disposable client that will execute the ingestion
using (IKustoIngestClient client = KustoIngestFactory.CreateDirectIngestClient(kustoConnectionStringBuilderEngine))
{
//Ingest from blobs according to the required properties
var kustoIngestionProperties = new KustoIngestionProperties(databaseName: "myDB", tableName: "myTable");
client.IngestFromStorageAsync(@"< Path to local file >", ingestionProperties: kustoIngestionProperties).GetAwaiter().GetResult();
}
}
public KustoNotifier(Configuration configuration, ILogger logger, IKustoIngestClient kustoIngestClient)
{
_configuration = configuration;
_logger = logger;
_kustoIngestClient = kustoIngestClient;
_kustoIngestionProperties = new KustoIngestionProperties(_configuration.KustoDatabaseName, _configuration.KustoTableName)
{
Format = DataSourceFormat.json,
};
Contract.RequiresNotNullOrEmpty(_configuration.KustoTableIngestionMappingName,
"Kusto ingestion will fail to authenticate without a proper ingestion mapping.");
_kustoIngestionProperties.JSONMappingReference = _configuration.KustoTableIngestionMappingName;
_queue = NagleQueue <T> .Create(FlushAsync,
_configuration.MaxDegreeOfParallelism,
_configuration.FlushInterval,
_configuration.BatchSize);
}
static void Main(string[] args)
{
if (!ParseCommandLine(args, out var cluster, out var database, out var table))
{
Usage();
return;
}
var kcsb = new KustoConnectionStringBuilder
{
DataSource = cluster
};
kcsb = kcsb.WithAadUserPromptAuthentication();
CreateJsonMappingIfNotExists(kcsb, database, table);
// Do ingestion using Kusto.Data client library
using (var siClient = KustoClientFactory.CreateCslStreamIngestClient(kcsb))
{
using (var data = CreateSampleEventLogCsvStream(10))
{
siClient.ExecuteStreamIngest(database, table, data);
}
using (var data = CreateSampleEventLogJsonStream(10))
{
siClient.ExecuteStreamIngestAsync(
database,
table,
data,
null,
Kusto.Data.Common.DataSourceFormat.json,
compressStream: false,
mappingName: s_jsonMappingName);
}
}
// Do ingestion using Kusto.Ingest client library. The data still goes directly to the engine cluster
// Just a convenience for applications already using IKustoIngest interface
using (var ingestClient = KustoIngestFactory.CreateStreamingIngestClient(kcsb))
{
using (var data = CreateSampleEventLogCsvStream(10))
{
var ingestProperties = new KustoIngestionProperties(database, table)
{
Format = DataSourceFormat.csv,
};
ingestClient.IngestFromStreamAsync(data, ingestProperties);
}
using (var data = CreateSampleEventLogJsonStream(10))
{
var ingestProperties = new KustoIngestionProperties(database, table)
{
Format = DataSourceFormat.json,
JSONMappingReference = s_jsonMappingName
};
ingestClient.IngestFromStreamAsync(data, ingestProperties);
}
}
}
public Task <IKustoIngestionResult> IngestFromStreamAsync(Stream stream, KustoIngestionProperties ingestionProperties, StreamSourceOptions sourceOptions = null)
{
return(_successfulResultInstance);
}
static void Main(string[] args)
{
var kcsb = new KustoConnectionStringBuilder();
kcsb.DataSource = "https://kustolab.kusto.windows.net";
kcsb.FederatedSecurity = true;
string databaseName = "StreamingIngestionSample";
string tableName = "EventLog";
CreateJsonMappingIfNotExists(kcsb, databaseName, tableName);
// Do ingestion using Kusto.Data client library
using (var siClient = KustoClientFactory.CreateCslStreamIngestClient(kcsb))
{
using (var data = CreateSampleEventLogCsvStream(10))
{
siClient.ExecuteStreamIngestAsync(
databaseName,
tableName,
data,
null,
DataSourceFormat.csv);
}
using (var data = CreateSampleEventLogJsonStream(10))
{
siClient.ExecuteStreamIngestAsync(
databaseName,
tableName,
data,
null,
DataSourceFormat.json,
compressStream: false,
mappingName: s_jsonMappingName).ResultEx();
}
}
// Do ingestion using Kusto.Ingest client library. The data still goes directly to the engine cluster
// Just a convenience for applications already using IKustoIngest interface
using (var ingestClient = KustoIngestFactory.CreateStreamingIngestClient(kcsb))
{
using (var data = CreateSampleEventLogCsvStream(10))
{
var ingestProperties = new KustoIngestionProperties(databaseName, tableName)
{
Format = DataSourceFormat.csv,
};
ingestClient.IngestFromStreamAsync(data, ingestProperties).ResultEx();
}
using (var data = CreateSampleEventLogJsonStream(10))
{
var ingestProperties = new KustoIngestionProperties(databaseName, tableName)
{
Format = DataSourceFormat.json,
IngestionMapping = new IngestionMapping {
IngestionMappingKind = Kusto.Data.Ingestion.IngestionMappingKind.Json, IngestionMappingReference = s_jsonMappingName
}
};
ingestClient.IngestFromStream(data, ingestProperties);
}
}
}
public Task <IKustoIngestionResult> IngestFromStorageAsync(string uri, KustoIngestionProperties ingestionProperties, StorageSourceOptions sourceOptions = null)
{
return(_successfulResultInstance);
}
public ActionResult Upload(string instanceUrl, string databaseName, string tenantId, string clientId, string secret)
{
ADXModel model = new ADXModel
{
InstanceUrl = instanceUrl,
DatabaseName = databaseName,
TenantId = tenantId,
ClientId = clientId,
Secret = secret,
};
try
{
var kustoConnectionStringBuilder = new KustoConnectionStringBuilder(instanceUrl).WithAadApplicationKeyAuthentication(clientId, secret, tenantId);
string tableName = DTDL._nodesetNamespaceURI.Replace("http://", "").Replace('/', '_').Replace('.', '_').TrimEnd('_');
using (var kustoClient = KustoClientFactory.CreateCslAdminProvider(kustoConnectionStringBuilder))
{
kustoClient.ExecuteControlCommand(databaseName, ".create table " + tableName + " (ExpandedNodeID: string, DisplayName: string, Type: string, ParentNodeID: string)");
string command = ".create table " + tableName + " ingestion json mapping 'Nodeset_Mapping' '["
+ "{ \"properties\": { \"path\": \"$.Key\" }, \"column\": \"ExpandedNodeID\", \"datatype\": \"string\" },"
+ "{ \"properties\": { \"path\": \"$.Value.Item1\" }, \"column\": \"DisplayName\", \"datatype\": \"string\" } ,"
+ "{ \"properties\": { \"path\": \"$.Value.Item2\" }, \"column\": \"Type\", \"datatype\": \"string\" },"
+ "{ \"properties\": { \"path\": \"$.Value.Item3\" }, \"column\": \"ParentNodeID\", \"datatype\": \"string\" } ]'";
kustoClient.ExecuteControlCommand(databaseName, command);
command = ".alter table " + tableName + " policy ingestionbatching @'{"
+ "\"MaximumBatchingTimeSpan\": \"00:00:05\","
+ "\"MaximumNumberOfItems\": 100,"
+ "\"MaximumRawDataSizeMB\": 1024 }'";
kustoClient.ExecuteControlCommand(databaseName, command);
}
using (IKustoIngestClient client = KustoIngestFactory.CreateDirectIngestClient(kustoConnectionStringBuilder))
{
var kustoIngestionProperties = new KustoIngestionProperties(databaseName, tableName);
kustoIngestionProperties.Format = DataSourceFormat.multijson;
kustoIngestionProperties.IngestionMapping = new IngestionMapping()
{
IngestionMappingReference = "Nodeset_Mapping",
IngestionMappingKind = IngestionMappingKind.Json
};
foreach (KeyValuePair <string, Tuple <string, string, string> > entry in DTDL._nodeList.ToArray(100))
{
string content = JsonConvert.SerializeObject(entry, Formatting.Indented);
MemoryStream stream = new MemoryStream(Encoding.UTF8.GetBytes(content));
client.IngestFromStream(stream, kustoIngestionProperties);
stream.Dispose();
}
}
using (var cslQueryProvider = KustoClientFactory.CreateCslQueryProvider(kustoConnectionStringBuilder))
{
string query = $"{tableName} | count";
var results = cslQueryProvider.ExecuteQuery <long>(databaseName, query);
foreach (long result in results)
{
model.StatusMessage = "ADX Ingestion succeeded.";
return(View("Index", model));
}
}
model.StatusMessage = "ADX Ingestion failed!";
return(View("Error", model));
}
catch (Exception ex)
{
model.StatusMessage = ex.Message;
return(View("Index", model));
}
}
static void IngestData(KustoConnectionStringBuilder engineKustoConnectionStringBuilder, KustoConnectionStringBuilder dmKustoConnectionStringBuilderDM,
string databaseName, string tableName)
{
// 1. Ingest by connecting directly to the Kustolab cluster and sending a command
using (IKustoIngestClient directClient = KustoIngestFactory.CreateDirectIngestClient(engineKustoConnectionStringBuilder))
{
var kustoIngestionProperties = new KustoIngestionProperties(databaseName, tableName);
directClient.IngestFromDataReaderAsync(GetDataAsIDataReader(), kustoIngestionProperties);
}
// 2. Ingest by submitting the data to the Kustolab ingestion cluster
// Note that this is an async operation, so data might not appear immediately
using (IKustoIngestClient queuedClient = KustoIngestFactory.CreateQueuedIngestClient(dmKustoConnectionStringBuilderDM))
{
var kustoIngestionProperties = new KustoIngestionProperties(databaseName, tableName);
queuedClient.IngestFromDataReaderAsync(GetDataAsIDataReader(), kustoIngestionProperties);
}
// 3. Ingest by submitting the data to the Kustolab ingestion cluster -
// This time, update the report method and level so you can track the status of your ingestion
// using the IKustoIngestionResult returned from the ingest operation.
IKustoIngestionResult ingestionResult;
using (IKustoIngestClient queuedClient = KustoIngestFactory.CreateQueuedIngestClient(dmKustoConnectionStringBuilderDM))
{
var kustoIngestionProperties = new KustoQueuedIngestionProperties(databaseName, tableName)
{
// The default ReportLevel is set to FailuresOnly.
// In this case we want to check the status of successful ingestions as well.
ReportLevel = IngestionReportLevel.FailuresAndSuccesses,
// You can use either a queue or a table to track the status of your ingestion.
ReportMethod = IngestionReportMethod.Table
};
ingestionResult = queuedClient.IngestFromDataReaderAsync(GetDataAsIDataReader(), kustoIngestionProperties).Result;
}
// Obtain the status of our ingestion
var ingestionStatus = ingestionResult.GetIngestionStatusCollection().First();
var watch = System.Diagnostics.Stopwatch.StartNew();
var shouldContinue = true;
while ((ingestionStatus.Status == Status.Pending) && (shouldContinue))
{
// Wait a minute...
Thread.Sleep(TimeSpan.FromMinutes(1));
// Try again
ingestionStatus = ingestionResult.GetIngestionStatusBySourceId(ingestionStatus.IngestionSourceId);
shouldContinue = watch.ElapsedMilliseconds < TimeOutInMilliSeconds;
}
watch.Stop();
if (ingestionStatus.Status == Status.Pending)
{
// The status of the ingestion did not change.
Console.WriteLine(
"Ingestion with ID:'{0}' did not complete. Timed out after :'{1}' Milliseconds"
.FormatWithInvariantCulture(
ingestionStatus.IngestionSourceId, watch.ElapsedMilliseconds));
}
else
{
// The status of the ingestion has changed
Console.WriteLine(
"Ingestion with ID:'{0}' is complete. Ingestion Status:'{1}'".FormatWithInvariantCulture(
ingestionStatus.IngestionSourceId, ingestionStatus.Status));
}
// 4. Show the contents of the table
using (var client = Kusto.Data.Net.Client.KustoClientFactory.CreateCslQueryProvider(engineKustoConnectionStringBuilder))
{
while (true)
{
var query = string.Format("{0}", tableName);
var reader = client.ExecuteQuery(query);
Kusto.Cloud.Platform.Data.ExtendedDataReader.WriteAsText(reader, "Data ingested into the table:", tabify: true, firstOnly: true);
Console.WriteLine("Press 'r' to retry retrieving data from the table, any other key to quit");
var key = Console.ReadKey();
if (key.KeyChar != 'r' && key.KeyChar != 'R')
{
break;
}
Console.WriteLine();
}
}
}
public Task <IKustoIngestionResult> IngestFromDataReaderAsync(IDataReader dataReader, KustoIngestionProperties ingestionProperties, DataReaderSourceOptions sourceOptions = null)
{
return(_successfulResultInstance);
}
