/// <summary>
/// Constructor. Initializes this object and does nothing else.
/// </summary>
/// <param name="connectionString">Kusto connection string.</param>
/// <param name="database">Database into which to ingest.</param>
/// <param name="table">Table into which to ingest.</param>
/// <param name="csvMapping">Csv mapping that applies to all CSV files to be uploaded via <see cref="UploadSingleCsvFile"/></param>
/// <param name="deleteFilesOnSuccess">Whether to delete files upon successful upload.</param>
/// <param name="checkForIngestionErrors">
/// Whether to check for ingestion errors before disposing this object.
/// Note that at this time not all uploaded files have necessarily been ingested; this class
/// does not wait for ingestions to complete, it only checks for failures of those that have completed.
/// </param>
/// <param name="log">Optional log to which to write some debug information.</param>
public KustoUploader
(
string connectionString,
string database,
string table,
IEnumerable <CsvColumnMapping> csvMapping,
bool deleteFilesOnSuccess,
bool checkForIngestionErrors,
ILog log = null
)
{
_log = log;
_deleteFilesOnSuccess = deleteFilesOnSuccess;
_checkForIngestionErrors = checkForIngestionErrors;
_client = KustoIngestFactory.CreateQueuedIngestClient(connectionString);
_hasUploadErrors = false;
_ingestionProperties = new KustoQueuedIngestionProperties(database, table)
{
CSVMapping = csvMapping,
ReportLevel = IngestionReportLevel.FailuresOnly,
ReportMethod = IngestionReportMethod.Queue,
};
_block = new ActionBlock <FileDescription>
(
UploadSingleCsvFile,
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = 1,
BoundedCapacity = DataflowBlockOptions.Unbounded
}
);
}
private KustoQueuedIngestionProperties GetQueuedModelProperties(Type eventType)
{
lock (_gate)
{
KustoQueuedIngestionProperties ingestProps;
if (!KustoQueuedMappings.ContainsKey(eventType))
{
ingestProps = new KustoQueuedIngestionProperties(KustoDbName, "Unknown")
{
TableName = KustoAdminClient.GenerateTableFromType(eventType).Result,
JSONMappingReference = KustoAdminClient.GenerateTableJsonMappingFromType(eventType).Result,
ReportLevel = IngestionReportLevel.FailuresOnly,
ReportMethod = IngestionReportMethod.Queue,
FlushImmediately = _kustoOptionsBuilder?.BufferOptions.FlushImmediately ?? true,
IgnoreSizeLimit = true,
ValidationPolicy = null,
Format = DataSourceFormat.json
};
KustoQueuedMappings.Add(eventType, ingestProps);
}
else
{
ingestProps = KustoQueuedMappings[eventType];
}
return(ingestProps);
}
}
static async Task Main(string[] args)
{
// Ingest From a Local File using KustoQueuedIngestClient and report status to a table
// Create Kusto connection string with App Authentication
var kustoConnectionStringBuilderDM =
new KustoConnectionStringBuilder(@"https://ingest-{clusterNameAndRegion}.kusto.windows.net").WithAadApplicationKeyAuthentication(
applicationClientId: "{Application Client ID}",
applicationKey: "{Application Key (secret)}",
authority: "{AAD TenantID or name}");
// Create a disposable client that will execute the ingestion
IKustoQueuedIngestClient client = KustoIngestFactory.CreateQueuedIngestClient(kustoConnectionStringBuilderDM);
// Ingest from a file according to the required properties
var kustoIngestionProperties = new KustoQueuedIngestionProperties(databaseName: "myDB", tableName: "myDB")
{
// Setting the report level to FailuresAndSuccesses will cause both successful and failed ingestions to be reported
// (Rather than the default "FailuresOnly" level)
ReportLevel = IngestionReportLevel.FailuresAndSuccesses,
// Choose the report method of choice
ReportMethod = IngestionReportMethod.Table
};
var filePath = @"< Path to file >";
var fileIdentifier = Guid.NewGuid();
var fileDescription = new FileDescription()
{
FilePath = filePath, SourceId = fileIdentifier
};
var sourceOptions = new StorageSourceOptions()
{
SourceId = fileDescription.SourceId.Value
};
// Execute the ingest operation and save the result.
var clientResult = await client.IngestFromStorageAsync(fileDescription.FilePath,
ingestionProperties : kustoIngestionProperties, sourceOptions);
// Use the fileIdentifier you supplied to get the status of your ingestion
var ingestionStatus = clientResult.GetIngestionStatusBySourceId(fileIdentifier);
while (ingestionStatus.Status == Status.Pending)
{
// Wait a minute...
Thread.Sleep(TimeSpan.FromMinutes(1));
// Try again
ingestionStatus = clientResult.GetIngestionStatusBySourceId(fileIdentifier);
}
// Verify the results of the ingestion
Ensure.ConditionIsMet(ingestionStatus.Status == Status.Succeeded,
"The file should have been ingested successfully");
// Dispose of the client
client.Dispose();
}
public static void Run([CosmosDBTrigger(
databaseName: "data",
collectionName: "device",
ConnectionStringSetting = "rbdemocosmosdb_DOCUMENTDB",
LeaseCollectionName = "leases", CreateLeaseCollectionIfNotExists = true)] IReadOnlyList <Document> input, ILogger log)
{
if (input != null && input.Count > 0)
{
log.LogInformation("Documents modified " + input.Count);
log.LogInformation("First document Id " + input[0].Id);
//input[0].ToString() is the data that needs to be ingested.
var tenantId = System.Environment.GetEnvironmentVariable($"tenantId");
var applicationClientId = System.Environment.GetEnvironmentVariable($"applicationClientId");
var applicationKey = System.Environment.GetEnvironmentVariable($"applicationKey");
var serviceNameAndRegion = System.Environment.GetEnvironmentVariable($"serviceNameAndRegion");
var kustoDatabase = "device";
var kustoTableName = "Staging";
var mappingName = "deviceMapping";
var kustoIngestConnectionStringBuilder =
new KustoConnectionStringBuilder($"https://ingest-{serviceNameAndRegion}.kusto.windows.net")
{
FederatedSecurity = true,
InitialCatalog = kustoDatabase,
ApplicationClientId = applicationClientId,
ApplicationKey = applicationKey,
Authority = tenantId
};
using (var ingestClient = KustoIngestFactory.CreateQueuedIngestClient(kustoIngestConnectionStringBuilder))
{
var properties = new KustoQueuedIngestionProperties(kustoDatabase, kustoTableName)
{
Format = DataSourceFormat.json,
JSONMappingReference = mappingName,
ReportLevel = IngestionReportLevel.FailuresAndSuccesses,
ReportMethod = IngestionReportMethod.Queue
};
string inputValue = input[0].ToString().Replace("\r\n", "");
//string inputValue = "{ \"Id\":\"1\", \"Timestamp\":\"2\", \"Message\":\"This is a test message\" }";
//object val = Newtonsoft.Json.JsonConvert.DeserializeObject(inputValue);
var stream = new System.IO.MemoryStream(System.Text.Encoding.UTF8.GetBytes(inputValue));
stream.Seek(0, SeekOrigin.Begin);
// Post ingestion message
ingestClient.IngestFromStream(stream, properties, leaveOpen: true);
// Wait and retrieve all notifications
// - Actual duration should be decided based on the effective Ingestion Batching Policy set on the table/database
Thread.Sleep(10000);
var errors = ingestClient.GetAndDiscardTopIngestionFailures().GetAwaiter().GetResult();
var successes = ingestClient.GetAndDiscardTopIngestionSuccesses().GetAwaiter().GetResult();
errors.ForEach((f) => { Console.WriteLine($"Ingestion error: {f.Info.Details}"); });
successes.ForEach((s) => { Console.WriteLine($"Ingested: {s.Info.IngestionSourcePath}"); });
}
}
}
/// <summary>
/// Performs a write to the indicated table, sending the appropriately filtered test report to Kusto
/// Returns the number of lines written
/// Skips the final write step when testing
/// </summary>
public bool WriteDataForTable(bool testing, List <AdoBuildResultRow> reports)
{
try
{
// Create Kusto connection string with App Authentication
var kustoConnectionStringBuilderEngine =
new KustoConnectionStringBuilder(config.clusterUri).WithAadApplicationKeyAuthentication(
applicationClientId: config.clientId,
applicationKey: config.secretKey,
authority: config.authority);
using (var ingestClient = KustoIngestFactory.CreateDirectIngestClient(kustoConnectionStringBuilderEngine))
{
var ingestProps = new KustoQueuedIngestionProperties(config.dbName, config.tableName);
// For the sake of getting both failure and success notifications we set this to IngestionReportLevel.FailuresAndSuccesses
// Usually the recommended level is IngestionReportLevel.FailuresOnly
ingestProps.ReportLevel = IngestionReportLevel.FailuresAndSuccesses;
ingestProps.ReportMethod = IngestionReportMethod.Queue;
ingestProps.IngestionMapping = new IngestionMapping {
IngestionMappingReference = config.mappingName
};
ingestProps.Format = DataSourceFormat.json;
// Prepare data for ingestion
using (var memStream = new MemoryStream())
using (var writer = new StreamWriter(memStream))
{
foreach (AdoBuildResultRow eachReport in reports)
{
writer.WriteLine(JsonConvert.SerializeObject(eachReport));
Console.WriteLine($"Writing: {eachReport.jobname}, ({eachReport.passed}/{eachReport.failed}/{eachReport.other}/{eachReport.total}) for {eachReport.timestamp}");
}
writer.Flush();
memStream.Seek(0, SeekOrigin.Begin);
// Post ingestion message
if (!testing)
{
ingestClient.IngestFromStream(memStream, ingestProps, leaveOpen: true);
}
}
}
return(true);
}
catch (Exception e)
{
Console.WriteLine($"ERROR: Could not write to: {config.clusterUri}.{config.dbName}.{config.tableName}, using Mapping: {config.mappingName}");
Console.WriteLine(e.ToString());
return(false);
}
}
public IKustoClusterBuilder UseKusto()
{
return(new KustoOptionsBuilder(builder =>
{
_kustoOptionsBuilder = builder;
UseKustoInternal(builder.DbProperties.Engine, builder.DbProperties.Region, builder.DbProperties.DbName, builder.DbProperties.ClientId);
foreach (var type in _kustoOptionsBuilder.RegisteredDirectTypes)
{
if (KustoDirectMappings.ContainsKey(type))
{
continue;
}
var tableName = KustoAdminClient.GenerateTableFromType(type).Result;
var ingestProps = new KustoIngestionProperties(KustoDbName, tableName)
{
TableName = tableName,
JSONMappingReference = KustoAdminClient.GenerateTableJsonMappingFromType(type).Result,
IgnoreSizeLimit = true,
ValidationPolicy = null,
Format = DataSourceFormat.json
};
KustoDirectMappings.Add(type, ingestProps);
}
foreach (var type in _kustoOptionsBuilder.RegisteredQueuedTypes)
{
if (KustoQueuedMappings.ContainsKey(type))
{
continue;
}
var tableName = KustoAdminClient.GenerateTableFromType(type).Result;
var ingestProps = new KustoQueuedIngestionProperties(KustoDbName, tableName)
{
TableName = tableName,
JSONMappingReference = KustoAdminClient.GenerateTableJsonMappingFromType(type).Result,
ReportLevel = IngestionReportLevel.FailuresOnly,
ReportMethod = IngestionReportMethod.Queue,
FlushImmediately = _kustoOptionsBuilder?.BufferOptions.FlushImmediately ?? true,
IgnoreSizeLimit = true,
ValidationPolicy = null,
Format = DataSourceFormat.json
};
KustoQueuedMappings.Add(type, ingestProps);
}
}));
}
static void Main(string[] args)
{
// Ingest From a Local Files using KustoQueuedIngestClient and report status to a queue
// Create Kusto connection string with App Authentication
var kustoConnectionStringBuilderDM =
new KustoConnectionStringBuilder(@"https://ingest-{clusterNameAndRegion}.kusto.windows.net").WithAadApplicationKeyAuthentication(
applicationClientId: "{Application Client ID}",
applicationKey: "{Application Key (secret)}",
authority: "{AAD TenantID or name}");
// Create a disposable client that will execute the ingestion
IKustoQueuedIngestClient client = KustoIngestFactory.CreateQueuedIngestClient(kustoConnectionStringBuilderDM);
// Ingest from a file according to the required properties
var kustoIngestionProperties = new KustoQueuedIngestionProperties(databaseName: "myDB", tableName: "myTable")
{
// Setting the report level to FailuresAndSuccesses will cause both successful and failed ingestions to be reported
// (Rather than the default "FailuresOnly" level - which is demonstrated in the
// 'Ingest From Local File(s) using KustoQueuedIngestClient and Ingestion Validation' section)
ReportLevel = IngestionReportLevel.FailuresAndSuccesses,
// Choose the report method of choice. 'Queue' is the default method.
// For the sake of the example, we will choose it anyway.
ReportMethod = IngestionReportMethod.Queue
};
client.IngestFromStorageAsync("ValidTestFile.csv", kustoIngestionProperties);
client.IngestFromStorageAsync("InvalidTestFile.csv", kustoIngestionProperties);
// Waiting for the aggregation
Thread.Sleep(TimeSpan.FromMinutes(8));
// Retrieve and validate failures
var ingestionFailures = client.PeekTopIngestionFailures().GetAwaiter().GetResult();
Ensure.IsTrue((ingestionFailures.Count() > 0), "The failed ingestion should have been reported to the failed ingestions queue");
// Retrieve, delete and validate failures
ingestionFailures = client.GetAndDiscardTopIngestionFailures().GetAwaiter().GetResult();
Ensure.IsTrue((ingestionFailures.Count() > 0), "The failed ingestion should have been reported to the failed ingestions queue");
// Verify the success has also been reported to the queue
var ingestionSuccesses = client.GetAndDiscardTopIngestionSuccesses().GetAwaiter().GetResult();
Ensure.ConditionIsMet((ingestionSuccesses.Count() > 0),
"The successful ingestion should have been reported to the successful ingestions queue");
// Dispose of the client
client.Dispose();
}
public KustoTable(KustoConnectionStringBuilder connectionStringBuilder, string databaseName, string tableName, IEnumerable <ColumnMapping> columnMappings)
{
_connectionStringBuilder = connectionStringBuilder;
_ingestionProperties = new KustoQueuedIngestionProperties(databaseName, tableName)
{
ReportLevel = IngestionReportLevel.FailuresOnly,
ReportMethod = IngestionReportMethod.Queue,
IngestionMapping = new IngestionMapping()
{
IngestionMappingKind = global::Kusto.Data.Ingestion.IngestionMappingKind.Json,
IngestionMappings = columnMappings
},
Format = DataSourceFormat.json
};
}
public void IngestData(string table, string mappingName, Stream memStream)
{
var ingestProps =
new KustoQueuedIngestionProperties(DatabaseName, table)
{
ReportLevel = IngestionReportLevel.FailuresAndSuccesses,
ReportMethod = IngestionReportMethod.Queue,
JSONMappingReference = mappingName,
Format = DataSourceFormat.json
};
_ingestionClient.IngestFromStream(memStream, ingestProps, leaveOpen: true);
// Wait and retrieve all notifications
Thread.Sleep(10000);
var errors = _ingestionClient.GetAndDiscardTopIngestionFailuresAsync().GetAwaiter().GetResult();
var successes = _ingestionClient.GetAndDiscardTopIngestionSuccessesAsync().GetAwaiter().GetResult();
errors.ForEach((f) => { Logger.Error($"Ingestion error: {f.Info.Details}."); });
successes.ForEach((s) => { Logger.Info($"Ingested : {s.Info.IngestionSourcePath}"); });
}
public async Task <bool> UpdateStormEvent(string update)
{
try
{
var kcsb = new KustoConnectionStringBuilder(_options.ADXCluster, _options.ADXDatabase)
.WithAadUserPromptAuthentication();
using (var queryProvider = KustoIngestFactory.CreateDirectIngestClient(kcsb))
{
// Ingest from a file according to the required properties
var kustoIngestionProperties = new KustoQueuedIngestionProperties(databaseName: _options.ADXDatabase, tableName: _options.ADXTable)
{
// Setting the report level to FailuresAndSuccesses will cause both successful and failed ingestions to be reported
// (Rather than the default "FailuresOnly" level - which is demonstrated in the
// 'Ingest From Local File(s) using KustoQueuedIngestClient and Ingestion Validation' section)
ReportLevel = IngestionReportLevel.FailuresAndSuccesses,
// Choose the report method of choice. 'Queue' is the default method.
// For the sake of the example, we will choose it anyway.
ReportMethod = IngestionReportMethod.Queue,
Format = DataSourceFormat.json
};
StreamDescription sd = new StreamDescription
{
SourceId = Guid.NewGuid(),
Stream = GenericHelper.GenerateStreamFromString(update)
};
await queryProvider.IngestFromStreamAsync(sd, kustoIngestionProperties);
}
return(true);
}
catch
{
return(false);
}
}
/// <summary>
/// Saves the details of the <see cref="HttpResponseMessage"/> to
/// </summary>
/// <param name="monitorName"></param>
/// <param name="httpResponse"></param>
/// <returns></returns>
public async Task ReportUrlAccessAsync(string monitorName, HttpResponseMessage httpResponse, CancellationToken cancellationToken = default)
{
HttpRequestLogEntry logEntry = new HttpRequestLogEntry()
{
MonitorName = monitorName,
EventTime = DateTime.UtcNow,
RequestedUrl = httpResponse.RequestMessage.RequestUri.AbsoluteUri,
HttpStatusCode = (int)httpResponse.StatusCode
};
KustoConnectionStringBuilder kcsb = new KustoConnectionStringBuilder($"https://ingest-{ServiceNameAndRegion}.kusto.windows.net")
.WithAadManagedIdentity("system");
using IKustoQueuedIngestClient ingestClient = KustoIngestFactory.CreateQueuedIngestClient(kcsb);
KustoQueuedIngestionProperties ingestProps = new KustoQueuedIngestionProperties(DatabaseName, TableName)
{
ReportLevel = IngestionReportLevel.FailuresOnly,
ReportMethod = IngestionReportMethod.Queue,
IngestionMapping = new IngestionMapping()
{
IngestionMappingKind = Kusto.Data.Ingestion.IngestionMappingKind.Json,
IngestionMappings = HttpRequestLogColumnMapping
},
Format = DataSourceFormat.json
};
using MemoryStream memStream = new MemoryStream();
using StreamWriter writer = new StreamWriter(memStream);
writer.WriteLine(JsonConvert.SerializeObject(logEntry));
writer.Flush();
memStream.Seek(0, SeekOrigin.Begin);
// IKustoQueuedIngestClient doesn't support cancellation at the moment. Update the line below if it does in the future.
await ingestClient.IngestFromStreamAsync(memStream, ingestProps, leaveOpen : true);
}
static void IngestData(KustoConnectionStringBuilder engineKustoConnectionStringBuilder, KustoConnectionStringBuilder dmKustoConnectionStringBuilderDM,
string databaseName, string tableName)
{
// 1. Ingest by connecting directly to the Kustolab cluster and sending a command
using (IKustoIngestClient directClient = KustoIngestFactory.CreateDirectIngestClient(engineKustoConnectionStringBuilder))
{
var kustoIngestionProperties = new KustoIngestionProperties(databaseName, tableName);
directClient.IngestFromDataReaderAsync(GetDataAsIDataReader(), kustoIngestionProperties);
}
// 2. Ingest by submitting the data to the Kustolab ingestion cluster
// Note that this is an async operation, so data might not appear immediately
using (IKustoIngestClient queuedClient = KustoIngestFactory.CreateQueuedIngestClient(dmKustoConnectionStringBuilderDM))
{
var kustoIngestionProperties = new KustoIngestionProperties(databaseName, tableName);
queuedClient.IngestFromDataReaderAsync(GetDataAsIDataReader(), kustoIngestionProperties);
}
// 3. Ingest by submitting the data to the Kustolab ingestion cluster -
// This time, update the report method and level so you can track the status of your ingestion
// using the IKustoIngestionResult returned from the ingest operation.
IKustoIngestionResult ingestionResult;
using (IKustoIngestClient queuedClient = KustoIngestFactory.CreateQueuedIngestClient(dmKustoConnectionStringBuilderDM))
{
var kustoIngestionProperties = new KustoQueuedIngestionProperties(databaseName, tableName)
{
// The default ReportLevel is set to FailuresOnly.
// In this case we want to check the status of successful ingestions as well.
ReportLevel = IngestionReportLevel.FailuresAndSuccesses,
// You can use either a queue or a table to track the status of your ingestion.
ReportMethod = IngestionReportMethod.Table
};
ingestionResult = queuedClient.IngestFromDataReaderAsync(GetDataAsIDataReader(), kustoIngestionProperties).Result;
}
// Obtain the status of our ingestion
var ingestionStatus = ingestionResult.GetIngestionStatusCollection().First();
var watch = System.Diagnostics.Stopwatch.StartNew();
var shouldContinue = true;
while ((ingestionStatus.Status == Status.Pending) && (shouldContinue))
{
// Wait a minute...
Thread.Sleep(TimeSpan.FromMinutes(1));
// Try again
ingestionStatus = ingestionResult.GetIngestionStatusBySourceId(ingestionStatus.IngestionSourceId);
shouldContinue = watch.ElapsedMilliseconds < TimeOutInMilliSeconds;
}
watch.Stop();
if (ingestionStatus.Status == Status.Pending)
{
// The status of the ingestion did not change.
Console.WriteLine(
"Ingestion with ID:'{0}' did not complete. Timed out after :'{1}' Milliseconds"
.FormatWithInvariantCulture(
ingestionStatus.IngestionSourceId, watch.ElapsedMilliseconds));
}
else
{
// The status of the ingestion has changed
Console.WriteLine(
"Ingestion with ID:'{0}' is complete. Ingestion Status:'{1}'".FormatWithInvariantCulture(
ingestionStatus.IngestionSourceId, ingestionStatus.Status));
}
// 4. Show the contents of the table
using (var client = Kusto.Data.Net.Client.KustoClientFactory.CreateCslQueryProvider(engineKustoConnectionStringBuilder))
{
while (true)
{
var query = string.Format("{0}", tableName);
var reader = client.ExecuteQuery(query);
Kusto.Cloud.Platform.Data.ExtendedDataReader.WriteAsText(reader, "Data ingested into the table:", tabify: true, firstOnly: true);
Console.WriteLine("Press 'r' to retry retrieving data from the table, any other key to quit");
var key = Console.ReadKey();
if (key.KeyChar != 'r' && key.KeyChar != 'R')
{
break;
}
Console.WriteLine();
}
}
}
public static async Task Run(int numMessagesToSend = 100000, bool direct = true)
{
var tenantId = "1a59e398-83d8-4052-aec9-74a7d6461c5e";
//var user = "******";
//var password = "******";
var clientid = "24c6b350-9b47-4e49-b794-7861cf0ce116";
var clientsecret = "=BYPyhmHf0efu7[-zuoOLGtsT=xcO6v3";
var database = "trillsample";
var rand = new Random((int)DateTime.Now.Ticks);
var m1 = new List <MeasureT1>();
var m3 = new List <MeasureT3>();
var m4 = new List <MeasureT4>();
var j1 = new StringBuilder();
var j3 = new StringBuilder();
var j4 = new StringBuilder();
if (direct)
{
var ingestUri = "Data Source=https://trillsample.westeurope.kusto.windows.net;Initial Catalog=trillsample";
var ingestConnectionStringBuilder = new KustoConnectionStringBuilder(ingestUri)
{
FederatedSecurity = true,
InitialCatalog = database,
#if false
UserID = user,
Password = password,
#else
ApplicationClientId = clientid,
ApplicationKey = clientsecret,
#endif
Authority = tenantId
};
using (var ingestClient = KustoIngestFactory.CreateDirectIngestClient(ingestConnectionStringBuilder))
{
for (var i = 1; i < numMessagesToSend; i++)
{
var uri = (1002030 + i).ToString();
var min = 200 + ((float)rand.Next(100, 500) / 10.0f);
var now = DateTime.Now;
var value = 23f + (float)rand.Next(10) / 10f;
j1.Append(JsonConvert.SerializeObject(Program.CreateMeasureT1(rand, i, uri, min, now, value))).Append("\n");
j3.Append(JsonConvert.SerializeObject(Program.CreateMeasureT3(i, uri, now, value))).Append("\n");
j4.Append(JsonConvert.SerializeObject(Program.CreateMeasureT4(rand, uri, now))).Append("\n");
if (i % 5000 == 0)
{
Console.Title = $"{i}/{numMessagesToSend}";
m1 = new List <MeasureT1>();
m3 = new List <MeasureT3>();
m4 = new List <MeasureT4>();
using (var s1 = new MemoryStream(Encoding.ASCII.GetBytes(j1.ToString())))
using (var s3 = new MemoryStream(Encoding.ASCII.GetBytes(j1.ToString())))
using (var s4 = new MemoryStream(Encoding.ASCII.GetBytes(j1.ToString())))
{
Console.Write("+");
var props = new KustoQueuedIngestionProperties(database, "MeasuresT1")
{
Format = DataSourceFormat.json, JSONMappingReference = "MapMeasureT1"
};
var result = Task.Run(() => ingestClient.IngestFromStream(s1, props, leaveOpen: true));
props = new KustoQueuedIngestionProperties(database, "MeasuresT3")
{
Format = DataSourceFormat.json, JSONMappingReference = "MapMeasureT3"
};
result = ingestClient.IngestFromStreamAsync(s3, props, leaveOpen: true);
props = new KustoQueuedIngestionProperties(database, "MeasuresT4")
{
Format = DataSourceFormat.json, JSONMappingReference = "MapMeasureT4"
};
result = ingestClient.IngestFromStreamAsync(s4, props, leaveOpen: true);
Console.Write("-");
}
}
}
}
}
else
{
var storageAccount = CloudStorageAccount.Parse(StorageConnectionString);
var blobClient = storageAccount.CreateCloudBlobClient();
var blobContainer = blobClient.GetContainerReference("uploads");
blobContainer.CreateIfNotExistsAsync().Wait();
var ingestUri = "https://ingest-trillsample.westeurope.kusto.windows.net/";
var kustoConnectionStringBuilder = new KustoConnectionStringBuilder(ingestUri)
{
FederatedSecurity = true,
InitialCatalog = database,
#if false
UserID = user,
Password = password,
#else
ApplicationClientId = clientid,
ApplicationKey = clientsecret,
#endif
Authority = tenantId
};
for (var i = 1; i < numMessagesToSend; i++)
{
var uri = (1002030 + 10000000 + i).ToString();
var min = 200 + ((float)rand.Next(100, 500) / 10.0f);
var now = DateTime.Now;
var value = 23f + (float)rand.Next(10) / 10f;
j1.Append(JsonConvert.SerializeObject(Program.CreateMeasureT1(rand, i, uri, min, now, value))).Append("\n");
j3.Append(JsonConvert.SerializeObject(Program.CreateMeasureT3(i, uri, now, value))).Append("\n");
j4.Append(JsonConvert.SerializeObject(Program.CreateMeasureT4(rand, uri, now))).Append("\n");
if (i % 5000 == 0)
{
Console.Title = $"{i}/{numMessagesToSend}";
var blockBlob1 = blobContainer.GetBlockBlobReference($"MeasuresT1-{i}");
using (var stream = new StreamWriter(await blockBlob1.OpenWriteAsync()))
await stream.WriteAsync(j1);
var blockBlob3 = blobContainer.GetBlockBlobReference($"MeasuresT3-{i}");
using (var stream = new StreamWriter(await blockBlob3.OpenWriteAsync()))
await stream.WriteAsync(j3);
var blockBlob4 = blobContainer.GetBlockBlobReference($"MeasuresT4-{i}");
using (var stream = new StreamWriter(await blockBlob4.OpenWriteAsync()))
await stream.WriteAsync(j4);
var adHocPolicy = new SharedAccessBlobPolicy()
{
SharedAccessExpiryTime = DateTime.UtcNow.AddHours(24),
Permissions = SharedAccessBlobPermissions.Read | SharedAccessBlobPermissions.Delete
};
Console.WriteLine("+");
using (var ingestClient = KustoIngestFactory.CreateQueuedIngestClient(kustoConnectionStringBuilder))
{
var props = new KustoQueuedIngestionProperties(database, "MeasuresT1")
{
Format = DataSourceFormat.json, JSONMappingReference = "MapMeasureT1"
};
await ingestClient.IngestFromStorageAsync($"{blockBlob1.Uri.AbsoluteUri}{blockBlob1.GetSharedAccessSignature(adHocPolicy).ToString()}", props, new StorageSourceOptions { DeleteSourceOnSuccess = true });
props = new KustoQueuedIngestionProperties(database, "MeasuresT3")
{
Format = DataSourceFormat.json, JSONMappingReference = "MapMeasureT3"
};
await ingestClient.IngestFromStorageAsync($"{blockBlob3.Uri.AbsoluteUri}{blockBlob3.GetSharedAccessSignature(adHocPolicy).ToString()}", props, new StorageSourceOptions { DeleteSourceOnSuccess = true });
props = new KustoQueuedIngestionProperties(database, "MeasuresT4")
{
Format = DataSourceFormat.json, JSONMappingReference = "MapMeasureT4"
};
await ingestClient.IngestFromStorageAsync($"{blockBlob4.Uri.AbsoluteUri}{blockBlob4.GetSharedAccessSignature(adHocPolicy).ToString()}", props, new StorageSourceOptions { DeleteSourceOnSuccess = true });
}
Console.WriteLine("-");
}
}
}
}
public static void Main(string[] args)
{
KustoConnectionStringBuilder readStringBuilder;
KustoConnectionStringBuilder ingestStringBuilder;
readStringBuilder = new KustoConnectionStringBuilder(@"https://kuskusops.kustomfa.windows.net")
{
FederatedSecurity = true,
InitialCatalog = "testdb",
Authority = "72f988bf-86f1-41af-91ab-2d7cd011db47"
};
ingestStringBuilder = new KustoConnectionStringBuilder($"https://ingest-kuskusops.kustomfa.windows.net")
{
FederatedSecurity = true,
InitialCatalog = "testdb",
Authority = "72f988bf-86f1-41af-91ab-2d7cd011db47"
};
using (IKustoIngestClient kustoIngestClient = KustoIngestFactory.CreateQueuedIngestClient(ingestStringBuilder))
{
var kustoIngestionProperties = new KustoQueuedIngestionProperties("testdb", "test_table");
var dataReader = GetDataAsIDataReader();
IKustoIngestionResult ingestionResult =
kustoIngestClient.IngestFromDataReader(dataReader, kustoIngestionProperties);
var ingestionStatus = ingestionResult.GetIngestionStatusCollection().First();
var watch = System.Diagnostics.Stopwatch.StartNew();
var shouldContinue = true;
while ((ingestionStatus.Status == Status.Pending) && (shouldContinue))
{
// Wait a minute...
Thread.Sleep(TimeSpan.FromMinutes(1));
// Try again
ingestionStatus = ingestionResult.GetIngestionStatusBySourceId(ingestionStatus.IngestionSourceId);
shouldContinue = watch.ElapsedMilliseconds < 360000;
}
watch.Stop();
if (ingestionStatus.Status == Status.Pending)
{
// The status of the ingestion did not change.
Console.WriteLine(
"Ingestion with ID:'{0}' did not complete. Timed out after :'{1}' Milliseconds"
.FormatWithInvariantCulture(
ingestionStatus.IngestionSourceId, watch.ElapsedMilliseconds));
}
else
{
// The status of the ingestion has changed
Console.WriteLine(
"Ingestion with ID:'{0}' is complete. Ingestion Status:'{1}'".FormatWithInvariantCulture(
ingestionStatus.IngestionSourceId, ingestionStatus.Status));
}
}
// 4. Show the contents of the table
using (var client = Kusto.Data.Net.Client.KustoClientFactory.CreateCslQueryProvider(readStringBuilder))
{
while (true)
{
var query = "test_table | count";
var reader = client.ExecuteQuery(query);
Kusto.Cloud.Platform.Data.ExtendedDataReader.WriteAsText(reader, "Data ingested into the table:", tabify: true, firstOnly: true);
Console.WriteLine("Press 'r' to retry retrieving data from the table, any other key to quit");
var key = Console.ReadKey();
if (key.KeyChar != 'r' || key.KeyChar != 'R')
{
break;
}
}
}
}
/// <param name="engineConnectionString">Indicates the connection to the Kusto engine service.</param>
/// <param name="dmConnectionString">Indicates the connection to the Kusto data management service.</param>
public static void KustoIngest(string ingestURI, string kustoConnectURI, string engineURI, string dmsURI, string database, string adxtable,
string tableMapping, Stream ingestionStream,
string AppclientID, string AppKey, string key, bool isBatch,
bool isMulti, bool createTables, bool getFromStorage, string blobPath)
{
//var adxtenantId = "<TenantId>";
// var kustoUri = "https://<ClusterName>.<Region>.kusto.windows.net/";
var adxingestUri = ingestURI;// "https://ingest-<ClusterName>.<Region>.kusto.windows.net";
var adxingestConnectionStringBuilder = new KustoConnectionStringBuilder(adxingestUri)
.WithAadApplicationKeyAuthentication(AppclientID, AppKey, key);
var ConnectionStringBuilder = new KustoConnectionStringBuilder(kustoConnectURI)
.WithAadApplicationKeyAuthentication(AppclientID, AppKey, key);
var streamAdxEngineConnectionStringBuilder = new KustoConnectionStringBuilder(kustoConnectURI)
.WithAadApplicationKeyAuthentication(AppclientID, AppKey, key);
var streamAdxDmsURIConnectionStringBuilder = new KustoConnectionStringBuilder(ingestURI)
.WithAadApplicationKeyAuthentication(AppclientID, AppKey, key);
if (createTables)
{
Console.WriteLine("Creating ADX table: " + adxtable);
var createCommand = CreateADXTable(database, adxtable, ConnectionStringBuilder);
Console.WriteLine("Successfully created ADX table: " + adxtable);
CreateTableMapping(database, adxtable, tableMapping, ConnectionStringBuilder);
Console.WriteLine("Successfully created ADX table mapping: " + tableMapping);
}
var kustoClient = KustoClientFactory.CreateCslAdminProvider(ConnectionStringBuilder);
var ingestClient = KustoIngestFactory.CreateQueuedIngestClient(adxingestConnectionStringBuilder);
var streamIngestClient = KustoIngestFactory.
CreateManagedStreamingIngestClient(streamAdxEngineConnectionStringBuilder, streamAdxDmsURIConnectionStringBuilder);
if (isMulti)
{
using (kustoClient)
{
var tablePolicyAlterCommand =
CslCommandGenerator.GenerateTableAlterStreamingIngestionPolicyCommand(adxtable, isEnabled: true);
kustoClient.ExecuteControlCommand(database, tablePolicyAlterCommand);
var command =
CslCommandGenerator.GenerateTableAlterIngestionBatchingPolicyCommand(
database,
adxtable,
new IngestionBatchingPolicy(maximumBatchingTimeSpan: TimeSpan.FromSeconds(2.0),
maximumNumberOfItems: 10000, maximumRawDataSizeMB: 1024));
kustoClient.ExecuteControlCommand(command);
}
}
if (isBatch)
{
using (kustoClient)
{
var command =
CslCommandGenerator.GenerateTableAlterIngestionBatchingPolicyCommand(
database,
adxtable,
new IngestionBatchingPolicy(maximumBatchingTimeSpan: TimeSpan.FromSeconds(2.0),
maximumNumberOfItems: 10000, maximumRawDataSizeMB: 1024));
kustoClient.ExecuteControlCommand(command);
}
}
else
{
if (!isMulti)
{
var tablePolicyAlterCommand =
CslCommandGenerator.GenerateTableAlterStreamingIngestionPolicyCommand(adxtable, isEnabled: true);
kustoClient.ExecuteControlCommand(database, tablePolicyAlterCommand);
}
}
var properties =
new KustoQueuedIngestionProperties(database, adxtable)
{
Format = DataSourceFormat.csv,
IngestionMapping = new IngestionMapping()
{
IngestionMappingReference = tableMapping,
IngestionMappingKind = Kusto.Data.Ingestion.IngestionMappingKind.Csv
},
IgnoreFirstRecord = true
};
//ingestClient.IngestFromStreamAsync(ingestionStream, ingestionProperties: properties).GetAwaiter().GetResult();
if (isBatch)
{
if (getFromStorage)
{
//can occour in stream still!!
ingestClient.IngestFromStorageAsync(blobPath, ingestionProperties: properties).Wait();
Console.WriteLine("Ingestion from blob completed successfully!!");
}
else
{
ingestClient.IngestFromStream(ingestionStream, ingestionProperties: properties).GetIngestionStatusCollection();
Console.WriteLine("Ingestion from stream completed successfully!!");
}
}
else
{
if (getFromStorage)
{
//can occour in stream still!!
streamIngestClient.IngestFromStorageAsync(blobPath, ingestionProperties: properties).Wait();
Console.WriteLine("Ingestion from blob completed successfully!!");
}
else
{
streamIngestClient.IngestFromStream(ingestionStream, ingestionProperties: properties).GetIngestionStatusCollection();
Console.WriteLine("Ingestion from stream completed successfully!!");
}
}
}
static void Main(string[] args)
{
var clusterName = "KustoLab";
var db = "KustoIngestClientDemo";
var table = "Table1";
var mappingName = "Table1_mapping_1";
var serviceNameAndRegion = "clusterNameAndRegion"; // For example, "mycluster.westus"
var authority = "AAD Tenant or name"; // For example, "microsoft.com"
// Set up table
var kcsbEngine =
new KustoConnectionStringBuilder($"https://{serviceNameAndRegion}.kusto.windows.net").WithAadUserPromptAuthentication(authority: $"{authority}");
using (var kustoAdminClient = KustoClientFactory.CreateCslAdminProvider(kcsbEngine))
{
var columns = new List <Tuple <string, string> >()
{
new Tuple <string, string>("Column1", "System.Int64"),
new Tuple <string, string>("Column2", "System.DateTime"),
new Tuple <string, string>("Column3", "System.String"),
};
var command = CslCommandGenerator.GenerateTableCreateCommand(table, columns);
kustoAdminClient.ExecuteControlCommand(databaseName: db, command: command);
// Set up mapping
var columnMappings = new List <JsonColumnMapping>();
columnMappings.Add(new JsonColumnMapping()
{
ColumnName = "Column1", JsonPath = "$.Id"
});
columnMappings.Add(new JsonColumnMapping()
{
ColumnName = "Column2", JsonPath = "$.Timestamp"
});
columnMappings.Add(new JsonColumnMapping()
{
ColumnName = "Column3", JsonPath = "$.Message"
});
command = CslCommandGenerator.GenerateTableJsonMappingCreateCommand(
table, mappingName, columnMappings);
kustoAdminClient.ExecuteControlCommand(databaseName: db, command: command);
}
// Create Ingest Client
var kcsbDM =
new KustoConnectionStringBuilder($"https://ingest-{serviceNameAndRegion}.kusto.windows.net").WithAadUserPromptAuthentication(authority: $"{authority}");
using (var ingestClient = KustoIngestFactory.CreateQueuedIngestClient(kcsbDM))
{
var ingestProps = new KustoQueuedIngestionProperties(db, table);
// For the sake of getting both failure and success notifications we set this to IngestionReportLevel.FailuresAndSuccesses
// Usually the recommended level is IngestionReportLevel.FailuresOnly
ingestProps.ReportLevel = IngestionReportLevel.FailuresAndSuccesses;
ingestProps.ReportMethod = IngestionReportMethod.Queue;
// Setting FlushImmediately to 'true' overrides any aggregation preceding the ingestion.
// Not recommended unless you are certain you know what you are doing
ingestProps.FlushImmediately = true;
ingestProps.JSONMappingReference = mappingName;
ingestProps.Format = DataSourceFormat.json;
// Prepare data for ingestion
using (var memStream = new MemoryStream())
using (var writer = new StreamWriter(memStream))
{
for (int counter = 1; counter <= 10; ++counter)
{
writer.WriteLine(
"{{ \"Id\":\"{0}\", \"Timestamp\":\"{1}\", \"Message\":\"{2}\" }}",
counter, DateTime.UtcNow.AddSeconds(100 * counter),
$"This is a dummy message number {counter}");
}
writer.Flush();
memStream.Seek(0, SeekOrigin.Begin);
// Post ingestion message
var res = ingestClient.IngestFromStreamAsync(memStream, ingestProps, leaveOpen: true);
}
// Wait a bit (20s) and retrieve all notifications:
Thread.Sleep(20000);
var errors = ingestClient.GetAndDiscardTopIngestionFailures().GetAwaiter().GetResult();
var successes = ingestClient.GetAndDiscardTopIngestionSuccesses().GetAwaiter().GetResult();
errors.ForEach((f) => { Console.WriteLine($"Ingestion error: {f.Info.Details}"); });
successes.ForEach((s) => { Console.WriteLine($"Ingested: {s.Info.IngestionSourcePath}"); });
}
}
