private static Task TriggerIngestCommand(DateTime insertDate, string urlOfToBeInsertedBlob, long rawDatasize)
{
var ingestProperties = new KustoIngestionProperties(GetEnvVariable("KustoDatabase"), GetEnvVariable("KustoTableName"));
switch (GetEnvVariable("KustoMappingType"))
{
case "json":
ingestProperties.JSONMappingReference = GetEnvVariable("KustoMappingRef");
break;
case "csv":
ingestProperties.CSVMappingReference = GetEnvVariable("KustoMappingRef");
break;
case "avro":
ingestProperties.AvroMappingReference = GetEnvVariable("KustoMappingRef");
break;
default:
ingestProperties.JSONMappingReference = GetEnvVariable("KustoMappingRef");
break;
}
ingestProperties.AdditionalProperties.Add(CREATIONDATEKEY, insertDate.ToString());
ingestProperties.AdditionalTags = new List <String> {
insertDate.ToString()
};
StorageSourceOptions sourceOptions = new StorageSourceOptions();
sourceOptions.Size = rawDatasize;
sourceOptions.DeleteSourceOnSuccess = Boolean.Parse(GetEnvVariable("DeleteAfterInsert"));
return(adx.IngestFromStorageAsync(urlOfToBeInsertedBlob + Environment.GetEnvironmentVariable("SasToken"), ingestProperties, sourceOptions));
}
static async Task Main(string[] args)
{
// Async Ingestion From a Single Azure Blob using KustoQueuedIngestClient with (optional) RetryPolicy:
//Create Kusto connection string with App Authentication
var kustoConnectionStringBuilderDM =
new KustoConnectionStringBuilder(@"https://ingest-{clusterNameAndRegion}.kusto.windows.net").WithAadApplicationKeyAuthentication(
applicationClientId: "{Application Client ID}",
applicationKey: "{Application Key (secret)}",
authority: "{AAD TenantID or name}");
// Create an ingest client
// Note, that creating a separate instance per ingestion operation is an anti-pattern.
// IngestClient classes are thread-safe and intended for reuse
IKustoIngestClient client = KustoIngestFactory.CreateQueuedIngestClient(kustoConnectionStringBuilderDM);
// Ingest from blobs according to the required properties
var kustoIngestionProperties = new KustoIngestionProperties(databaseName: "myDB", tableName: "myTable");
var sourceOptions = new StorageSourceOptions()
{
DeleteSourceOnSuccess = true
};
//// Create your custom implementation of IRetryPolicy, which will affect how the ingest client handles retrying on transient failures
IRetryPolicy retryPolicy = new NoRetry();
//// This line sets the retry policy on the ingest client that will be enforced on every ingest call from here on
((IKustoQueuedIngestClient)client).QueueRetryPolicy = retryPolicy;
await client.IngestFromStorageAsync(uri : @"BLOB-URI-WITH-SAS-KEY", ingestionProperties : kustoIngestionProperties, sourceOptions);
client.Dispose();
}
static async Task Main(string[] args)
{
// Ingest From a Local File using KustoQueuedIngestClient and report status to a table
// Create Kusto connection string with App Authentication
var kustoConnectionStringBuilderDM =
new KustoConnectionStringBuilder(@"https://ingest-{clusterNameAndRegion}.kusto.windows.net").WithAadApplicationKeyAuthentication(
applicationClientId: "{Application Client ID}",
applicationKey: "{Application Key (secret)}",
authority: "{AAD TenantID or name}");
// Create a disposable client that will execute the ingestion
IKustoQueuedIngestClient client = KustoIngestFactory.CreateQueuedIngestClient(kustoConnectionStringBuilderDM);
// Ingest from a file according to the required properties
var kustoIngestionProperties = new KustoQueuedIngestionProperties(databaseName: "myDB", tableName: "myDB")
{
// Setting the report level to FailuresAndSuccesses will cause both successful and failed ingestions to be reported
// (Rather than the default "FailuresOnly" level)
ReportLevel = IngestionReportLevel.FailuresAndSuccesses,
// Choose the report method of choice
ReportMethod = IngestionReportMethod.Table
};
var filePath = @"< Path to file >";
var fileIdentifier = Guid.NewGuid();
var fileDescription = new FileDescription()
{
FilePath = filePath, SourceId = fileIdentifier
};
var sourceOptions = new StorageSourceOptions()
{
SourceId = fileDescription.SourceId.Value
};
// Execute the ingest operation and save the result.
var clientResult = await client.IngestFromStorageAsync(fileDescription.FilePath,
ingestionProperties : kustoIngestionProperties, sourceOptions);
// Use the fileIdentifier you supplied to get the status of your ingestion
var ingestionStatus = clientResult.GetIngestionStatusBySourceId(fileIdentifier);
while (ingestionStatus.Status == Status.Pending)
{
// Wait a minute...
Thread.Sleep(TimeSpan.FromMinutes(1));
// Try again
ingestionStatus = clientResult.GetIngestionStatusBySourceId(fileIdentifier);
}
// Verify the results of the ingestion
Ensure.ConditionIsMet(ingestionStatus.Status == Status.Succeeded,
"The file should have been ingested successfully");
// Dispose of the client
client.Dispose();
}
public Task <IKustoIngestionResult> IngestFromStorageAsync(string uri, KustoIngestionProperties ingestionProperties, StorageSourceOptions sourceOptions = null)
{
return(_successfulResultInstance);
}
