public KustoIngestionMessage PrepareIngestionMessage(string blobUriWithSas, long blobSizeBytes, string ingestionMapping)
{
string id = Guid.NewGuid().ToString();
KustoIngestionMessage message = new KustoIngestionMessage()
{
Id = id,
BlobPath = blobUriWithSas,
RawDataSize = Convert.ToInt32(blobSizeBytes),
DatabaseName = Endpoint.DatabaseName,
TableName = Endpoint.TableName,
RetainBlobOnSuccess = !Config.KustoUseBlobAsSource,
Format = FileExtensionTypesEnum.csv.ToString(),
FlushImmediately = true,
ReportLevel = Config.KustoUseIngestMessage ? 2 : 1, //(int)IngestionReportLevel.FailuresAndSuccesses, // 2 FailuresAndSuccesses, 0 failures, 1 none
ReportMethod = Convert.ToInt32(!Config.KustoUseIngestMessage), //(int)IngestionReportMethod.Table, // 0 queue, 1 table, 2 both
AdditionalProperties = new KustoAdditionalProperties()
{
authorizationContext = Endpoint.IdentityToken,
compressed = Config.KustoCompressed,
csvMapping = ingestionMapping
}
};
return(message);
}
public void PostMessageToQueue(string queueUriWithSas, KustoIngestionMessage message, FileObject fileObject)
{
Log.Info($"post: {queueUriWithSas}", ConsoleColor.Magenta);
_totalBlobIngestQueued++;
CloudQueue queue = new CloudQueue(new Uri(queueUriWithSas));
CloudQueueMessage queueMessage = new CloudQueueMessage(JsonConvert.SerializeObject(message));
OperationContext context = new OperationContext()
{
ClientRequestID = message.Id,
};
queue.AddMessage(queueMessage, _messageTimeToLive, null, null, context);
if (Config.KustoUseIngestMessage)
{
_messageList.Add(message.Id);
}
else
{
_messageList.Add(fileObject.RelativeUri);
}
Log.Info($"queue message id: {message.Id}");
}
private void PostMessageToQueue(string queueUriWithSas, KustoIngestionMessage message, FileObject fileObject)
{
Log.Info($"post: {queueUriWithSas ?? "(null ingest uri)"}", ConsoleColor.Magenta);
CloudQueue queue = new CloudQueue(new Uri(queueUriWithSas));
CloudQueueMessage queueMessage = new CloudQueueMessage(JsonConvert.SerializeObject(message));
OperationContext context = new OperationContext()
{
ClientRequestID = message.Id
};
queue.AddMessage(queueMessage, _messageTimeToLive, null, null, context);
fileObject.Status = FileStatus.uploading;
fileObject.MessageId = message.Id;
Log.Info($"fileobject uploading FileUri:{fileObject.FileUri} RelativeUri: {fileObject.RelativeUri} message id: {message.Id}", ConsoleColor.Cyan);
}
private void PostMessageToQueue(string queueUriWithSas, KustoIngestionMessage message, FileObject fileObject)
{
Log.Info($"post: {queueUriWithSas}", ConsoleColor.Magenta);
_totalBlobIngestQueued++;
CloudQueue queue = new CloudQueue(new Uri(queueUriWithSas));
CloudQueueMessage queueMessage = new CloudQueueMessage(JsonConvert.SerializeObject(message));
OperationContext context = new OperationContext()
{
ClientRequestID = message.Id,
};
queue.AddMessage(queueMessage, _messageTimeToLive, null, null, context);
IngestFileObjectsPending.Add(fileObject.FileUri, fileObject.RelativeUri, message.Id);
Log.Info($"IngestFileObjectsPending.Add fileobject to pending queue FileUri: {fileObject.FileUri} RelativeUri: {fileObject.RelativeUri} message id: {message.Id}");
}
