public KustoIngestionMessage PrepareIngestionMessage(string blobUriWithSas, long blobSizeBytes, string ingestionMapping) { string id = Guid.NewGuid().ToString(); KustoIngestionMessage message = new KustoIngestionMessage() { Id = id, BlobPath = blobUriWithSas, RawDataSize = Convert.ToInt32(blobSizeBytes), DatabaseName = Endpoint.DatabaseName, TableName = Endpoint.TableName, RetainBlobOnSuccess = !Config.KustoUseBlobAsSource, Format = FileExtensionTypesEnum.csv.ToString(), FlushImmediately = true, ReportLevel = Config.KustoUseIngestMessage ? 2 : 1, //(int)IngestionReportLevel.FailuresAndSuccesses, // 2 FailuresAndSuccesses, 0 failures, 1 none ReportMethod = Convert.ToInt32(!Config.KustoUseIngestMessage), //(int)IngestionReportMethod.Table, // 0 queue, 1 table, 2 both AdditionalProperties = new KustoAdditionalProperties() { authorizationContext = Endpoint.IdentityToken, compressed = Config.KustoCompressed, csvMapping = ingestionMapping } }; return(message); }
public void PostMessageToQueue(string queueUriWithSas, KustoIngestionMessage message, FileObject fileObject) { Log.Info($"post: {queueUriWithSas}", ConsoleColor.Magenta); _totalBlobIngestQueued++; CloudQueue queue = new CloudQueue(new Uri(queueUriWithSas)); CloudQueueMessage queueMessage = new CloudQueueMessage(JsonConvert.SerializeObject(message)); OperationContext context = new OperationContext() { ClientRequestID = message.Id, }; queue.AddMessage(queueMessage, _messageTimeToLive, null, null, context); if (Config.KustoUseIngestMessage) { _messageList.Add(message.Id); } else { _messageList.Add(fileObject.RelativeUri); } Log.Info($"queue message id: {message.Id}"); }
private void PostMessageToQueue(string queueUriWithSas, KustoIngestionMessage message, FileObject fileObject) { Log.Info($"post: {queueUriWithSas ?? "(null ingest uri)"}", ConsoleColor.Magenta); CloudQueue queue = new CloudQueue(new Uri(queueUriWithSas)); CloudQueueMessage queueMessage = new CloudQueueMessage(JsonConvert.SerializeObject(message)); OperationContext context = new OperationContext() { ClientRequestID = message.Id }; queue.AddMessage(queueMessage, _messageTimeToLive, null, null, context); fileObject.Status = FileStatus.uploading; fileObject.MessageId = message.Id; Log.Info($"fileobject uploading FileUri:{fileObject.FileUri} RelativeUri: {fileObject.RelativeUri} message id: {message.Id}", ConsoleColor.Cyan); }
private void PostMessageToQueue(string queueUriWithSas, KustoIngestionMessage message, FileObject fileObject) { Log.Info($"post: {queueUriWithSas}", ConsoleColor.Magenta); _totalBlobIngestQueued++; CloudQueue queue = new CloudQueue(new Uri(queueUriWithSas)); CloudQueueMessage queueMessage = new CloudQueueMessage(JsonConvert.SerializeObject(message)); OperationContext context = new OperationContext() { ClientRequestID = message.Id, }; queue.AddMessage(queueMessage, _messageTimeToLive, null, null, context); IngestFileObjectsPending.Add(fileObject.FileUri, fileObject.RelativeUri, message.Id); Log.Info($"IngestFileObjectsPending.Add fileobject to pending queue FileUri: {fileObject.FileUri} RelativeUri: {fileObject.RelativeUri} message id: {message.Id}"); }