private void PurgeMessages(string tableName)
{
Log.Info($"dropping ingestion messages for table:{tableName}");
while (true)
{
IEnumerable <CloudQueueMessage> successes = PopTopMessagesFromQueue(Endpoint.IngestionResources.SuccessNotificationsQueue);
foreach (CloudQueueMessage success in successes)
{
KustoSuccessMessage message = JsonConvert.DeserializeObject <KustoSuccessMessage>(success.AsString);
Log.Debug("success:", message);
if (message.Table.Equals(tableName))
{
RemoveMessageFromQueue(Endpoint.IngestionResources.SuccessNotificationsQueue, success);
}
}
if (successes.Count() < _maxMessageCount)
{
break;
}
}
while (true)
{
IEnumerable <CloudQueueMessage> errors = PopTopMessagesFromQueue(Endpoint.IngestionResources.FailureNotificationsQueue);
foreach (CloudQueueMessage error in errors)
{
KustoSuccessMessage message = JsonConvert.DeserializeObject <KustoSuccessMessage>(error.AsString);
Log.Debug("error:", message);
if (message.Table.Equals(tableName))
{
RemoveMessageFromQueue(Endpoint.IngestionResources.FailureNotificationsQueue, error);
}
}
if (errors.Count() < _maxMessageCount)
{
break;
}
}
}
private void QueueMessageMonitor()
{
// read success notifications
while (true)
{
IEnumerable <CloudQueueMessage> successes = PopTopMessagesFromQueue(Endpoint.IngestionResources.SuccessNotificationsQueue);
foreach (CloudQueueMessage success in successes)
{
KustoSuccessMessage message = JsonConvert.DeserializeObject <KustoSuccessMessage>(success.AsString);
Log.Debug("success:", message);
if (_messageList.Exists(x => x.Equals(message.IngestionSourceId)))
{
_messageList.Remove(message.IngestionSourceId);
RemoveMessageFromQueue(Endpoint.IngestionResources.SuccessNotificationsQueue, success);
_totalBlobIngestResults++;
Log.Info($"Ingestion completed total:({_totalBlobIngestResults}/{_totalBlobIngestQueued}): {JsonConvert.DeserializeObject(success.AsString)}", ConsoleColor.Green);
}
else if (message.SucceededOn + _messageTimeToLive < DateTime.Now)
{
// remove sas key
message.IngestionSourcePath = Regex.Replace(message.IngestionSourcePath, @"\?(.*)", "");
Log.Warning($"cleaning stale message", message);
RemoveMessageFromQueue(Endpoint.IngestionResources.SuccessNotificationsQueue, success);
}
}
if (successes.Count() < _maxMessageCount)
{
break;
}
}
while (true)
{
// read failure notifications
IEnumerable <CloudQueueMessage> errors = PopTopMessagesFromQueue(Endpoint.IngestionResources.FailureNotificationsQueue);
foreach (CloudQueueMessage error in errors)
{
KustoErrorMessage message = JsonConvert.DeserializeObject <KustoErrorMessage>(error.AsString);
Log.Debug("error:", message);
if (_messageList.Exists(x => x.Equals(message.IngestionSourceId)))
{
_messageList.Remove(message.IngestionSourceId);
RemoveMessageFromQueue(Endpoint.IngestionResources.FailureNotificationsQueue, error);
_totalBlobIngestResults++;
_failureCount++;
Log.Error($"Ingestion error total:({_failureCount}): {JsonConvert.DeserializeObject(error.AsString)}");
}
else if (message.FailedOn + _messageTimeToLive < DateTime.Now)
{
Log.Warning($"cleaning stale message", message);
RemoveMessageFromQueue(Endpoint.IngestionResources.FailureNotificationsQueue, error);
}
}
if (errors.Count() < _maxMessageCount)
{
break;
}
}
}
private void QueueMessageMonitor()
{
// read success notifications
while (true)
{
IEnumerable <CloudQueueMessage> successes = PopTopMessagesFromQueue(Endpoint.IngestionResources.SuccessNotificationsQueue);
foreach (CloudQueueMessage success in successes)
{
KustoSuccessMessage message = JsonConvert.DeserializeObject <KustoSuccessMessage>(success.AsString);
Log.Debug("success:", message);
FileObject fileObject = _instance.FileObjects.FindByMessageId(message.IngestionSourceId);
if (fileObject.IsPopulated)
{
fileObject.Status = FileStatus.succeeded;
RemoveMessageFromQueue(Endpoint.IngestionResources.SuccessNotificationsQueue, success);
Log.Info($"Ingestion completed total:({_instance.FileObjects.Count()}/{_instance.FileObjects.Count(FileStatus.uploading)}): {JsonConvert.DeserializeObject(success.AsString)}", ConsoleColor.Green);
}
else if (message.SucceededOn + _messageTimeToLive < DateTime.Now)
{
// remove sas key
message.IngestionSourcePath = Regex.Replace(message.IngestionSourcePath, @"\?(.*)", "");
Log.Warning($"cleaning stale message", message);
RemoveMessageFromQueue(Endpoint.IngestionResources.SuccessNotificationsQueue, success);
}
}
if (successes.Count() < _maxMessageCount)
{
break;
}
}
while (true)
{
// read failure notifications
IEnumerable <CloudQueueMessage> errors = PopTopMessagesFromQueue(Endpoint.IngestionResources.FailureNotificationsQueue);
foreach (CloudQueueMessage error in errors)
{
KustoErrorMessage message = JsonConvert.DeserializeObject <KustoErrorMessage>(error.AsString);
Log.Debug("error:", message);
FileObject fileObject = _instance.FileObjects.FindByMessageId(message.IngestionSourceId);
if (fileObject.IsPopulated)
{
fileObject.Status = FileStatus.failed;
RemoveMessageFromQueue(Endpoint.IngestionResources.FailureNotificationsQueue, error);
Log.Error($"Ingestion error total:({_instance.FileObjects.Count(FileStatus.failed)}): {JsonConvert.DeserializeObject(error.AsString)}");
}
else if (message.FailedOn + _messageTimeToLive < DateTime.Now)
{
Log.Warning($"cleaning stale message", message);
RemoveMessageFromQueue(Endpoint.IngestionResources.FailureNotificationsQueue, error);
}
}
if (errors.Count() < _maxMessageCount)
{
break;
}
}
}