private async Task CleanQueue(bool checkEntireQueueOnCleanup)
{
long totalCleanupCount = 0;
long totalCleanupStoreCount = 0;
while (true)
{
foreach (KeyValuePair <string, ISequentialStore <MessageRef> > endpointSequentialStore in this.messageStore.endpointSequentialStores)
{
var messageQueueId = endpointSequentialStore.Key;
try
{
if (this.cancellationTokenSource.IsCancellationRequested)
{
return;
}
var(endpointId, priority) = MessageQueueIdHelper.ParseMessageQueueId(messageQueueId);
Events.CleanupTaskStarted(messageQueueId);
CheckpointData checkpointData = await this.messageStore.checkpointStore.GetCheckpointDataAsync(messageQueueId, CancellationToken.None);
ISequentialStore <MessageRef> sequentialStore = endpointSequentialStore.Value;
Events.CleanupCheckpointState(messageQueueId, checkpointData);
int cleanupEntityStoreCount = 0;
// If checkEntireQueueOnCleanup is set to false, we only peek the head, message counts is tailOffset-headOffset+1
// otherwise count while iterating over the queue.
var headOffset = 0L;
var tailOffset = sequentialStore.GetTailOffset(CancellationToken.None);
var messageCount = 0L;
async Task <bool> DeleteMessageCallback(long offset, MessageRef messageRef)
{
var expiry = messageRef.TimeStamp + messageRef.TimeToLive;
if (offset > checkpointData.Offset && expiry > DateTime.UtcNow)
{
// message is not sent and not expired, increase message counts
messageCount++;
return(false);
}
headOffset = Math.Max(headOffset, offset);
bool deleteMessage = false;
// Decrement ref count.
var message = await this.messageStore.messageEntityStore.Update(
messageRef.EdgeMessageId,
m =>
{
if (m.RefCount > 0)
{
m.RefCount--;
}
if (m.RefCount == 0)
{
deleteMessage = true;
}
return(m);
});
if (deleteMessage)
{
if (offset > checkpointData.Offset && expiry <= DateTime.UtcNow)
{
this.expiredCounter.Increment(1, new[] { "ttl_expiry", message?.Message.GetSenderId(), message?.Message.GetOutput(), bool.TrueString });
}
await this.messageStore.messageEntityStore.Remove(messageRef.EdgeMessageId);
cleanupEntityStoreCount++;
}
return(true);
}
// With the addition of PriorityQueues, the CleanupProcessor assumptions change slightly:
// Previously, we could always assume that if a message at the head of the queue should not be deleted,
// then none of the other messages in the queue should be either. Now, because we can have different TTL's
// for messages within the same queue, there can be messages that have expired in the queue after the head.
// The checkEntireQueueOnCleanup flag is an environment variable for edgeHub. If it is set to true, we will
// check the entire queue every time cleanup processor runs. If it is set to false, we just remove the oldest
// items in the queue until we get to one that is not expired.
int cleanupCount = 0;
if (checkEntireQueueOnCleanup)
{
IEnumerable <(long, MessageRef)> batch;
long offset = sequentialStore.GetHeadOffset(this.cancellationTokenSource.Token);
do
{
batch = await sequentialStore.GetBatch(offset, CleanupBatchSize);
foreach ((long, MessageRef)messageWithOffset in batch)
{
if (await sequentialStore.RemoveOffset(DeleteMessageCallback, messageWithOffset.Item1, this.cancellationTokenSource.Token))
{
cleanupCount++;
}
}
offset += CleanupBatchSize;
}while (batch.Any());
}
else
{
while (await sequentialStore.RemoveFirst(DeleteMessageCallback))
{
cleanupCount++;
}
messageCount = tailOffset - headOffset + 1;
}
// update Metrics for message counts
Checkpointer.Metrics.QueueLength.Set(messageCount, new[] { endpointId, priority.ToString(), bool.TrueString });
totalCleanupCount += cleanupCount;
totalCleanupStoreCount += cleanupEntityStoreCount;
Events.CleanupCompleted(messageQueueId, cleanupCount, cleanupEntityStoreCount, totalCleanupCount, totalCleanupStoreCount);
await Task.Delay(MinCleanupSleepTime, this.cancellationTokenSource.Token);
}
catch (Exception ex)
{
Events.ErrorCleaningMessagesForEndpoint(ex, messageQueueId);
}
}
await Task.Delay(this.GetCleanupTaskSleepTime());
}
}
private async Task CleanQueue(bool checkEntireQueueOnCleanup)
{
long totalCleanupCount = 0;
long totalCleanupStoreCount = 0;
while (true)
{
foreach (KeyValuePair <string, ISequentialStore <MessageRef> > endpointSequentialStore in this.messageStore.endpointSequentialStores)
{
try
{
if (this.cancellationTokenSource.IsCancellationRequested)
{
return;
}
Events.CleanupTaskStarted(endpointSequentialStore.Key);
CheckpointData checkpointData = await this.messageStore.checkpointStore.GetCheckpointDataAsync(endpointSequentialStore.Key, CancellationToken.None);
ISequentialStore <MessageRef> sequentialStore = endpointSequentialStore.Value;
Events.CleanupCheckpointState(endpointSequentialStore.Key, checkpointData);
int cleanupEntityStoreCount = 0;
async Task <bool> DeleteMessageCallback(long offset, MessageRef messageRef)
{
if (checkpointData.Offset < offset &&
DateTime.UtcNow - messageRef.TimeStamp < messageRef.TimeToLive)
{
return(false);
}
bool deleteMessage = false;
// Decrement ref count.
await this.messageStore.messageEntityStore.Update(
messageRef.EdgeMessageId,
m =>
{
if (m.RefCount > 0)
{
m.RefCount--;
}
if (m.RefCount == 0)
{
deleteMessage = true;
}
return(m);
});
if (deleteMessage)
{
await this.messageStore.messageEntityStore.Remove(messageRef.EdgeMessageId);
cleanupEntityStoreCount++;
}
return(true);
}
// With the addition of PriorityQueues, the CleanupProcessor assumptions change slightly:
// Previously, we could always assume that if a message at the head of the queue should not be deleted,
// then none of the other messages in the queue should be either. Now, because we can have different TTL's
// for messages within the same queue, there can be messages that have expired in the queue after the head.
// The checkEntireQueueOnCleanup flag is an environment variable for edgeHub. If it is set to true, we will
// check the entire queue every time cleanup processor runs. If it is set to false, we just remove the oldest
// items in the queue until we get to one that is not expired.
int cleanupCount = 0;
if (checkEntireQueueOnCleanup)
{
IEnumerable <(long, MessageRef)> batch;
long offset = sequentialStore.GetHeadOffset(this.cancellationTokenSource.Token);
do
{
batch = await sequentialStore.GetBatch(offset, CleanupBatchSize);
foreach ((long, MessageRef)messageWithOffset in batch)
{
if (await sequentialStore.RemoveOffset(DeleteMessageCallback, messageWithOffset.Item1, this.cancellationTokenSource.Token))
{
cleanupCount++;
}
}
offset = offset + CleanupBatchSize;
}while (batch.Any());
}
else
{
while (await sequentialStore.RemoveFirst(DeleteMessageCallback))
{
cleanupCount++;
}
}
totalCleanupCount += cleanupCount;
totalCleanupStoreCount += cleanupEntityStoreCount;
Events.CleanupCompleted(endpointSequentialStore.Key, cleanupCount, cleanupEntityStoreCount, totalCleanupCount, totalCleanupStoreCount);
await Task.Delay(MinCleanupSleepTime, this.cancellationTokenSource.Token);
}
catch (Exception ex)
{
Events.ErrorCleaningMessagesForEndpoint(ex, endpointSequentialStore.Key);
}
}
await Task.Delay(this.GetCleanupTaskSleepTime());
}
}