public async Task Invoke(IMessage message, uint priority, uint timeToLiveSecs)
{
try
{
if (this.closed)
{
throw new InvalidOperationException($"Endpoint executor for endpoint {this.Endpoint} is closed.");
}
// Get the checkpointer corresponding to the queue for this priority
ImmutableDictionary <uint, EndpointExecutorFsm> snapshot = this.prioritiesToFsms;
ICheckpointer checkpointer = snapshot[priority].Checkpointer;
IMessage storedMessage = await this.messageStore.Add(MessageQueueIdHelper.GetMessageQueueId(this.Endpoint.Id, priority), message, timeToLiveSecs);
checkpointer.Propose(storedMessage);
Events.AddMessageSuccess(this, storedMessage.Offset, priority, timeToLiveSecs);
this.hasMessagesInQueue.Set();
}
catch (Exception ex)
{
Routing.UserMetricLogger.LogIngressFailureMetric(1, this.Endpoint.IotHubName, message, "storage_failure");
Events.AddMessageFailure(this, ex);
throw;
}
}
public void TestGetMessageQueueIdWithDefaultPriority()
{
// Message queue with default priority should remain as endpoint id
var endpointId = Guid.NewGuid().ToString();
var messageQueueId = MessageQueueIdHelper.GetMessageQueueId(endpointId, Core.RouteFactory.DefaultPriority);
Assert.Equal(endpointId, messageQueueId);
}
public async Task UpdatePriorities(IList <uint> priorities, Option <Endpoint> newEndpoint)
{
Preconditions.CheckArgument(priorities.Count > 0);
Events.UpdatePriorities(this, priorities);
if (this.closed)
{
throw new InvalidOperationException($"Endpoint executor for endpoint {this.Endpoint} is closed.");
}
// Update priorities by merging the new ones with the existing.
// We don't ever remove stale priorities, otherwise stored messages
// pending for a removed priority will never get sent.
ImmutableDictionary <uint, EndpointExecutorFsm> snapshot = this.prioritiesToFsms;
Dictionary <uint, EndpointExecutorFsm> updatedSnapshot = new Dictionary <uint, EndpointExecutorFsm>(snapshot);
foreach (uint priority in priorities)
{
if (!updatedSnapshot.ContainsKey(priority))
{
string id = MessageQueueIdHelper.GetMessageQueueId(this.Endpoint.Id, priority);
// Create a message queue in the store for every priority we have
await this.messageStore.AddEndpoint(id);
// Create a checkpointer and a FSM for every message queue
ICheckpointer checkpointer = await this.checkpointerFactory.CreateAsync(id, this.Endpoint.Id, priority);
EndpointExecutorFsm fsm = new EndpointExecutorFsm(this.Endpoint, checkpointer, this.config);
// Add it to our dictionary
updatedSnapshot.Add(priority, fsm);
}
else
{
// Update the existing FSM with the new endpoint
EndpointExecutorFsm fsm = updatedSnapshot[priority];
await newEndpoint.ForEachAsync(e => fsm.RunAsync(Commands.UpdateEndpoint(e)));
}
}
if (this.prioritiesToFsms.CompareAndSet(snapshot, updatedSnapshot.ToImmutableDictionary()))
{
Events.UpdatePrioritiesSuccess(this, updatedSnapshot.Keys.ToList());
// Update the lastUsedFsm to be the initial one, we always
// have at least one priority->FSM pair at this point.
this.lastUsedFsm = updatedSnapshot[priorities[0]];
}
else
{
Events.UpdatePrioritiesFailure(this, updatedSnapshot.Keys.ToList());
}
}
public void TestGetMessageQueueIdWithNonDefaultPriority()
{
// Message queue with non-default priority should combain endpoint id and priority with MessageQueueIdHelper.MessageQueueIdDelimiter
var endpointId = Guid.NewGuid().ToString();
uint priority = 1234;
var expectMessageQueueId = $"{endpointId}{MessageQueueIdHelper.MessageQueueIdDelimiter}{priority}";
var messageQueueId = MessageQueueIdHelper.GetMessageQueueId(endpointId, priority);
Assert.Equal(expectMessageQueueId, messageQueueId);
}
public void TestGetMessageQueueId()
{
// Message queue with non-default priority should combain endpoint id and priority with MessageQueueIdHelper.MessageQueueIdDelimiter
var endpointId = "abc";
uint priority = 1234;
var expectMessageQueueId = "abc_Pri1234";
var messageQueueId = MessageQueueIdHelper.GetMessageQueueId(endpointId, priority);
Assert.Equal(expectMessageQueueId, messageQueueId);
}
public void TestParseMessageQueueIdWithDefaultPriorityAndInvalidSuffixInEndpointId()
{
// Endpoint id with delimiter should be ignored
var expectedEndpointId = $"{Guid.NewGuid()}{MessageQueueIdHelper.MessageQueueIdDelimiter}4321a";
var messageQueueId = expectedEndpointId;
uint expectedPriority = Core.RouteFactory.DefaultPriority;
var(endpointId, priority) = MessageQueueIdHelper.ParseMessageQueueId(messageQueueId);
Assert.Equal(expectedEndpointId, endpointId);
Assert.Equal(expectedPriority, priority);
}
public void TestParseMessageQueueIdWithPriorityAndDelimiterInEndpointId()
{
// Endpoint id with delimiter should be ignored
var expectedEndpointId = $"{Guid.NewGuid()}{MessageQueueIdHelper.MessageQueueIdDelimiter}4321";
uint expectedPriority = 1234;
var messageQueueId = $"{expectedEndpointId}{MessageQueueIdHelper.MessageQueueIdDelimiter}{expectedPriority}";
var(endpointId, priority) = MessageQueueIdHelper.ParseMessageQueueId(messageQueueId);
Assert.Equal(expectedEndpointId, endpointId);
Assert.Equal(expectedPriority, priority);
}
public void TestParseMessageQueueIdWithPriority()
{
// Message queue id with priority should return its priority
var expectedEndpointId = Guid.NewGuid().ToString();
uint expectedPriority = 1234;
var messageQueueId = $"{expectedEndpointId}{MessageQueueIdHelper.MessageQueueIdDelimiter}{expectedPriority}";
var(endpointId, priority) = MessageQueueIdHelper.ParseMessageQueueId(messageQueueId);
Assert.Equal(expectedEndpointId, endpointId);
Assert.Equal(expectedPriority, priority);
}
public void TestParseMessageQueueIdWithoutPriority()
{
// Message queue id without priority should return default priority
var expectedEndpointId = Guid.NewGuid().ToString();
var messageQueueId = expectedEndpointId;
var expectedPriority = Core.RouteFactory.DefaultPriority;
var(endpointId, priority) = MessageQueueIdHelper.ParseMessageQueueId(messageQueueId);
Assert.Equal(expectedEndpointId, endpointId);
Assert.Equal(expectedPriority, priority);
}
private async Task CleanQueue(bool checkEntireQueueOnCleanup)
{
long totalCleanupCount = 0;
long totalCleanupStoreCount = 0;
while (true)
{
foreach (KeyValuePair <string, ISequentialStore <MessageRef> > endpointSequentialStore in this.messageStore.endpointSequentialStores)
{
var messageQueueId = endpointSequentialStore.Key;
try
{
if (this.cancellationTokenSource.IsCancellationRequested)
{
return;
}
var(endpointId, priority) = MessageQueueIdHelper.ParseMessageQueueId(messageQueueId);
Events.CleanupTaskStarted(messageQueueId);
CheckpointData checkpointData = await this.messageStore.checkpointStore.GetCheckpointDataAsync(messageQueueId, CancellationToken.None);
ISequentialStore <MessageRef> sequentialStore = endpointSequentialStore.Value;
Events.CleanupCheckpointState(messageQueueId, checkpointData);
int cleanupEntityStoreCount = 0;
// If checkEntireQueueOnCleanup is set to false, we only peek the head, message counts is tailOffset-headOffset+1
// otherwise count while iterating over the queue.
var headOffset = 0L;
var tailOffset = sequentialStore.GetTailOffset(CancellationToken.None);
var messageCount = 0L;
async Task <bool> DeleteMessageCallback(long offset, MessageRef messageRef)
{
var expiry = messageRef.TimeStamp + messageRef.TimeToLive;
if (offset > checkpointData.Offset && expiry > DateTime.UtcNow)
{
// message is not sent and not expired, increase message counts
messageCount++;
return(false);
}
headOffset = Math.Max(headOffset, offset);
bool deleteMessage = false;
// Decrement ref count.
var message = await this.messageStore.messageEntityStore.Update(
messageRef.EdgeMessageId,
m =>
{
if (m.RefCount > 0)
{
m.RefCount--;
}
if (m.RefCount == 0)
{
deleteMessage = true;
}
return(m);
});
if (deleteMessage)
{
if (offset > checkpointData.Offset && expiry <= DateTime.UtcNow)
{
this.expiredCounter.Increment(1, new[] { "ttl_expiry", message?.Message.GetSenderId(), message?.Message.GetOutput(), bool.TrueString });
}
await this.messageStore.messageEntityStore.Remove(messageRef.EdgeMessageId);
cleanupEntityStoreCount++;
}
return(true);
}
// With the addition of PriorityQueues, the CleanupProcessor assumptions change slightly:
// Previously, we could always assume that if a message at the head of the queue should not be deleted,
// then none of the other messages in the queue should be either. Now, because we can have different TTL's
// for messages within the same queue, there can be messages that have expired in the queue after the head.
// The checkEntireQueueOnCleanup flag is an environment variable for edgeHub. If it is set to true, we will
// check the entire queue every time cleanup processor runs. If it is set to false, we just remove the oldest
// items in the queue until we get to one that is not expired.
int cleanupCount = 0;
if (checkEntireQueueOnCleanup)
{
IEnumerable <(long, MessageRef)> batch;
long offset = sequentialStore.GetHeadOffset(this.cancellationTokenSource.Token);
do
{
batch = await sequentialStore.GetBatch(offset, CleanupBatchSize);
foreach ((long, MessageRef)messageWithOffset in batch)
{
if (await sequentialStore.RemoveOffset(DeleteMessageCallback, messageWithOffset.Item1, this.cancellationTokenSource.Token))
{
cleanupCount++;
}
}
offset += CleanupBatchSize;
}while (batch.Any());
}
else
{
while (await sequentialStore.RemoveFirst(DeleteMessageCallback))
{
cleanupCount++;
}
messageCount = tailOffset - headOffset + 1;
}
// update Metrics for message counts
Checkpointer.Metrics.QueueLength.Set(messageCount, new[] { endpointId, priority.ToString(), bool.TrueString });
totalCleanupCount += cleanupCount;
totalCleanupStoreCount += cleanupEntityStoreCount;
Events.CleanupCompleted(messageQueueId, cleanupCount, cleanupEntityStoreCount, totalCleanupCount, totalCleanupStoreCount);
await Task.Delay(MinCleanupSleepTime, this.cancellationTokenSource.Token);
}
catch (Exception ex)
{
Events.ErrorCleaningMessagesForEndpoint(ex, messageQueueId);
}
}
await Task.Delay(this.GetCleanupTaskSleepTime());
}
}
async Task SendMessagesPump()
{
try
{
Events.StartSendMessagesPump(this);
int batchSize = this.options.BatchSize * this.Endpoint.FanOutFactor;
// Keep the stores and prefetchers for each priority loaded
// for the duration of the pump
var messageProviderPairs = new Dictionary <uint, (IMessageIterator, StoreMessagesProvider)>();
// Outer loop to maintain the message pump until the executor shuts down
while (!this.cts.IsCancellationRequested)
{
try
{
await this.hasMessagesInQueue.WaitAsync(this.options.BatchTimeout);
ImmutableDictionary <uint, EndpointExecutorFsm> snapshot = this.prioritiesToFsms;
bool haveMessagesRemaining = false;
uint[] orderedPriorities = snapshot.Keys.OrderBy(k => k).ToArray();
// Iterate through all the message queues in priority order
foreach (uint priority in orderedPriorities)
{
// Also check for cancellation in every inner loop,
// since it could take time to send a batch of messages
if (this.cts.IsCancellationRequested)
{
break;
}
EndpointExecutorFsm fsm = snapshot[priority];
// Update the lastUsedFsm to be the current FSM
this.lastUsedFsm = fsm;
(IMessageIterator, StoreMessagesProvider)pair;
if (!messageProviderPairs.TryGetValue(priority, out pair))
{
// Create and cache a new pair for the message provider
// so we can reuse it every loop
pair.Item1 = this.messageStore.GetMessageIterator(MessageQueueIdHelper.GetMessageQueueId(this.Endpoint.Id, priority), fsm.Checkpointer.Offset + 1);
pair.Item2 = new StoreMessagesProvider(pair.Item1, batchSize);
messageProviderPairs.Add(priority, pair);
}
StoreMessagesProvider storeMessagesProvider = pair.Item2;
IMessage[] messages = await storeMessagesProvider.GetMessages();
if (messages.Length > 0)
{
// Tag the message with the priority that we're currently
// processing, so it can be used by metrics later
foreach (IMessage msg in messages)
{
msg.ProcessedPriority = priority;
}
Events.ProcessingMessages(this, messages, priority);
await this.ProcessMessages(messages, fsm);
Events.SendMessagesSuccess(this, messages, priority);
// Only move on to the next priority if the queue for the current
// priority is empty. If we processed any messages, break out of
// the inner loop to restart at the beginning of the priorities list
// again. This is so we can catch and process any higher priority
// messages that came in while we were sending the current batch
haveMessagesRemaining = true;
break;
}
}
if (!haveMessagesRemaining)
{
// All the message queues have been drained, reset the hasMessagesInQueue flag.
this.hasMessagesInQueue.Reset();
}
}
catch (Exception ex)
{
Events.SendMessagesError(this, ex);
// Swallow exception and keep trying.
}
}
}
catch (Exception ex)
{
Events.SendMessagesPumpFailure(this, ex);
}
}
