async Task ProcessMessages(IMessage[] messages, EndpointExecutorFsm fsm)
{
SendMessage command = Commands.SendMessage(messages);
await fsm.RunAsync(command);
await command.Completion;
}
public async Task UpdatePriorities(IList <uint> priorities, Option <Endpoint> newEndpoint)
{
Preconditions.CheckArgument(priorities.Count > 0);
Events.UpdatePriorities(this, priorities);
if (this.closed)
{
throw new InvalidOperationException($"Endpoint executor for endpoint {this.Endpoint} is closed.");
}
// Update priorities by merging the new ones with the existing.
// We don't ever remove stale priorities, otherwise stored messages
// pending for a removed priority will never get sent.
ImmutableDictionary <uint, EndpointExecutorFsm> snapshot = this.prioritiesToFsms;
Dictionary <uint, EndpointExecutorFsm> updatedSnapshot = new Dictionary <uint, EndpointExecutorFsm>(snapshot);
foreach (uint priority in priorities)
{
if (!updatedSnapshot.ContainsKey(priority))
{
string id = GetMessageQueueId(this.Endpoint.Id, priority);
// Create a message queue in the store for every priority we have
await this.messageStore.AddEndpoint(id);
// Create a checkpointer and a FSM for every message queue
ICheckpointer checkpointer = await this.checkpointerFactory.CreateAsync(id, this.Endpoint.Id, priority);
EndpointExecutorFsm fsm = new EndpointExecutorFsm(this.Endpoint, checkpointer, this.config);
// Add it to our dictionary
updatedSnapshot.Add(priority, fsm);
}
else
{
// Update the existing FSM with the new endpoint
EndpointExecutorFsm fsm = updatedSnapshot[priority];
await newEndpoint.ForEachAsync(e => fsm.RunAsync(Commands.UpdateEndpoint(e)));
}
}
if (this.prioritiesToFsms.CompareAndSet(snapshot, updatedSnapshot.ToImmutableDictionary()))
{
Events.UpdatePrioritiesSuccess(this, updatedSnapshot.Keys.ToList());
// Update the lastUsedFsm to be the initial one, we always
// have at least one priority->FSM pair at this point.
this.lastUsedFsm = updatedSnapshot[priorities[0]];
}
else
{
Events.UpdatePrioritiesFailure(this, updatedSnapshot.Keys.ToList());
}
}
public SyncEndpointExecutor(Endpoint endpoint, ICheckpointer checkpointer, EndpointExecutorConfig config)
{
Preconditions.CheckNotNull(endpoint);
Preconditions.CheckNotNull(config);
this.checkpointer = Preconditions.CheckNotNull(checkpointer);
this.machine = new EndpointExecutorFsm(endpoint, checkpointer, config);
this.cts = new CancellationTokenSource();
this.closed = new AtomicBoolean();
this.sync = new AsyncLock();
}
public StoringAsyncEndpointExecutor(Endpoint endpoint,
ICheckpointer checkpointer,
EndpointExecutorConfig config,
AsyncEndpointExecutorOptions options,
IMessageStore messageStore)
{
Preconditions.CheckNotNull(endpoint);
Preconditions.CheckNotNull(config);
this.checkpointer = Preconditions.CheckNotNull(checkpointer);
this.options = Preconditions.CheckNotNull(options);
this.machine = new EndpointExecutorFsm(endpoint, checkpointer, config);
this.messageStore = messageStore;
this.sendMessageTask = Task.Run(this.SendMessagesPump);
}
public StoringAsyncEndpointExecutor(
Endpoint endpoint,
IList <uint> priorities,
ICheckpointer checkpointer,
EndpointExecutorConfig config,
AsyncEndpointExecutorOptions options,
IMessageStore messageStore)
{
Preconditions.CheckNotNull(endpoint);
Preconditions.CheckNotNull(config);
Preconditions.CheckNotNull(priorities);
Preconditions.CheckArgument(priorities.Count != 0);
this.checkpointer = Preconditions.CheckNotNull(checkpointer);
this.options = Preconditions.CheckNotNull(options);
this.machine = new EndpointExecutorFsm(endpoint, checkpointer, config);
this.messageStore = messageStore;
this.sendMessageTask = Task.Run(this.SendMessagesPump);
this.UpdatePriorities(priorities);
}
public AsyncEndpointExecutor(Endpoint endpoint, ICheckpointer checkpointer, EndpointExecutorConfig config, AsyncEndpointExecutorOptions options)
{
Preconditions.CheckNotNull(endpoint);
Preconditions.CheckNotNull(config);
this.checkpointer = Preconditions.CheckNotNull(checkpointer);
this.cts = new CancellationTokenSource();
this.options = Preconditions.CheckNotNull(options);
this.machine = new EndpointExecutorFsm(endpoint, checkpointer, config);
this.closed = new AtomicBoolean();
// The three size variables below adjust the following parameters:
// 1. MaxMessagesPerTask - the maximum number of messages the batch block will process before yielding
// 2. BoundedCapacity - the size of the batch blocks input buffer
// 3. BatchBlock ctor - the maximum size of each batch emitted by the block (can be smaller because of the timer)
var batchOptions = new GroupingDataflowBlockOptions
{
MaxMessagesPerTask = MaxMessagesPerTask,
BoundedCapacity = options.BatchSize
};
var batchBlock = new BatchBlock <IMessage>(options.BatchSize, batchOptions);
this.batchTimer = new Timer(Trigger, batchBlock, options.BatchTimeout, options.BatchTimeout);
var processOptions = new ExecutionDataflowBlockOptions
{
BoundedCapacity = 1
};
var process = new ActionBlock <IMessage[]>(this.MessagesAction, processOptions);
var linkOptions = new DataflowLinkOptions {
PropagateCompletion = true
};
batchBlock.LinkTo(process, linkOptions);
this.head = batchBlock;
this.tail = process;
}
async Task SendMessagesPump()
{
try
{
Events.StartSendMessagesPump(this);
int batchSize = this.options.BatchSize * this.Endpoint.FanOutFactor;
// Keep the stores and prefetchers for each priority loaded
// for the duration of the pump
var messageProviderPairs = new Dictionary <uint, (IMessageIterator, StoreMessagesProvider)>();
// Outer loop to maintain the message pump until the executor shuts down
while (!this.cts.IsCancellationRequested)
{
try
{
await this.hasMessagesInQueue.WaitAsync(this.options.BatchTimeout);
ImmutableDictionary <uint, EndpointExecutorFsm> snapshot = this.prioritiesToFsms;
bool haveMessagesRemaining = false;
uint[] orderedPriorities = snapshot.Keys.OrderBy(k => k).ToArray();
// Iterate through all the message queues in priority order
foreach (uint priority in orderedPriorities)
{
// Also check for cancellation in every inner loop,
// since it could take time to send a batch of messages
if (this.cts.IsCancellationRequested)
{
break;
}
EndpointExecutorFsm fsm = snapshot[priority];
// Update the lastUsedFsm to be the current FSM
this.lastUsedFsm = fsm;
(IMessageIterator, StoreMessagesProvider)pair;
if (!messageProviderPairs.TryGetValue(priority, out pair))
{
// Create and cache a new pair for the message provider
// so we can reuse it every loop
pair.Item1 = this.messageStore.GetMessageIterator(GetMessageQueueId(this.Endpoint.Id, priority), fsm.Checkpointer.Offset + 1);
pair.Item2 = new StoreMessagesProvider(pair.Item1, batchSize);
messageProviderPairs.Add(priority, pair);
}
StoreMessagesProvider storeMessagesProvider = pair.Item2;
IMessage[] messages = await storeMessagesProvider.GetMessages();
if (messages.Length > 0)
{
// Tag the message with the priority that we're currently
// processing, so it can be used by metrics later
foreach (IMessage msg in messages)
{
msg.ProcessedPriority = priority;
}
Events.ProcessingMessages(this, messages, priority);
await this.ProcessMessages(messages, fsm);
Events.SendMessagesSuccess(this, messages, priority);
MetricsV0.DrainedCountIncrement(this.Endpoint.Id, messages.Length, priority);
// Only move on to the next priority if the queue for the current
// priority is empty. If we processed any messages, break out of
// the inner loop to restart at the beginning of the priorities list
// again. This is so we can catch and process any higher priority
// messages that came in while we were sending the current batch
haveMessagesRemaining = true;
break;
}
}
if (!haveMessagesRemaining)
{
// All the message queues have been drained, reset the hasMessagesInQueue flag.
this.hasMessagesInQueue.Reset();
}
}
catch (Exception ex)
{
Events.SendMessagesError(this, ex);
// Swallow exception and keep trying.
}
}
}
catch (Exception ex)
{
Events.SendMessagesPumpFailure(this, ex);
}
}
