protected override async Task ProcessQueue(CancellationToken cancellationToken, ReferenceWrapper subscriber, string queueName)
{
var queue = await TimeoutRetryHelper.Execute((token, state) => StateManager.GetOrAddAsync <IReliableConcurrentQueue <MessageWrapper> >(queueName), cancellationToken : cancellationToken);
long messageCount = queue.Count;
if (messageCount == 0L)
{
return;
}
messageCount = Math.Min(messageCount, MaxDequeuesInOneIteration);
LogProvider?.LogMessage($"Processing {messageCount} items from queue {queue.Name} for subscriber: {subscriber.Name}");
for (long i = 0; i < messageCount; i++)
{
cancellationToken.ThrowIfCancellationRequested();
await StateManager.ExecuteInTransaction(async (tx, token, state) =>
{
var result = await queue.TryDequeueAsync(tx, cancellationToken);
if (result.HasValue)
{
await subscriber.PublishAsync(result.Value);
}
}, cancellationToken : cancellationToken);
}
}
/// <summary>
/// Registers a Service or Actor <paramref name="reference"/> as subscriber for messages of type <paramref name="messageTypeName"/>
/// </summary>
/// <param name="reference">Reference to the Service or Actor to register.</param>
/// <param name="messageTypeName">Full type name of message object.</param>
/// <returns></returns>
public async Task SubscribeAsync(ReferenceWrapper reference, string messageTypeName)
{
await WaitForInitializeAsync(CancellationToken.None);
var myDictionary = await TimeoutRetryHelper.Execute((token, state) => StateManager.GetOrAddAsync <IReliableDictionary <string, BrokerServiceState> >(messageTypeName));
await TimeoutRetryHelper.ExecuteInTransaction(StateManager, async (tx, token, state) =>
{
var queueName = CreateQueueName(reference, messageTypeName);
Func <string, BrokerServiceState> addValueFactory = key =>
{
var newState = new BrokerServiceState(messageTypeName);
var subscriber = new Reference(reference, queueName);
newState = BrokerServiceState.AddSubscriber(newState, subscriber);
return(newState);
};
Func <string, BrokerServiceState, BrokerServiceState> updateValueFactory = (key, current) =>
{
var subscriber = new Reference(reference, queueName);
var newState = BrokerServiceState.AddSubscriber(current, subscriber);
return(newState);
};
await myDictionary.AddOrUpdateAsync(tx, Subscribers, addValueFactory, updateValueFactory);
await CreateQueueAsync(tx, queueName);
_queues.AddOrUpdate(queueName, reference, (key, old) => reference);
ServiceEventSourceMessage($"Registered subscriber: {reference.Name}");
await BrokerEventsManager.OnSubscribedAsync(queueName, reference, messageTypeName);
}, cancellationToken : CancellationToken.None);
}
/// <summary>
/// Unregisters a Service or Actor <paramref name="reference"/> as subscriber for messages of type <paramref name="messageTypeName"/>
/// </summary>
/// <param name="reference"></param>
/// <param name="messageTypeName"></param>
/// <returns></returns>
public async Task UnsubscribeAsync(ReferenceWrapper reference, string messageTypeName)
{
await WaitForInitializeAsync(CancellationToken.None);
var myDictionary = await TimeoutRetryHelper.Execute((token, state) => StateManager.GetOrAddAsync <IReliableDictionary <string, BrokerServiceState> >(messageTypeName));
var queueName = CreateQueueName(reference, messageTypeName);
await TimeoutRetryHelper.ExecuteInTransaction(StateManager, async (tx, token, state) =>
{
var subscribers = await myDictionary.TryGetValueAsync(tx, Subscribers, LockMode.Update);
if (subscribers.HasValue)
{
var newState = BrokerServiceState.RemoveSubscriber(subscribers.Value, reference);
await myDictionary.SetAsync(tx, Subscribers, newState);
}
await StateManager.RemoveAsync(tx, queueName);
ServiceEventSourceMessage($"Unregistered subscriber: {reference.Name}");
_queues.TryRemove(queueName, out reference);
await BrokerEventsManager.OnUnsubscribedAsync(queueName, reference, messageTypeName);
});
}
/// <summary>
/// Takes a published message and forwards it (indirectly) to all Subscribers.
/// </summary>
/// <param name="message">The message to publish</param>
/// <returns></returns>
public async Task PublishMessageAsync(MessageWrapper message)
{
await WaitForInitializeAsync(CancellationToken.None);
await _brokerEventsManager.OnMessagePublishedAsync(message);
var myDictionary = await TimeoutRetryHelper.Execute((token, state) => StateManager.GetOrAddAsync <IReliableDictionary <string, BrokerServiceState> >(message.MessageType));
var subscriptions = await TimeoutRetryHelper.ExecuteInTransaction(StateManager, async (tx, token, state) =>
{
var result = await myDictionary.TryGetValueAsync(tx, Subscribers);
return(result.HasValue ? result.Value.Subscribers.ToArray() : null);
});
if (subscriptions == null || subscriptions.Length == 0)
{
return;
}
ServiceEventSourceMessage($"Publishing message '{message.MessageType}' to {subscriptions.Length} subscribers.");
await TimeoutRetryHelper.ExecuteInTransaction(StateManager, async (tx, token, state) =>
{
foreach (var subscriptionDetails in subscriptions)
{
await _subscriptions[subscriptionDetails.QueueName].EnqueueMessageAsync(tx, message);
await _brokerEventsManager.OnMessageQueuedToSubscriberAsync(subscriptionDetails.QueueName, subscriptionDetails.ServiceOrActorReference, message);
}
ServiceEventSourceMessage($"Published message '{message.MessageType}' to {subscriptions.Length} subscribers.");
});
}
/// <summary>
/// Loads all registered message queues from state and keeps them in memory. Avoids some locks in the statemanager.
/// </summary>
/// <param name="cancellationToken"></param>
/// <returns></returns>
private async Task InitializeAsync(CancellationToken cancellationToken)
{
if (_initializer.IsSet)
{
return;
}
try
{
SetupEvents(_brokerEventsManager);
_semaphore.Wait(cancellationToken);
if (_initializer.IsSet)
{
return;
}
await TimeoutRetryHelper.ExecuteInTransaction(StateManager, async (tx, token, state) =>
{
_subscriptions.Clear();
var enumerator = StateManager.GetAsyncEnumerator();
while (await enumerator.MoveNextAsync(cancellationToken))
{
var current = enumerator.Current as IReliableDictionary <string, BrokerServiceState>;
if (current == null)
{
continue;
}
var result = await current.TryGetValueAsync(tx, Subscribers);
if (!result.HasValue)
{
continue;
}
var subscriptions = result.Value.Subscribers.ToList();
foreach (var subscriptionDetails in subscriptions)
{
var subscription = await _subscriptionFactory.CreateAsync(tx, subscriptionDetails);
_subscriptions.TryAdd(subscriptionDetails.QueueName, subscription);
}
}
}, cancellationToken : cancellationToken);
_initializer.Set();
}
finally
{
_semaphore.Release();
}
}
/// <inheritdoc />
public async Task ReceiveMessageAsync(MessageWrapper message)
{
//assume that message contains 'TaskDescription'
var description = message.CreateMessage <TaskDescription>();
if (description == null)
{
return; //wrong message
}
var queue = await TimeoutRetryHelper.Execute((token, state) => StateManager.GetOrAddAsync <IReliableQueue <MessageWrapper> >(_queueName));
await TimeoutRetryHelper
.ExecuteInTransaction(StateManager, (tran, token, state) => queue.EnqueueAsync(tran, message))
.ConfigureAwait(false);
}
/// <summary>
/// Registers a Service or Actor <paramref name="reference"/> as subscriber for messages of type <paramref name="messageTypeName"/>
/// </summary>
/// <param name="reference">Reference to the Service or Actor to register.</param>
/// <param name="messageTypeName">Full type name of message object.</param>
/// <param name="isOrdered"></param>
/// <returns></returns>
public async Task SubscribeAsync(ReferenceWrapper reference, string messageTypeName, bool isOrdered = true)
{
await WaitForInitializeAsync(CancellationToken.None);
var brokerState = await TimeoutRetryHelper.Execute((token, state) => StateManager.GetOrAddAsync <IReliableDictionary <string, BrokerServiceState> >(messageTypeName));
await TimeoutRetryHelper.ExecuteInTransaction(StateManager, async (tx, token, state) =>
{
var subscriptionDetails = new SubscriptionDetails(reference, messageTypeName, isOrdered);
var subscription = await _subscriptionFactory.CreateAsync(tx, subscriptionDetails);
await brokerState.AddOrUpdateSubscription(tx, Subscribers, subscriptionDetails);
_subscriptions.AddOrUpdate(subscriptionDetails.QueueName, subscription, (key, old) => subscription);
ServiceEventSourceMessage($"Registered subscriber: {reference.Name}");
await _brokerEventsManager.OnSubscribedAsync(subscriptionDetails.QueueName, reference, messageTypeName);
}, cancellationToken : CancellationToken.None);
}
/// <summary>
/// Unregisters a Service or Actor <paramref name="reference"/> as subscriber for messages of type <paramref name="messageTypeName"/>
/// </summary>
/// <param name="reference"></param>
/// <param name="messageTypeName"></param>
/// <returns></returns>
public async Task UnsubscribeAsync(ReferenceWrapper reference, string messageTypeName)
{
await WaitForInitializeAsync(CancellationToken.None);
var brokerState = await TimeoutRetryHelper.Execute((token, state) => StateManager.GetOrAddAsync <IReliableDictionary <string, BrokerServiceState> >(messageTypeName));
await TimeoutRetryHelper.ExecuteInTransaction(StateManager, async (tx, token, state) =>
{
var queueName = SubscriptionDetails.CreateQueueName(reference, messageTypeName);
if (_subscriptions.TryGetValue(queueName, out var subscription))
{
await brokerState.RemoveSubscription(tx, Subscribers, subscription.SubscriptionDetails);
}
await StateManager.RemoveAsync(tx, queueName);
ServiceEventSourceMessage($"Unregistered subscriber: {reference.Name}");
_subscriptions.TryRemove(queueName, out _);
await _brokerEventsManager.OnUnsubscribedAsync(queueName, reference, messageTypeName);
});
}
/// <summary>
/// Sends out queued messages for the provided queue.
/// </summary>
/// <param name="cancellationToken"></param>
/// <param name="subscriber"></param>
/// <param name="queueName"></param>
/// <returns></returns>
protected sealed override async Task ProcessQueues(CancellationToken cancellationToken, ReferenceWrapper subscriber, string queueName)
{
var queue = await TimeoutRetryHelper.Execute((token, state) => StateManager.GetOrAddAsync <IReliableConcurrentQueue <MessageWrapper> >(queueName), cancellationToken : cancellationToken);
long messageCount = queue.Count;
if (messageCount == 0L)
{
return;
}
messageCount = Math.Min(messageCount, MaxDequeuesInOneIteration);
ServiceEventSourceMessage($"Processing {messageCount} items from queue {queue.Name} for subscriber: {subscriber.Name}");
for (long i = 0; i < messageCount; i++)
{
cancellationToken.ThrowIfCancellationRequested();
await TimeoutRetryHelper.ExecuteInTransaction(StateManager, async (tx, token, state) =>
{
var result = await queue.TryDequeueAsync(tx, cancellationToken);
if (result.HasValue)
{
try
{
await subscriber.PublishAsync(result.Value);
await BrokerEventsManager.OnMessageDeliveredAsync(queueName, subscriber, result.Value);
}
catch (Exception ex)
{
await BrokerEventsManager.OnMessageDeliveryFailedAsync(queueName, subscriber, result.Value, ex);
throw;
}
}
}, cancellationToken : cancellationToken);
}
}
/// <summary>
/// Sends out queued messages for the provided queue.
/// </summary>
/// <param name="cancellationToken"></param>
/// <param name="subscription"></param>
/// <returns></returns>
private async Task ProcessQueues(CancellationToken cancellationToken, ISubscription subscription)
{
var details = subscription.SubscriptionDetails;
var messageCount = await subscription.GetQueueCount(cancellationToken);
if (messageCount == 0L)
{
return;
}
messageCount = Math.Min(messageCount, MaxDequeuesInOneIteration);
ServiceEventSourceMessage($"Processing {messageCount} items from queue {details.QueueName} for subscriber: {details.ServiceOrActorReference.Name}");
for (long i = 0; i < messageCount; i++)
{
cancellationToken.ThrowIfCancellationRequested();
await TimeoutRetryHelper.ExecuteInTransaction(StateManager, async (tx, token, state) =>
{
var message = await subscription.DequeueMessageAsync(tx, cancellationToken);
if (message.HasValue)
{
try
{
await subscription.DeliverMessageAsync(message.Value, _proxyFactories);
await _brokerEventsManager.OnMessageDeliveredAsync(details.QueueName, details.ServiceOrActorReference, message.Value);
}
catch (Exception ex)
{
await _brokerEventsManager.OnMessageDeliveryFailedAsync(details.QueueName, details.ServiceOrActorReference, message.Value, ex, ThrottleFactor);
throw;
}
}
}, cancellationToken : cancellationToken);
}
}
protected Task <IReliableDictionary <IndexableActorReference, long> > GetReferenceMapAsync()
{
return(TimeoutRetryHelper.Execute(token => StateAccessor.Get <IReliableDictionary <IndexableActorReference, long> >("referenceMap")));
}
public async Task <long> GetQueueCount(CancellationToken cancellationToken)
{
return(await TimeoutRetryHelper.ExecuteInTransaction(_stateManager, (tx, token, state) => Queue.GetCountAsync(tx), cancellationToken : cancellationToken));
}
Task <IReliableDictionary <string, BrokerServiceState> > GetStateFor(string messageTypeName)
{
return(TimeoutRetryHelper.Execute((token, state) => StateManager.GetOrAddAsync <IReliableDictionary <string, BrokerServiceState> >(messageTypeName)));
}
protected Task <IReliableDictionary <Identity, long> > GetIdentityMapAsync()
{
return(TimeoutRetryHelper.Execute(token => StateAccessor.Get <IReliableDictionary <Identity, long> >("identityMap")));
}
protected Task <IReliableDictionary <long, IndexState> > GetIndexMapAsync()
{
return(TimeoutRetryHelper.Execute(token => StateAccessor.Get <IReliableDictionary <long, IndexState> >("indexMap")));
}
/// <inheritdoc />
protected override async Task RunAsync(CancellationToken cancellationToken)
{
var queue = await TimeoutRetryHelper.Execute(
(token, state) => StateManager.GetOrAddAsync <IReliableQueue <MessageWrapper> >(_queueName),
cancellationToken : cancellationToken)
.ConfigureAwait(false);
while (true)
{
try
{
cancellationToken.ThrowIfCancellationRequested();
//peek task description
var taskDescriptionMessage = await TimeoutRetryHelper
.ExecuteInTransaction(StateManager,
(tran, token, state) => queue.TryPeekAsync(tran, TimeSpan.FromSeconds(4), token), cancellationToken : cancellationToken)
.ConfigureAwait(false);
if (!taskDescriptionMessage.HasValue)
{
continue;
}
//deserialize task description, create task implementation
var description = taskDescriptionMessage.Value.CreateMessage <TaskDescription>();
var implementation = TaskDescription.ToTaskImplementation(_typeLocator, description);
if (implementation == null)
{
ServiceEventSourceMessageCallback?
.Invoke($"Received TaskDescription message with type {description.TaskType} and id {description.TaskId} that could not be transformed into a TaskImplementation type.");
continue;
}
//run task
ServiceEventSourceMessageCallback?
.Invoke($"Processing TaskDescription message with type {description.TaskType} and id {description.TaskId}, using implementation {implementation.GetType().Name}.");
await implementation.ExecuteAsync().ConfigureAwait(false);
var taskDescriptionMessageDequeued = await TimeoutRetryHelper
.ExecuteInTransaction(StateManager,
(tran, token, state) => queue.TryDequeueAsync(tran, TimeSpan.FromSeconds(4), token), cancellationToken : cancellationToken)
.ConfigureAwait(false);
//dequeue task, compare it with the peeked task
if (taskDescriptionMessageDequeued.HasValue &&
taskDescriptionMessageDequeued.Value.Payload.GetHashCode()
== taskDescriptionMessage.Value.Payload.GetHashCode())
{
ServiceEventSourceMessageCallback?
.Invoke($"Processed TaskDescription message with type {description.TaskType} and id {description.TaskId}, using implementation {implementation.GetType().Name}.");
}
else
{
ServiceEventSourceMessageCallback?
.Invoke($"Unexpected error. The peeked message is not the dequeued message.");
}
}
catch (TaskCanceledException)
{//swallow and move on..
}
catch (OperationCanceledException)
{//swallow and move on..
}
catch (ObjectDisposedException)
{//swallow and move on..
}
catch (Exception ex)
{
ServiceEventSourceMessageCallback?.Invoke($"Exception caught while processing messages:'{ex.Message}'");
//swallow and move on..
}
finally
{
await Task.Delay(Period, cancellationToken).ConfigureAwait(false);
}
}
}
protected Task <IReliableDictionary <string, object> > GetPropertiesMapAsync()
{
return(TimeoutRetryHelper.Execute(token => StateAccessor.Get <IReliableDictionary <string, object> >("propertiesMap")));
}
