/// <summary>
/// <exception cref="ArgumentException">Thrown if the supplied stream filter function is not suitable.
/// Usually this is because it is not a static method. </exception>
/// </summary>
/// <typeparam name="T">The type of object produced by the observable.</typeparam>
/// <param name="handle">The subscription handle.</param>
/// <param name="onNextAsync">Delegate that is called for IAsyncObserver.OnNextAsync.</param>
/// <param name="token">The stream sequence to be used as an offset to start the subscription from.</param>
/// <returns>A promise for a StreamSubscriptionHandle that represents the subscription.
/// The consumer may unsubscribe by using this handle.
/// The subscription remains active for as long as it is not explicitly unsubscribed.
/// </returns>
public static Task <StreamSubscriptionHandle <T> > ResumeAsync <T>(this StreamSubscriptionHandle <T> handle,
Func <T, StreamSequenceToken, Task> onNextAsync,
StreamSequenceToken token = null)
{
return(handle.ResumeAsync(onNextAsync, DefaultOnError, DefaultOnCompleted, token));
}
/// <summary>
/// Read from queue.
/// Returns true, if data was read, false if it was not
/// </summary>
/// <param name="myQueueId"></param>
/// <param name="rcvr"></param>
/// <param name="maxCacheAddCount"></param>
/// <returns></returns>
private async Task <bool> ReadFromQueue(QueueId myQueueId, IQueueAdapterReceiver rcvr, int maxCacheAddCount)
{
if (rcvr == null)
{
return(false);
}
var now = DateTime.UtcNow;
// Try to cleanup the pubsub cache at the cadence of 10 times in the configurable StreamInactivityPeriod.
if ((now - lastTimeCleanedPubSubCache) >= this.options.StreamInactivityPeriod.Divide(StreamInactivityCheckFrequency))
{
lastTimeCleanedPubSubCache = now;
CleanupPubSubCache(now);
}
if (queueCache != null)
{
IList <IBatchContainer> purgedItems;
if (queueCache.TryPurgeFromCache(out purgedItems))
{
try
{
await rcvr.MessagesDeliveredAsync(purgedItems);
}
catch (Exception exc)
{
logger.Warn(ErrorCode.PersistentStreamPullingAgent_27,
$"Exception calling MessagesDeliveredAsync on queue {myQueueId}. Ignoring.", exc);
}
}
}
if (queueCache != null && queueCache.IsUnderPressure())
{
// Under back pressure. Exit the loop. Will attempt again in the next timer callback.
logger.Info((int)ErrorCode.PersistentStreamPullingAgent_24, "Stream cache is under pressure. Backing off.");
return(false);
}
// Retrieve one multiBatch from the queue. Every multiBatch has an IEnumerable of IBatchContainers, each IBatchContainer may have multiple events.
IList <IBatchContainer> multiBatch = await rcvr.GetQueueMessagesAsync(maxCacheAddCount);
if (multiBatch == null || multiBatch.Count == 0)
{
return(false); // queue is empty. Exit the loop. Will attempt again in the next timer callback.
}
queueCache?.AddToCache(multiBatch);
numMessages += multiBatch.Count;
numReadMessagesCounter.IncrementBy(multiBatch.Count);
if (logger.IsEnabled(LogLevel.Trace))
{
logger.Trace(ErrorCode.PersistentStreamPullingAgent_11, "Got {0} messages from queue {1}. So far {2} msgs from this queue.",
multiBatch.Count, myQueueId.ToStringWithHashCode(), numMessages);
}
foreach (var group in
multiBatch
.Where(m => m != null)
.GroupBy(container => new Tuple <Guid, string>(container.StreamGuid, container.StreamNamespace)))
{
var streamId = StreamId.GetStreamId(group.Key.Item1, queueAdapter.Name, group.Key.Item2);
StreamSequenceToken startToken = group.First().SequenceToken;
StreamConsumerCollection streamData;
if (pubSubCache.TryGetValue(streamId, out streamData))
{
streamData.RefreshActivity(now);
StartInactiveCursors(streamData, startToken); // if this is an existing stream, start any inactive cursors
}
else
{
RegisterStream(streamId, startToken, now).Ignore(); // if this is a new stream register as producer of stream in pub sub system
}
}
return(true);
}
private async Task <bool> ErrorProtocol(StreamConsumerData consumerData, Exception exceptionOccured, bool isDeliveryError, IBatchContainer batch, StreamSequenceToken token)
{
// for loss of client, we just remove the subscription
if (exceptionOccured is ClientNotAvailableException)
{
logger.Warn(ErrorCode.Stream_ConsumerIsDead,
"Consumer {0} on stream {1} is no longer active - permanently removing Consumer.", consumerData.StreamConsumer, consumerData.StreamId);
pubSub.UnregisterConsumer(consumerData.SubscriptionId, consumerData.StreamId, consumerData.StreamId.ProviderName).Ignore();
return(true);
}
// notify consumer about the error or that the data is not available.
await OrleansTaskExtentions.ExecuteAndIgnoreException(
() => DeliverErrorToConsumer(
consumerData, exceptionOccured, batch));
// record that there was a delivery failure
if (isDeliveryError)
{
await OrleansTaskExtentions.ExecuteAndIgnoreException(
() => streamFailureHandler.OnDeliveryFailure(
consumerData.SubscriptionId, streamProviderName, consumerData.StreamId, token));
}
else
{
await OrleansTaskExtentions.ExecuteAndIgnoreException(
() => streamFailureHandler.OnSubscriptionFailure(
consumerData.SubscriptionId, streamProviderName, consumerData.StreamId, token));
}
// if configured to fault on delivery failure and this is not an implicit subscription, fault and remove the subscription
if (streamFailureHandler.ShouldFaultSubsriptionOnError && !SubscriptionMarker.IsImplicitSubscription(consumerData.SubscriptionId.Guid))
{
try
{
// notify consumer of faulted subscription, if we can.
await OrleansTaskExtentions.ExecuteAndIgnoreException(
() => DeliverErrorToConsumer(
consumerData, new FaultedSubscriptionException(consumerData.SubscriptionId, consumerData.StreamId), batch));
// mark subscription as faulted.
await pubSub.FaultSubscription(consumerData.StreamId, consumerData.SubscriptionId);
}
finally
{
// remove subscription
RemoveSubscriber_Impl(consumerData.SubscriptionId, consumerData.StreamId);
}
return(true);
}
return(false);
}
static public bool Older(this StreamSequenceToken me, StreamSequenceToken other)
{
return(me.CompareTo(other) < 0);
}
private async Task <bool> DoHandshakeWithConsumer(
StreamConsumerData consumerData,
StreamSequenceToken cacheToken)
{
StreamHandshakeToken requestedHandshakeToken = null;
// if not cache, then we can't get cursor and there is no reason to ask consumer for token.
if (queueCache != null)
{
Exception exceptionOccured = null;
try
{
requestedHandshakeToken = await AsyncExecutorWithRetries.ExecuteWithRetries(
i => consumerData.StreamConsumer.GetSequenceToken(consumerData.SubscriptionId),
AsyncExecutorWithRetries.INFINITE_RETRIES,
(exception, i) => !(exception is ClientNotAvailableException),
this.options.MaxEventDeliveryTime,
DeliveryBackoffProvider);
if (requestedHandshakeToken != null)
{
consumerData.SafeDisposeCursor(logger);
consumerData.Cursor = queueCache.GetCacheCursor(consumerData.StreamId, requestedHandshakeToken.Token);
}
else
{
if (consumerData.Cursor == null) // if the consumer did not ask for a specific token and we already have a cursor, jsut keep using it.
{
consumerData.Cursor = queueCache.GetCacheCursor(consumerData.StreamId, cacheToken);
}
}
}
catch (Exception exception)
{
exceptionOccured = exception;
}
if (exceptionOccured != null)
{
bool faultedSubscription = await ErrorProtocol(consumerData, exceptionOccured, false, null, requestedHandshakeToken?.Token);
if (faultedSubscription)
{
return(false);
}
}
}
consumerData.LastToken = requestedHandshakeToken; // use what ever the consumer asked for as LastToken for next handshake (even if he asked for null).
// if we don't yet have a cursor (had errors in the handshake or data not available exc), get a cursor at the event that triggered that consumer subscription.
if (consumerData.Cursor == null && queueCache != null)
{
try
{
consumerData.Cursor = queueCache.GetCacheCursor(consumerData.StreamId, cacheToken);
}
catch (Exception)
{
consumerData.Cursor = queueCache.GetCacheCursor(consumerData.StreamId, null); // just in case last GetCacheCursor failed.
}
}
return(true);
}
public async Task <StreamHandshakeToken> DeliverMutable(GuidId subscriptionId, StreamId streamId, object item, StreamSequenceToken currentToken, StreamHandshakeToken handshakeToken)
{
if (logger.IsEnabled(LogLevel.Trace))
{
var itemString = item.ToString();
itemString = (itemString.Length > MAXIMUM_ITEM_STRING_LOG_LENGTH) ? itemString.Substring(0, MAXIMUM_ITEM_STRING_LOG_LENGTH) + "..." : itemString;
logger.Trace("DeliverItem {0} for subscription {1}", itemString, subscriptionId);
}
IStreamSubscriptionHandle observer;
if (allStreamObservers.TryGetValue(subscriptionId, out observer))
{
return(await observer.DeliverItem(item, currentToken, handshakeToken));
}
else if (this.subscriptionChangeHandler != null)
{
await this.subscriptionChangeHandler.HandleNewSubscription(subscriptionId, streamId, item.GetType());
//check if an observer were attached after handling the new subscription, deliver on it if attached
if (allStreamObservers.TryGetValue(subscriptionId, out observer))
{
return(await observer.DeliverItem(item, currentToken, handshakeToken));
}
}
logger.Warn((int)(ErrorCode.StreamProvider_NoStreamForItem), "{0} got an item for subscription {1}, but I don't have any subscriber for that stream. Dropping on the floor.",
providerRuntime.ExecutingEntityIdentity(), subscriptionId);
// We got an item when we don't think we're the subscriber. This is a normal race condition.
// We can drop the item on the floor, or pass it to the rendezvous, or ...
return(default(StreamHandshakeToken));
}
public Task <StreamHandshakeToken> DeliverImmutable(GuidId subscriptionId, StreamId streamId, Immutable <object> item, StreamSequenceToken currentToken, StreamHandshakeToken handshakeToken)
{
return(DeliverMutable(subscriptionId, streamId, item.Value, currentToken, handshakeToken));
}
internal StreamSubscriptionHandleImpl <T> SetObserver <T>(GuidId subscriptionId, StreamImpl <T> stream, IAsyncObserver <T> observer, StreamSequenceToken token, IStreamFilterPredicateWrapper filter)
{
if (null == stream)
{
throw new ArgumentNullException("stream");
}
if (null == observer)
{
throw new ArgumentNullException("observer");
}
try
{
if (logger.IsEnabled(LogLevel.Debug))
{
logger.Debug("{0} AddObserver for stream {1}", providerRuntime.ExecutingEntityIdentity(), stream.StreamId);
}
// Note: The caller [StreamConsumer] already handles locking for Add/Remove operations, so we don't need to repeat here.
var handle = new StreamSubscriptionHandleImpl <T>(subscriptionId, observer, stream, filter, token);
return(allStreamObservers.AddOrUpdate(subscriptionId, handle, (key, old) => handle) as StreamSubscriptionHandleImpl <T>);
}
catch (Exception exc)
{
logger.Error(ErrorCode.StreamProvider_AddObserverException,
$"{providerRuntime.ExecutingEntityIdentity()} StreamConsumerExtension.AddObserver({stream.StreamId}) caugth exception.", exc);
throw;
}
}
private async Task <StreamSubscriptionHandle <T> > SubscribeAsyncImpl(
IAsyncObserver <T> observer,
IAsyncBatchObserver <T> batchObserver,
StreamSequenceToken token,
string filterData = null)
{
if (token != null && !IsRewindable)
{
throw new ArgumentNullException("token", "Passing a non-null token to a non-rewindable IAsyncObservable.");
}
if (observer is GrainReference)
{
throw new ArgumentException("On-behalf subscription via grain references is not supported. Only passing of object references is allowed.", nameof(observer));
}
if (batchObserver is GrainReference)
{
throw new ArgumentException("On-behalf subscription via grain references is not supported. Only passing of object references is allowed.", nameof(batchObserver));
}
_ = RequestContextExtensions.SuppressCurrentCallChainFlow();
if (logger.IsEnabled(LogLevel.Debug))
{
logger.Debug("Subscribe Token={Token}", token);
}
await BindExtensionLazy();
if (logger.IsEnabled(LogLevel.Debug))
{
logger.Debug("Subscribe - Connecting to Rendezvous {0} My GrainRef={1} Token={2}",
pubSub, myGrainReference, token);
}
GuidId subscriptionId = pubSub.CreateSubscriptionId(stream.InternalStreamId, myGrainReference);
// Optimistic Concurrency:
// In general, we should first register the subsription with the pubsub (pubSub.RegisterConsumer)
// and only if it succeeds store it locally (myExtension.SetObserver).
// Basicaly, those 2 operations should be done as one atomic transaction - either both or none and isolated from concurrent reads.
// BUT: there is a distributed race here: the first msg may arrive before the call is awaited
// (since the pubsub notifies the producer that may immideately produce)
// and will thus not find the subriptionHandle in the extension, basically violating "isolation".
// Therefore, we employ Optimistic Concurrency Control here to guarantee isolation:
// we optimisticaly store subscriptionId in the handle first before calling pubSub.RegisterConsumer
// and undo it in the case of failure.
// There is no problem with that we call myExtension.SetObserver too early before the handle is registered in pub sub,
// since this subscriptionId is unique (random Guid) and no one knows it anyway, unless successfully subscribed in the pubsub.
var subriptionHandle = myExtension.SetObserver(subscriptionId, stream, observer, batchObserver, token, filterData);
try
{
await pubSub.RegisterConsumer(subscriptionId, stream.InternalStreamId, myGrainReference, filterData);
return(subriptionHandle);
}
catch (Exception)
{
// Undo the previous call myExtension.SetObserver.
myExtension.RemoveObserver(subscriptionId);
throw;
}
}
public Task <StreamSubscriptionHandle <T> > SubscribeAsync(IAsyncBatchObserver <T> batchObserver, StreamSequenceToken token)
{
return(SubscribeAsyncImpl(null, batchObserver, token));
}