public async Task <StreamHandshakeToken> DeliverBatch(IBatchContainer batch, StreamHandshakeToken handshakeToken)
{
// we validate expectedToken only for ordered (rewindable) streams
if (expectedToken != null)
{
if (!expectedToken.Equals(handshakeToken))
{
return(expectedToken);
}
}
if (batch is IBatchContainerBatch)
{
var batchContainerBatch = batch as IBatchContainerBatch;
await NextBatch(batchContainerBatch);
}
else
{
foreach (var itemTuple in batch.GetEvents <T>())
{
await NextItem(itemTuple.Item1, itemTuple.Item2);
}
}
if (IsRewindable)
{
expectedToken = StreamHandshakeToken.CreateDeliveyToken(batch.SequenceToken);
}
return(null);
}
public async Task <StreamHandshakeToken> DeliverItem(object item, StreamSequenceToken currentToken, StreamHandshakeToken handshakeToken)
{
if (expectedToken != null)
{
if (!expectedToken.Equals(handshakeToken))
{
return(expectedToken);
}
}
await NextItem(item, currentToken);
// check again, in case the expectedToken was changed indiretly via ResumeAsync()
if (expectedToken != null)
{
if (!expectedToken.Equals(handshakeToken))
{
return(expectedToken);
}
}
expectedToken = StreamHandshakeToken.CreateDeliveyToken(currentToken);
return(null);
}
private async Task DeliverBatchToConsumer(StreamConsumerData consumerData, IBatchContainer batch)
{
StreamHandshakeToken prevToken = consumerData.LastToken;
Task<StreamHandshakeToken> batchDeliveryTask;
bool isRequestContextSet = batch.ImportRequestContext();
try
{
batchDeliveryTask = consumerData.StreamConsumer.DeliverBatch(consumerData.SubscriptionId, batch.AsImmutable(), prevToken);
}
finally
{
if (isRequestContextSet)
{
// clear RequestContext before await!
RequestContext.Clear();
}
}
StreamHandshakeToken newToken = await batchDeliveryTask;
if (newToken != null)
{
consumerData.LastToken = newToken;
consumerData.Cursor = queueCache.GetCacheCursor(consumerData.StreamId.Guid,
consumerData.StreamId.Namespace, newToken.Token);
}
else
{
consumerData.LastToken = StreamHandshakeToken.CreateDeliveyToken(batch.SequenceToken); // this is the currently delivered token
}
}
private async Task<bool> DoHandshakeWithConsumer(
StreamConsumerData consumerData,
StreamSequenceToken cacheToken)
{
StreamHandshakeToken requestedHandshakeToken = null;
// if not cache, then we can't get cursor and there is no reason to ask consumer for token.
if (queueCache != null)
{
Exception exceptionOccured = null;
try
{
requestedHandshakeToken = await AsyncExecutorWithRetries.ExecuteWithRetries(
i => consumerData.StreamConsumer.GetSequenceToken(consumerData.SubscriptionId),
AsyncExecutorWithRetries.INFINITE_RETRIES,
(exception, i) => true,
config.MaxEventDeliveryTime,
DefaultBackoffProvider);
if (requestedHandshakeToken != null)
{
consumerData.SafeDisposeCursor(logger);
consumerData.Cursor = queueCache.GetCacheCursor(consumerData.StreamId, requestedHandshakeToken.Token);
}
else
{
if (consumerData.Cursor == null) // if the consumer did not ask for a specific token and we already have a cursor, jsut keep using it.
consumerData.Cursor = queueCache.GetCacheCursor(consumerData.StreamId, cacheToken);
}
}
catch (Exception exception)
{
exceptionOccured = exception;
}
if (exceptionOccured != null)
{
bool faultedSubscription = await ErrorProtocol(consumerData, exceptionOccured, false, null, requestedHandshakeToken != null ? requestedHandshakeToken.Token : null);
if (faultedSubscription) return false;
}
}
consumerData.LastToken = requestedHandshakeToken; // use what ever the consumer asked for as LastToken for next handshake (even if he asked for null).
// if we don't yet have a cursor (had errors in the handshake or data not available exc), get a cursor at the event that triggered that consumer subscription.
if (consumerData.Cursor == null && queueCache != null)
{
try
{
consumerData.Cursor = queueCache.GetCacheCursor(consumerData.StreamId, cacheToken);
}
catch (Exception)
{
consumerData.Cursor = queueCache.GetCacheCursor(consumerData.StreamId, null); // just in case last GetCacheCursor failed.
}
}
return true;
}
public StreamSubscriptionHandleImpl(GuidId subscriptionId, IAsyncObserver <T> observer, IAsyncBatchObserver <T> batchObserver, StreamImpl <T> streamImpl, StreamSequenceToken token)
{
this.subscriptionId = subscriptionId ?? throw new ArgumentNullException("subscriptionId");
this.observer = observer;
this.batchObserver = batchObserver;
this.streamImpl = streamImpl ?? throw new ArgumentNullException("streamImpl");
this.isRewindable = streamImpl.IsRewindable;
if (IsRewindable)
{
expectedToken = StreamHandshakeToken.CreateStartToken(token);
}
}
private async Task <StreamHandshakeToken> DeliverBatchToConsumer(StreamConsumerData consumerData, IBatchContainer batch)
{
try
{
StreamHandshakeToken newToken = await ContextualizedDeliverBatchToConsumer(consumerData, batch);
consumerData.LastToken = StreamHandshakeToken.CreateDeliveyToken(batch.SequenceToken); // this is the currently delivered token
return(newToken);
}
catch (Exception ex)
{
this.logger.LogWarning(ex, "Failed to deliver message to consumer on {SubscriptionId} for stream {StreamId}, may retry.", consumerData.SubscriptionId, consumerData.StreamId);
throw;
}
}
public StreamSubscriptionHandleImpl(GuidId subscriptionId, IAsyncObserver <T> observer, StreamImpl <T> streamImpl, IStreamFilterPredicateWrapper filterWrapper, StreamSequenceToken token)
{
if (subscriptionId == null)
{
throw new ArgumentNullException("subscriptionId");
}
if (streamImpl == null)
{
throw new ArgumentNullException("streamImpl");
}
this.subscriptionId = subscriptionId;
this.observer = observer;
this.streamImpl = streamImpl;
this.filterWrapper = filterWrapper;
expectedToken = StreamHandshakeToken.CreateStartToken(token);
}
public async Task <StreamHandshakeToken> DeliverBatch(IBatchContainer batch, StreamHandshakeToken handshakeToken)
{
if (expectedToken != null)
{
if (!expectedToken.Equals(handshakeToken))
{
return(expectedToken);
}
}
foreach (var itemTuple in batch.GetEvents <T>())
{
await NextItem(itemTuple.Item1, itemTuple.Item2);
}
expectedToken = StreamHandshakeToken.CreateDeliveyToken(batch.SequenceToken);
return(null);
}
public async Task <StreamHandshakeToken> DeliverItem(object item, StreamSequenceToken currentToken, StreamHandshakeToken handshakeToken)
{
if (this.expectedToken != null)
{
if (!this.expectedToken.Equals(handshakeToken))
{
return(this.expectedToken);
}
}
T typedItem;
try
{
typedItem = (T)item;
}
catch (InvalidCastException)
{
// We got an illegal item on the stream -- close it with a Cast exception
throw new InvalidCastException("Received an item of type " + item.GetType().Name + ", expected " + typeof(T).FullName);
}
await((this.observer != null)
? NextItem(typedItem, currentToken)
: NextItems(new[] { Tuple.Create(typedItem, currentToken) }));
// check again, in case the expectedToken was changed indiretly via ResumeAsync()
if (this.expectedToken != null)
{
if (!this.expectedToken.Equals(handshakeToken))
{
return(this.expectedToken);
}
}
if (IsRewindable)
{
this.expectedToken = StreamHandshakeToken.CreateDeliveyToken(currentToken);
}
return(null);
}
private async Task<StreamHandshakeToken> DeliverBatchToConsumer(StreamConsumerData consumerData, IBatchContainer batch)
{
StreamHandshakeToken prevToken = consumerData.LastToken;
Task<StreamHandshakeToken> batchDeliveryTask;
bool isRequestContextSet = batch.ImportRequestContext();
try
{
batchDeliveryTask = consumerData.StreamConsumer.DeliverBatch(consumerData.SubscriptionId, batch.AsImmutable(), prevToken);
}
finally
{
if (isRequestContextSet)
{
// clear RequestContext before await!
RequestContext.Clear();
}
}
StreamHandshakeToken newToken = await batchDeliveryTask;
consumerData.LastToken = StreamHandshakeToken.CreateDeliveyToken(batch.SequenceToken); // this is the currently delivered token
return newToken;
}
private async Task RunConsumerCursor(StreamConsumerData consumerData)
{
try
{
// double check in case of interleaving
if (consumerData.State == StreamConsumerDataState.Active ||
consumerData.Cursor == null)
{
return;
}
consumerData.State = StreamConsumerDataState.Active;
while (consumerData.Cursor != null)
{
IBatchContainer batch = null;
Exception exceptionOccured = null;
try
{
batch = GetBatchForConsumer(consumerData.Cursor, consumerData.StreamId, consumerData.FilterData);
if (batch == null)
{
break;
}
}
catch (Exception exc)
{
exceptionOccured = exc;
consumerData.SafeDisposeCursor(logger);
consumerData.Cursor = queueCache.GetCacheCursor(consumerData.StreamId, null);
}
if (batch != null)
{
if (!ShouldDeliverBatch(consumerData.StreamId, batch, consumerData.FilterData))
{
continue;
}
}
try
{
numSentMessagesCounter.Increment();
if (batch != null)
{
StreamHandshakeToken newToken = await AsyncExecutorWithRetries.ExecuteWithRetries(
i => DeliverBatchToConsumer(consumerData, batch),
AsyncExecutorWithRetries.INFINITE_RETRIES,
(exception, i) => !(exception is ClientNotAvailableException) || IsShutdown,
this.options.MaxEventDeliveryTime,
DeliveryBackoffProvider);
if (newToken != null)
{
consumerData.LastToken = newToken;
IQueueCacheCursor newCursor = queueCache.GetCacheCursor(consumerData.StreamId, newToken.Token);
consumerData.SafeDisposeCursor(logger);
consumerData.Cursor = newCursor;
}
}
}
catch (Exception exc)
{
consumerData.Cursor?.RecordDeliveryFailure();
var message =
$"Exception while trying to deliver msgs to stream {consumerData.StreamId} in PersistentStreamPullingAgentGrain.RunConsumerCursor";
logger.Error(ErrorCode.PersistentStreamPullingAgent_14, message, exc);
exceptionOccured = exc is ClientNotAvailableException
? exc
: new StreamEventDeliveryFailureException(consumerData.StreamId);
}
// if we failed to deliver a batch
if (exceptionOccured != null)
{
bool faultedSubscription = await ErrorProtocol(consumerData, exceptionOccured, true, batch, batch?.SequenceToken);
if (faultedSubscription)
{
return;
}
}
}
consumerData.State = StreamConsumerDataState.Inactive;
}
catch (Exception exc)
{
// RunConsumerCursor is fired with .Ignore so we should log if anything goes wrong, because there is no one to catch the exception
logger.Error(ErrorCode.PersistentStreamPullingAgent_15, "Ignored RunConsumerCursor Error", exc);
consumerData.State = StreamConsumerDataState.Inactive;
throw;
}
}
public async Task <StreamHandshakeToken> DeliverMutable(GuidId subscriptionId, StreamId streamId, object item, StreamSequenceToken currentToken, StreamHandshakeToken handshakeToken)
{
if (logger.IsEnabled(LogLevel.Trace))
{
var itemString = item.ToString();
itemString = (itemString.Length > MAXIMUM_ITEM_STRING_LOG_LENGTH) ? itemString.Substring(0, MAXIMUM_ITEM_STRING_LOG_LENGTH) + "..." : itemString;
logger.Trace("DeliverItem {0} for subscription {1}", itemString, subscriptionId);
}
IStreamSubscriptionHandle observer;
if (allStreamObservers.TryGetValue(subscriptionId, out observer))
{
return(await observer.DeliverItem(item, currentToken, handshakeToken));
}
else if (this.streamSubscriptionObserver != null)
{
var streamProvider = this.providerRuntime.ServiceProvider.GetServiceByName <IStreamProvider>(streamId.ProviderName);
if (streamProvider != null)
{
var subscriptionHandlerFactory = new StreamSubscriptionHandlerFactory(streamProvider, streamId, streamId.ProviderName, subscriptionId);
await this.streamSubscriptionObserver.OnSubscribed(subscriptionHandlerFactory);
//check if an observer were attached after handling the new subscription, deliver on it if attached
if (allStreamObservers.TryGetValue(subscriptionId, out observer))
{
return(await observer.DeliverItem(item, currentToken, handshakeToken));
}
}
}
logger.Warn((int)(ErrorCode.StreamProvider_NoStreamForItem), "{0} got an item for subscription {1}, but I don't have any subscriber for that stream. Dropping on the floor.",
providerRuntime.ExecutingEntityIdentity(), subscriptionId);
// We got an item when we don't think we're the subscriber. This is a normal race condition.
// We can drop the item on the floor, or pass it to the rendezvous, or ...
return(default(StreamHandshakeToken));
}
public Task <StreamHandshakeToken> DeliverImmutable(GuidId subscriptionId, StreamId streamId, Immutable <object> item, StreamSequenceToken currentToken, StreamHandshakeToken handshakeToken)
{
return(DeliverMutable(subscriptionId, streamId, item.Value, currentToken, handshakeToken));
}
public async Task <StreamHandshakeToken> DeliverBatch(GuidId subscriptionId, StreamId streamId, Immutable <IBatchContainer> batch, StreamHandshakeToken handshakeToken)
{
if (logger.IsEnabled(LogLevel.Trace))
{
logger.Trace("DeliverBatch {0} for subscription {1}", batch.Value, subscriptionId);
}
IStreamSubscriptionHandle observer;
if (allStreamObservers.TryGetValue(subscriptionId, out observer))
{
return(await observer.DeliverBatch(batch.Value, handshakeToken));
}
else if (this.streamSubscriptionObserver != null)
{
var streamProvider = this.providerRuntime.ServiceProvider.GetServiceByName <IStreamProvider>(streamId.ProviderName);
if (streamProvider != null)
{
var subscriptionHandlerFactory = new StreamSubscriptionHandlerFactory(streamProvider, streamId, streamId.ProviderName, subscriptionId);
await this.streamSubscriptionObserver.OnSubscribed(subscriptionHandlerFactory);
// check if an observer were attached after handling the new subscription, deliver on it if attached
if (allStreamObservers.TryGetValue(subscriptionId, out observer))
{
return(await observer.DeliverBatch(batch.Value, handshakeToken));
}
}
}
logger.Warn((int)(ErrorCode.StreamProvider_NoStreamForBatch), "{0} got an item for subscription {1}, but I don't have any subscriber for that stream. Dropping on the floor.",
providerRuntime.ExecutingEntityIdentity(), subscriptionId);
// We got an item when we don't think we're the subscriber. This is a normal race condition.
// We can drop the item on the floor, or pass it to the rendezvous, or ...
return(default(StreamHandshakeToken));
}
public async Task <StreamHandshakeToken> DeliverBatch(GuidId subscriptionId, StreamId streamId, Immutable <IBatchContainer> batch, StreamHandshakeToken handshakeToken)
{
if (logger.IsVerbose3)
{
logger.Verbose3("DeliverBatch {0} for subscription {1}", batch.Value, subscriptionId);
}
IStreamSubscriptionHandle observer;
if (allStreamObservers.TryGetValue(subscriptionId, out observer))
{
return(await observer.DeliverBatch(batch.Value, handshakeToken));
}
else
{
// if no observer attached to the subscription, check if there's onSubscriptinChange actions defined
ISubscriptionChangeHandler handler;
if (this.onSubscriptionChangeActionMap.TryGetValue(batch.Value.GetType(), out handler))
{
//if the onAddAction attached an observer to the subscription
var streamProvider = this.providerRuntime.ServiceProvider
.GetService <IStreamProviderManager>()
.GetStreamProvider(streamId.ProviderName);
await handler.InvokeOnAdd(streamId, subscriptionId, isRewindable, streamProvider);
//if the onAddAction attached an observer to the subscription
if (allStreamObservers.TryGetValue(subscriptionId, out observer))
{
return(await observer.DeliverBatch(batch.Value, handshakeToken));
}
}
}
logger.Warn((int)(ErrorCode.StreamProvider_NoStreamForBatch), "{0} got an item for subscription {1}, but I don't have any subscriber for that stream. Dropping on the floor.",
providerRuntime.ExecutingEntityIdentity(), subscriptionId);
// We got an item when we don't think we're the subscriber. This is a normal race condition.
// We can drop the item on the floor, or pass it to the rendezvous, or ...
return(default(StreamHandshakeToken));
}
public Task <StreamHandshakeToken> DeliverItem(GuidId subscriptionId, Immutable <object> item, StreamSequenceToken currentToken, StreamHandshakeToken handshakeToken)
{
if (logger.IsVerbose3)
{
logger.Verbose3("DeliverItem {0} for subscription {1}", item.Value, subscriptionId);
}
IStreamSubscriptionHandle observer;
if (allStreamObservers.TryGetValue(subscriptionId, out observer))
{
return(observer.DeliverItem(item.Value, currentToken, handshakeToken));
}
logger.Warn((int)(ErrorCode.StreamProvider_NoStreamForItem), "{0} got an item for subscription {1}, but I don't have any subscriber for that stream. Dropping on the floor.",
providerRuntime.ExecutingEntityIdentity(), subscriptionId);
// We got an item when we don't think we're the subscriber. This is a normal race condition.
// We can drop the item on the floor, or pass it to the rendezvous, or ...
return(Task.FromResult(default(StreamHandshakeToken)));
}
private async Task RunConsumerCursor(StreamConsumerData consumerData, IStreamFilterPredicateWrapper filterWrapper)
{
try
{
// double check in case of interleaving
if (consumerData.State == StreamConsumerDataState.Active ||
consumerData.Cursor == null)
{
return;
}
consumerData.State = StreamConsumerDataState.Active;
while (consumerData.Cursor != null)
{
IBatchContainer batch = null;
Exception exceptionOccured = null;
try
{
Exception ignore;
if (!consumerData.Cursor.MoveNext())
{
break;
}
batch = consumerData.Cursor.GetCurrent(out ignore);
}
catch (Exception exc)
{
exceptionOccured = exc;
consumerData.SafeDisposeCursor(logger);
consumerData.Cursor = queueCache.GetCacheCursor(consumerData.StreamId, null);
}
// Apply filtering to this batch, if applicable
if (filterWrapper != null && batch != null)
{
try
{
// Apply batch filter to this input batch, to see whether we should deliver it to this consumer.
if (!batch.ShouldDeliver(
consumerData.StreamId,
filterWrapper.FilterData,
filterWrapper.ShouldReceive))
{
continue; // Skip this batch -- nothing to do
}
}
catch (Exception exc)
{
var message =
$"Ignoring exception while trying to evaluate subscription filter function {filterWrapper} on stream {consumerData.StreamId} in PersistentStreamPullingAgentGrain.RunConsumerCursor";
logger.Warn((int)ErrorCode.PersistentStreamPullingAgent_13, message, exc);
}
}
try
{
numSentMessagesCounter.Increment();
if (batch != null)
{
StreamHandshakeToken newToken = await AsyncExecutorWithRetries.ExecuteWithRetries(
i => DeliverBatchToConsumer(consumerData, batch),
AsyncExecutorWithRetries.INFINITE_RETRIES,
(exception, i) => !(exception is ClientNotAvailableException),
config.MaxEventDeliveryTime,
DeliveryBackoffProvider);
if (newToken != null)
{
consumerData.LastToken = newToken;
IQueueCacheCursor newCursor = queueCache.GetCacheCursor(consumerData.StreamId, newToken.Token);
consumerData.SafeDisposeCursor(logger);
consumerData.Cursor = newCursor;
}
}
}
catch (Exception exc)
{
consumerData.Cursor?.RecordDeliveryFailure();
var message =
$"Exception while trying to deliver msgs to stream {consumerData.StreamId} in PersistentStreamPullingAgentGrain.RunConsumerCursor";
logger.Error(ErrorCode.PersistentStreamPullingAgent_14, message, exc);
exceptionOccured = exc is ClientNotAvailableException
? exc
: new StreamEventDeliveryFailureException(consumerData.StreamId);
}
// if we failed to deliver a batch
if (exceptionOccured != null)
{
bool faultedSubscription = await ErrorProtocol(consumerData, exceptionOccured, true, batch, batch?.SequenceToken);
if (faultedSubscription)
{
return;
}
}
}
consumerData.State = StreamConsumerDataState.Inactive;
}
catch (Exception exc)
{
// RunConsumerCursor is fired with .Ignore so we should log if anything goes wrong, because there is no one to catch the exception
logger.Error(ErrorCode.PersistentStreamPullingAgent_15, "Ignored RunConsumerCursor Error", exc);
consumerData.State = StreamConsumerDataState.Inactive;
throw;
}
}
public async Task <StreamHandshakeToken> DeliverMutable(GuidId subscriptionId, StreamId streamId, object item, StreamSequenceToken currentToken, StreamHandshakeToken handshakeToken)
{
if (logger.IsVerbose3)
{
var itemString = item.ToString();
itemString = (itemString.Length > MAXIMUM_ITEM_STRING_LOG_LENGTH) ? itemString.Substring(0, MAXIMUM_ITEM_STRING_LOG_LENGTH) + "..." : itemString;
logger.Verbose3("DeliverItem {0} for subscription {1}", itemString, subscriptionId);
}
IStreamSubscriptionHandle observer;
if (allStreamObservers.TryGetValue(subscriptionId, out observer))
{
return(await observer.DeliverItem(item, currentToken, handshakeToken));
}
else
{
// if no observer attached to the subscription, check if there's onSubscriptinChange actions defined
ISubscriptionChangeHandler handler;
if (this.onSubscriptionChangeActionMap.TryGetValue(item.GetType(), out handler))
{
//if the onAddAction attached an observer to the subscription
var streamProvider = this.providerRuntime.ServiceProvider
.GetService <IStreamProviderManager>()
.GetStreamProvider(streamId.ProviderName);
await handler.InvokeOnAdd(streamId, subscriptionId, isRewindable, streamProvider);
if (allStreamObservers.TryGetValue(subscriptionId, out observer))
{
return(await observer.DeliverItem(item, currentToken, handshakeToken));
}
}
}
logger.Warn((int)(ErrorCode.StreamProvider_NoStreamForItem), "{0} got an item for subscription {1}, but I don't have any subscriber for that stream. Dropping on the floor.",
providerRuntime.ExecutingEntityIdentity(), subscriptionId);
// We got an item when we don't think we're the subscriber. This is a normal race condition.
// We can drop the item on the floor, or pass it to the rendezvous, or ...
return(default(StreamHandshakeToken));
}