public StreamConsumer(StreamImpl <T> stream, string streamProviderName, IStreamProviderRuntime providerUtilities, IStreamPubSub pubSub, bool isRewindable)
{
if (stream == null)
{
throw new ArgumentNullException("stream");
}
if (providerUtilities == null)
{
throw new ArgumentNullException("providerUtilities");
}
if (pubSub == null)
{
throw new ArgumentNullException("pubSub");
}
logger = TraceLogger.GetLogger(string.Format("StreamConsumer<{0}>-{1}", typeof(T).Name, stream), TraceLogger.LoggerType.Runtime);
this.stream = stream;
this.streamProviderName = streamProviderName;
providerRuntime = providerUtilities;
this.pubSub = pubSub;
IsRewindable = isRewindable;
myExtension = null;
myGrainReference = null;
bindExtLock = new AsyncLock();
}
private async Task BindExtensionLazy()
{
if (myExtension == null)
{
using (await bindExtLock.LockAsync())
{
if (myExtension == null)
{
if (logger.IsVerbose)
{
logger.Verbose("BindExtensionLazy - Binding local extension to stream runtime={0}", providerRuntime);
}
var tup = await providerRuntime.BindExtension <StreamConsumerExtension, IStreamConsumerExtension>(
() => new StreamConsumerExtension(providerRuntime, IsRewindable));
myExtension = tup.Item1;
myGrainReference = tup.Item2;
if (logger.IsVerbose)
{
logger.Verbose("BindExtensionLazy - Connected Extension={0} GrainRef={1}", myExtension, myGrainReference);
}
}
}
}
}
public async Task UnregisterConsumer(StreamId streamId, IStreamConsumerExtension streamConsumer)
{
int numRemoved = State.Consumers.RemoveWhere(c => c.Equals(streamId, streamConsumer));
counterConsumersRemoved.Increment();
counterConsumersTotal.DecrementBy(numRemoved);
LogPubSubCounts("UnregisterConsumer {0} NumRemoved={1}", streamConsumer, numRemoved);
await State.WriteStateAsync();
int numProducers = State.Producers.Count;
if (numProducers > 0)
{
if (logger.IsVerbose)
{
logger.Verbose("Notifying {0} existing producers about unregistered consumer.", numProducers);
}
// Notify producers about unregistered consumer.
var tasks = new List <Task>();
foreach (var producerState in State.Producers
.Where(producerState => IsActiveProducer(producerState.Producer)))
{
tasks.Add(producerState.Producer.RemoveSubscriber(streamId, streamConsumer));
}
await Task.WhenAll(tasks);
}
else if (State.Consumers.Count == 0) // + we already know that numProducers == 0 from previous if-clause
{
// No producers or consumers left now, so flag ourselves to expedite Deactivation
DeactivateOnIdle();
}
}
// Called by rendezvous when new remote subscriber subscribes to this stream.
private void AddSubscriber_Impl(
StreamId streamId,
IStreamConsumerExtension streamConsumer,
StreamSequenceToken token,
IStreamFilterPredicateWrapper filter)
{
StreamConsumerCollection streamDataCollection;
if (!pubSubCache.TryGetValue(streamId, out streamDataCollection))
{
streamDataCollection = new StreamConsumerCollection();
pubSubCache.Add(streamId, streamDataCollection);
}
StreamConsumerData data;
if (!streamDataCollection.TryGetConsumer(streamConsumer, out data))
{
data = streamDataCollection.AddConsumer(streamId, streamConsumer, token, filter);
}
// Set cursor if not cursor is set, or if subscription provides new token
if (data.Cursor == null || token != null)
{
data.Cursor = this.receiver.GetCacheCursor(streamId.Guid, streamId.Namespace, token);
}
if (data.State == StreamConsumerDataState.Inactive)
{
RunConsumerCursor(data, filter).Ignore(); // Start delivering events if not actively doing so
}
}
public StreamConsumerData(GuidId subscriptionId, StreamId streamId, IStreamConsumerExtension streamConsumer, IStreamFilterPredicateWrapper filter)
{
SubscriptionId = subscriptionId;
StreamId = streamId;
StreamConsumer = streamConsumer;
Filter = filter;
}
internal Task DeliverItem(StreamId streamId, object item, bool fireAndForgetDelivery, bool optimizeForImmutableData)
{
var tasks = fireAndForgetDelivery ? null : new List <Task>();
foreach (KeyValuePair <GuidId, Tuple <IStreamConsumerExtension, IStreamFilterPredicateWrapper> > subscriptionKvp in consumers)
{
IStreamConsumerExtension remoteConsumer = subscriptionKvp.Value.Item1;
// Apply filter(s) to see if we should forward this item to this consumer
IStreamFilterPredicateWrapper filter = subscriptionKvp.Value.Item2;
if (filter != null)
{
if (!filter.ShouldReceive(streamId, filter.FilterData, item))
{
continue;
}
}
Task task = DeliverToRemote(remoteConsumer, streamId, subscriptionKvp.Key, item, optimizeForImmutableData);
if (fireAndForgetDelivery)
{
task.Ignore();
}
else
{
tasks.Add(task);
}
}
// If there's no subscriber, presumably we just drop the item on the floor
return(fireAndForgetDelivery ? TaskDone.Done : Task.WhenAll(tasks));
}
public StreamConsumerData(GuidId subscriptionId, StreamId streamId, IStreamConsumerExtension streamConsumer, IStreamFilterPredicateWrapper filter)
{
SubscriptionId = subscriptionId;
StreamId = streamId;
StreamConsumer = streamConsumer;
Filter = filter;
}
// Called by rendezvous when new remote subscriber subscribes to this stream.
private async Task AddSubscriber_Impl(
GuidId subscriptionId,
StreamId streamId,
IStreamConsumerExtension streamConsumer,
StreamSequenceToken cacheToken,
IStreamFilterPredicateWrapper filter)
{
if (IsShutdown)
{
return;
}
StreamConsumerCollection streamDataCollection;
if (!pubSubCache.TryGetValue(streamId, out streamDataCollection))
{
streamDataCollection = new StreamConsumerCollection(DateTime.UtcNow);
pubSubCache.Add(streamId, streamDataCollection);
}
StreamConsumerData data;
if (!streamDataCollection.TryGetConsumer(subscriptionId, out data))
{
data = streamDataCollection.AddConsumer(subscriptionId, streamId, streamConsumer, filter ?? DefaultStreamFilter);
}
if (await DoHandshakeWithConsumer(data, cacheToken))
{
if (data.State == StreamConsumerDataState.Inactive)
{
RunConsumerCursor(data, data.Filter).Ignore(); // Start delivering events if not actively doing so
}
}
}
public StreamConsumer(
StreamImpl <T> stream,
string streamProviderName,
IStreamProviderRuntime runtime,
IStreamPubSub pubSub,
ILogger logger,
bool isRewindable)
{
if (stream == null)
{
throw new ArgumentNullException(nameof(stream));
}
if (runtime == null)
{
throw new ArgumentNullException(nameof(runtime));
}
if (pubSub == null)
{
throw new ArgumentNullException(nameof(pubSub));
}
if (logger == null)
{
throw new ArgumentNullException(nameof(logger));
}
this.logger = logger;
this.stream = stream;
this.streamProviderName = streamProviderName;
this.providerRuntime = runtime;
this.pubSub = pubSub;
this.IsRewindable = isRewindable;
this.myExtension = null;
this.myGrainReference = null;
this.bindExtLock = new AsyncLock();
}
// Called by rendezvous when new remote subscriber subscribes to this stream.
private async Task AddSubscriber_Impl(
GuidId subscriptionId,
StreamId streamId,
IStreamConsumerExtension streamConsumer,
StreamSequenceToken cacheToken,
IStreamFilterPredicateWrapper filter)
{
if (IsShutdown)
{
return;
}
StreamConsumerCollection streamDataCollection;
if (!pubSubCache.TryGetValue(streamId, out streamDataCollection))
{
// If stream is not in pubsub cache, then we've received no events on this stream, and will aquire the subscriptions from pubsub when we do.
return;
}
StreamConsumerData data;
if (!streamDataCollection.TryGetConsumer(subscriptionId, out data))
{
data = streamDataCollection.AddConsumer(subscriptionId, streamId, streamConsumer, filter ?? DefaultStreamFilter);
}
if (await DoHandshakeWithConsumer(data, cacheToken))
{
if (data.State == StreamConsumerDataState.Inactive)
{
RunConsumerCursor(data, data.Filter).Ignore(); // Start delivering events if not actively doing so
}
}
}
public StreamConsumerData AddConsumer(GuidId subscriptionId, StreamId streamId, IStreamConsumerExtension streamConsumer, IStreamFilterPredicateWrapper filter)
{
var consumerData = new StreamConsumerData(subscriptionId, streamId, streamConsumer, filter);
queueData.Add(subscriptionId, consumerData);
lastActivityTime = DateTime.UtcNow;
return consumerData;
}
private async Task DeliverToRemote(IStreamConsumerExtension remoteConsumer, InternalStreamId streamId, GuidId subscriptionId, object item, bool optimizeForImmutableData, bool fireAndForgetDelivery)
{
try
{
if (optimizeForImmutableData)
{
await remoteConsumer.DeliverImmutable(subscriptionId, streamId, new Immutable <object>(item), null, null);
}
else
{
await remoteConsumer.DeliverMutable(subscriptionId, streamId, item, null, null);
}
}
catch (ClientNotAvailableException)
{
if (consumers.TryRemove(subscriptionId, out _))
{
streamPubSub.UnregisterConsumer(subscriptionId, streamId).Ignore();
logger.Warn(ErrorCode.Stream_ConsumerIsDead,
"Consumer {0} on stream {1} is no longer active - permanently removing Consumer.", remoteConsumer, streamId);
}
}
catch (Exception ex)
{
if (!fireAndForgetDelivery)
{
throw;
}
this.logger.LogWarning(ex, "Failed to deliver message to consumer on {SubscriptionId} for stream {StreamId}.", subscriptionId, streamId);
}
}
/// <summary>
/// Retrieve a map of implicit subscriptionsIds to implicit subscribers, given a stream ID. This method throws an exception if there's no namespace associated with the stream ID.
/// </summary>
/// <param name="streamId">A stream ID.</param>
/// <returns>A set of references to implicitly subscribed grains. They are expected to support the streaming consumer extension.</returns>
/// <exception cref="System.ArgumentException">The stream ID doesn't have an associated namespace.</exception>
/// <exception cref="System.InvalidOperationException">Internal invariant violation.</exception>
internal IDictionary <Guid, IStreamConsumerExtension> GetImplicitSubscribers(StreamId streamId)
{
if (String.IsNullOrWhiteSpace(streamId.Namespace))
{
throw new ArgumentException("The stream ID doesn't have an associated namespace.", "streamId");
}
HashSet <int> entry;
var result = new Dictionary <Guid, IStreamConsumerExtension>();
if (table.TryGetValue(streamId.Namespace, out entry))
{
foreach (var i in entry)
{
IStreamConsumerExtension consumer = MakeConsumerReference(streamId.Guid, i);
Guid subscriptionGuid = MakeSubscriptionGuid(i, streamId);
if (result.ContainsKey(subscriptionGuid))
{
throw new InvalidOperationException(string.Format("Internal invariant violation: generated duplicate subscriber reference: {0}, subscriptionId: {1}", consumer, subscriptionGuid));
}
result.Add(subscriptionGuid, consumer);
}
return(result);
}
return(result);
}
public Task <List <GuidId> > GetAllSubscriptions(StreamId streamId, IStreamConsumerExtension streamConsumer)
{
List <GuidId> subscriptionIds = State.Consumers.Where(c => !c.IsFaulted && c.Consumer.Equals(streamConsumer))
.Select(c => c.SubscriptionId)
.ToList();
return(Task.FromResult(subscriptionIds));
}
public Task RegisterConsumer(GuidId subscriptionId, StreamId streamId, string streamProvider, IStreamConsumerExtension streamConsumer, IStreamFilterPredicateWrapper filter)
{
if (!IsImplicitSubscriber(streamConsumer, streamId))
{
throw new ArgumentOutOfRangeException(streamId.ToString(), "Only implicit subscriptions are supported.");
}
return TaskDone.Done;
}
public Task<List<GuidId>> GetAllSubscriptions(StreamId streamId, IStreamConsumerExtension streamConsumer)
{
if (!IsImplicitSubscriber(streamConsumer, streamId))
{
throw new ArgumentOutOfRangeException(streamId.ToString(), "Only implicit subscriptions are supported.");
}
return Task.FromResult(new List<GuidId> { GuidId.GetGuidId(streamConsumer.GetPrimaryKey()) });
}
public IStreamFilterPredicateWrapper FilterWrapper; // Serialized func info
internal PubSubSubscriptionState(StreamId streamId, IStreamConsumerExtension streamConsumer,
StreamSequenceToken token,
IStreamFilterPredicateWrapper filterWrapper)
{
Stream = streamId;
Consumer = streamConsumer;
StreamSequenceToken = token;
FilterWrapper = filterWrapper;
}
public Task <List <GuidId> > GetAllSubscriptions(StreamId streamId, IStreamConsumerExtension streamConsumer)
{
if (!IsImplicitSubscriber(streamConsumer, streamId))
{
throw new ArgumentOutOfRangeException(streamId.ToString(), "Only implicit subscriptions are supported.");
}
return(Task.FromResult(new List <GuidId> {
GuidId.GetGuidId(streamConsumer.GetPrimaryKey())
}));
}
public GuidId CreateSubscriptionId(StreamId streamId, IStreamConsumerExtension streamConsumer)
{
GrainId grainId = GrainExtensions.GetGrainId(streamConsumer);
Guid subscriptionGuid;
if (!implicitTable.TryGetImplicitSubscriptionGuid(grainId, streamId, out subscriptionGuid))
{
throw new ArgumentOutOfRangeException(streamId.ToString(), "Only implicit subscriptions are supported.");
}
return GuidId.GetGuidId(subscriptionGuid);
}
// This constructor has to be public for JSonSerialization to work!
// Implement ISerializable if changing it to non-public
public PubSubSubscriptionState(
GuidId subscriptionId,
StreamId streamId,
IStreamConsumerExtension streamConsumer)
{
SubscriptionId = subscriptionId;
Stream = streamId;
consumerReference = streamConsumer as GrainReference;
state = SubscriptionStates.Active;
}
public GuidId CreateSubscriptionId(StreamId streamId, IStreamConsumerExtension streamConsumer)
{
GrainId grainId = GrainExtensions.GetGrainId(streamConsumer);
Guid subscriptionGuid;
if (!implicitTable.TryGetImplicitSubscriptionGuid(grainId, streamId, out subscriptionGuid))
{
throw new ArgumentOutOfRangeException(streamId.ToString(), "Only implicit subscriptions are supported.");
}
return(GuidId.GetGuidId(subscriptionGuid));
}
internal void RemoveRemoteSubscriber(IStreamConsumerExtension streamConsumer)
{
Guid key = streamConsumer.GetPrimaryKey();
Tuple <IStreamConsumerExtension, IStreamFilterPredicateWrapper> ignore;
consumers.TryRemove(key, out ignore);
if (consumers.Count == 0)
{
// Unsubscribe from PubSub?
}
}
public bool RemoveConsumer(IStreamConsumerExtension streamConsumer)
{
StreamConsumerData consumer;
if (!queueData.TryGetValue(streamConsumer, out consumer))
{
return(false);
}
consumer.Cursor = null; // kill cursor activity and ensure it does not start again on this consumer data.
return(queueData.Remove(streamConsumer));
}
public bool Equals(StreamId streamId, IStreamConsumerExtension streamConsumer)
{
if (ReferenceEquals(null, Stream))
{
return(false);
}
if (ReferenceEquals(null, Consumer))
{
return(false);
}
return(Stream.Equals(streamId) && Consumer.Equals(streamConsumer));
}
// This constructor has to be public for JSonSerialization to work!
// Implement ISerializable if changing it to non-public
public PubSubSubscriptionState(
GuidId subscriptionId,
StreamId streamId,
IStreamConsumerExtension streamConsumer,
IStreamFilterPredicateWrapper filterWrapper)
{
SubscriptionId = subscriptionId;
Stream = streamId;
consumerReference = streamConsumer as GrainReference;
this.filterWrapper = filterWrapper;
state = SubscriptionStates.Active;
}
public Task AddSubscriber(
StreamId streamId,
IStreamConsumerExtension streamConsumer,
StreamSequenceToken token,
IStreamFilterPredicateWrapper filter)
{
if (logger.IsVerbose)
{
logger.Verbose((int)ErrorCode.PersistentStreamPullingAgent_09, "AddSubscriber: Stream={0} Subscriber={1} Token={2}.", streamId, streamConsumer, token);
}
AddSubscriber_Impl(streamId, streamConsumer, token, filter);
return(TaskDone.Done);
}
public IStreamFilterPredicateWrapper FilterWrapper; // Serialized func info
// This constructor has to be public for JSonSerialization to work!
// Implement ISerializable if changing it to non-public
public PubSubSubscriptionState(
GuidId subscriptionId,
StreamId streamId,
IStreamConsumerExtension streamConsumer,
StreamSequenceToken token,
IStreamFilterPredicateWrapper filterWrapper)
{
SubscriptionId = subscriptionId;
Stream = streamId;
Consumer = streamConsumer;
StreamSequenceToken = token;
FilterWrapper = filterWrapper;
}
private async Task NotifyComplete(IStreamConsumerExtension remoteConsumer, GuidId subscriptionId, StreamId streamId, bool fireAndForgetDelivery)
{
try
{
await remoteConsumer.CompleteStream(subscriptionId);
} catch (Exception ex)
{
if (!fireAndForgetDelivery)
{
throw;
}
this.logger.LogWarning(ex, "Failed to notify consumer of stream completion on {SubscriptionId} for stream {StreamId}.", subscriptionId, streamId);
}
}
public Task AddSubscriber(
GuidId subscriptionId,
StreamId streamId,
IStreamConsumerExtension streamConsumer,
IStreamFilterPredicateWrapper filter)
{
if (logger.IsVerbose) logger.Verbose((int)ErrorCode.PersistentStreamPullingAgent_09, "AddSubscriber: Stream={0} Subscriber={1}.", streamId, streamConsumer);
// cannot await here because explicit consumers trigger this call, so it could cause a deadlock.
AddSubscriber_Impl(subscriptionId, streamId, streamConsumer, null, filter)
.LogException(logger, ErrorCode.PersistentStreamPullingAgent_26,
String.Format("Failed to add subscription for stream {0}." , streamId))
.Ignore();
return TaskDone.Done;
}
private async Task NotifyError(IStreamConsumerExtension remoteConsumer, GuidId subscriptionId, Exception exc, StreamId streamId, bool fireAndForgetDelivery)
{
try
{
await remoteConsumer.ErrorInStream(subscriptionId, exc);
}
catch (Exception ex)
{
if (!fireAndForgetDelivery)
{
throw;
}
this.logger.LogWarning(ex, "Failed to notify consumer of stream error on {SubscriptionId} for stream {StreamId}. Error: {ErrorException}", subscriptionId, streamId, exc);
}
}
public Task AddSubscriber(
GuidId subscriptionId,
InternalStreamId streamId,
IStreamConsumerExtension streamConsumer)
{
if (logger.IsEnabled(LogLevel.Debug))
{
logger.Debug(ErrorCode.PersistentStreamPullingAgent_09, "AddSubscriber: Stream={0} Subscriber={1}.", streamId, streamConsumer);
}
// cannot await here because explicit consumers trigger this call, so it could cause a deadlock.
AddSubscriber_Impl(subscriptionId, streamId, streamConsumer, null)
.LogException(logger, ErrorCode.PersistentStreamPullingAgent_26,
$"Failed to add subscription for stream {streamId}.")
.Ignore();
return(Task.CompletedTask);
}
private async Task NotifyProducer(PubSubPublisherState producer, GuidId subscriptionId, StreamId streamId,
IStreamConsumerExtension streamConsumer, IStreamFilterPredicateWrapper filter)
{
try
{
await producer.Producer.AddSubscriber(subscriptionId, streamId, streamConsumer, filter);
}
catch (GrainExtensionNotInstalledException)
{
RemoveProducer(producer);
}
catch (ClientNotAvailableException)
{
RemoveProducer(producer);
}
}
internal Task CompleteStream(StreamId streamId, bool fireAndForgetDelivery)
{
var tasks = fireAndForgetDelivery ? null : new List <Task>();
foreach (var data in consumers.Values)
{
IStreamConsumerExtension remoteConsumer = data.Item1;
Task task = remoteConsumer.CompleteStream(streamId);
if (fireAndForgetDelivery)
{
task.Ignore();
}
else
{
tasks.Add(task);
}
}
// If there's no subscriber, presumably we just drop the item on the floor
return(fireAndForgetDelivery ? TaskDone.Done : Task.WhenAll(tasks));
}
// Called by rendezvous when new remote subsriber subscribes to this stream.
public Task AddSubscriber(GuidId subscriptionId, StreamId streamId, IStreamConsumerExtension streamConsumer, IStreamFilterPredicateWrapper filter)
{
if (logger.IsVerbose)
{
logger.Verbose("{0} AddSubscriber {1} for stream {2}", providerRuntime.ExecutingEntityIdentity(), streamConsumer, streamId);
}
StreamConsumerExtensionCollection consumers;
if (remoteConsumers.TryGetValue(streamId, out consumers))
{
consumers.AddRemoteSubscriber(subscriptionId, streamConsumer, filter);
}
else
{
// We got an item when we don't think we're the subscriber. This is a normal race condition.
// We can drop the item on the floor, or pass it to the rendezvous, or log a warning.
}
return(TaskDone.Done);
}
// Called by rendezvous when new remote subsriber subscribes to this stream.
public Task AddSubscriber(GuidId subscriptionId, InternalStreamId streamId, IStreamConsumerExtension streamConsumer, string filterData)
{
if (logger.IsEnabled(LogLevel.Debug))
{
logger.LogDebug("{Grain} AddSubscriber {Subscriber} for stream {SreamId}", providerRuntime.ExecutingEntityIdentity(), streamConsumer, streamId);
}
StreamConsumerExtensionCollection consumers;
if (remoteConsumers.TryGetValue(streamId, out consumers))
{
consumers.AddRemoteSubscriber(subscriptionId, streamConsumer, filterData);
}
else
{
// We got an item when we don't think we're the subscriber. This is a normal race condition.
// We can drop the item on the floor, or pass it to the rendezvous, or log a warning.
}
return(Task.CompletedTask);
}
// Called by rendezvous when new remote subscriber subscribes to this stream or when registering a new stream with the pubsub system.
private void AddSubscriberToSubscriptionCache(
GuidId subscriptionId,
StreamId streamId,
IStreamConsumerExtension streamConsumer,
IQueueCacheCursor newCursor,
StreamSequenceToken requestedToken,
IStreamFilterPredicateWrapper filter)
{
StreamConsumerCollection streamDataCollection;
if (!pubSubCache.TryGetValue(streamId, out streamDataCollection))
{
streamDataCollection = new StreamConsumerCollection(DateTime.UtcNow);
pubSubCache.Add(streamId, streamDataCollection);
}
StreamConsumerData data;
if (!streamDataCollection.TryGetConsumer(subscriptionId, out data))
data = streamDataCollection.AddConsumer(subscriptionId, streamId, streamConsumer, filter);
data.LastToken = requestedToken;
// if we have a new cursor, use it
if (newCursor != null)
{
data.Cursor = newCursor;
} // else if we don't yet have a cursor, get a cursor at the end of the cash (null sequence token).
else if (data.Cursor == null && queueCache != null)
{
data.Cursor = queueCache.GetCacheCursor(streamId.Guid, streamId.Namespace, null);
}
if (data.State == StreamConsumerDataState.Inactive)
RunConsumerCursor(data, filter).Ignore(); // Start delivering events if not actively doing so
}
internal void AddRemoteSubscriber(GuidId subscriptionId, IStreamConsumerExtension streamConsumer, IStreamFilterPredicateWrapper filter)
{
consumers.TryAdd(subscriptionId, Tuple.Create(streamConsumer, filter));
}
// Called by rendezvous when new remote subscriber subscribes to this stream.
private async Task AddSubscriber_Impl(
GuidId subscriptionId,
StreamId streamId,
IStreamConsumerExtension streamConsumer,
StreamSequenceToken token,
IStreamFilterPredicateWrapper filter)
{
IQueueCacheCursor cursor = null;
// if not cache, then we can't get cursor and there is no reason to ask consumer for token.
if (queueCache != null)
{
try
{
StreamSequenceToken consumerToken = await streamConsumer.GetSequenceToken(subscriptionId);
// Set cursor if not cursor is set, or if subscription provides new token
consumerToken = consumerToken ?? token;
if (token != null)
{
cursor = queueCache.GetCacheCursor(streamId.Guid, streamId.Namespace, consumerToken);
}
}
catch (DataNotAvailableException dataNotAvailableException)
{
// notify consumer that the data is not available, if we can.
streamConsumer.ErrorInStream(subscriptionId, dataNotAvailableException).Ignore();
}
}
AddSubscriberToSubscriptionCache(subscriptionId, streamId, streamConsumer, cursor, filter);
}
// Called by rendezvous when new remote subscriber subscribes to this stream.
private async Task AddSubscriber_Impl(
GuidId subscriptionId,
StreamId streamId,
IStreamConsumerExtension streamConsumer,
StreamSequenceToken cacheToken,
IStreamFilterPredicateWrapper filter)
{
if (IsShutdown) return;
StreamConsumerCollection streamDataCollection;
if (!pubSubCache.TryGetValue(streamId, out streamDataCollection))
{
streamDataCollection = new StreamConsumerCollection(DateTime.UtcNow);
pubSubCache.Add(streamId, streamDataCollection);
}
StreamConsumerData data;
if (!streamDataCollection.TryGetConsumer(subscriptionId, out data))
data = streamDataCollection.AddConsumer(subscriptionId, streamId, streamConsumer, filter ?? DefaultStreamFilter);
if (await DoHandshakeWithConsumer(data, cacheToken))
{
if (data.State == StreamConsumerDataState.Inactive)
RunConsumerCursor(data, data.Filter).Ignore(); // Start delivering events if not actively doing so
}
}
// Called by rendezvous when new remote subscriber subscribes to this stream.
private void AddSubscriber_Impl(
GuidId subscriptionId,
StreamId streamId,
IStreamConsumerExtension streamConsumer,
StreamSequenceToken token,
IStreamFilterPredicateWrapper filter)
{
StreamConsumerCollection streamDataCollection;
if (!pubSubCache.TryGetValue(streamId, out streamDataCollection))
{
streamDataCollection = new StreamConsumerCollection();
pubSubCache.Add(streamId, streamDataCollection);
}
StreamConsumerData data;
if (!streamDataCollection.TryGetConsumer(subscriptionId, out data))
data = streamDataCollection.AddConsumer(subscriptionId, streamId, streamConsumer, token, filter);
// Set cursor if not cursor is set, or if subscription provides new token
if ((data.Cursor == null || token != null) && queueCache != null)
data.Cursor = queueCache.GetCacheCursor(streamId.Guid, streamId.Namespace, token);
if (data.State == StreamConsumerDataState.Inactive)
RunConsumerCursor(data, filter).Ignore(); // Start delivering events if not actively doing so
}
public async Task<List<GuidId>> GetAllSubscriptions(StreamId streamId, IStreamConsumerExtension streamConsumer)
{
return implicitPubSub.IsImplicitSubscriber(streamConsumer, streamId)
? await implicitPubSub.GetAllSubscriptions(streamId, streamConsumer)
: await explicitPubSub.GetAllSubscriptions(streamId, streamConsumer);
}
public Task RegisterConsumer(GuidId subscriptionId, StreamId streamId, string streamProvider, IStreamConsumerExtension streamConsumer, IStreamFilterPredicateWrapper filter)
{
return implicitPubSub.IsImplicitSubscriber(streamConsumer, streamId)
? implicitPubSub.RegisterConsumer(subscriptionId, streamId, streamProvider, streamConsumer, filter)
: explicitPubSub.RegisterConsumer(subscriptionId, streamId, streamProvider, streamConsumer, filter);
}
Task<List<GuidId>> IStreamPubSub.GetAllSubscriptions(StreamId streamId, IStreamConsumerExtension streamConsumer)
{
return registry.GetAllSubscriptions(streamId, streamConsumer);
}
Task IStreamPubSub.RegisterConsumer(GuidId subscriptionId, StreamId streamId, string streamProvider, IStreamConsumerExtension streamConsumer, IStreamFilterPredicateWrapper filter)
{
return registry.RegisterConsumer(subscriptionId, streamId, streamProvider, streamConsumer, filter);
}
GuidId IStreamPubSub.CreateSubscriptionId(StreamId streamId, IStreamConsumerExtension streamConsumer)
{
return registry.CreateSubscriptionId(streamId, streamConsumer);
}
public StreamConsumerData AddConsumer(GuidId subscriptionId, StreamId streamId, IStreamConsumerExtension streamConsumer, StreamSequenceToken token, IStreamFilterPredicateWrapper filter)
{
var consumerData = new StreamConsumerData(subscriptionId, streamId, streamConsumer, filter);
queueData.Add(subscriptionId, consumerData);
return consumerData;
}
public Task<List<GuidId>> GetAllSubscriptions(StreamId streamId, IStreamConsumerExtension streamConsumer)
{
return Task.FromResult(new List<GuidId> { CreateSubscriptionId(streamId, streamConsumer) });
}
public async Task<List<GuidId>> GetAllSubscriptions(StreamId streamId, IStreamConsumerExtension streamConsumer)
{
return IsImplicitSubscriber(streamConsumer, streamId)
? new List<GuidId>( new [] { GuidId.GetGuidId(streamConsumer.GetPrimaryKey()) } )
: await explicitPubSub.GetAllSubscriptions(streamId, streamConsumer);
}
public Task AddSubscriber(
GuidId subscriptionId,
StreamId streamId,
IStreamConsumerExtension streamConsumer,
IStreamFilterPredicateWrapper filter)
{
if (logger.IsVerbose) logger.Verbose((int)ErrorCode.PersistentStreamPullingAgent_09, "AddSubscriber: Stream={0} Subscriber={1}.", streamId, streamConsumer);
// cannot await here because explicit consumers trigger this call, so it could cause a deadlock.
AddSubscriber_Impl(subscriptionId, streamId, streamConsumer, null, filter)
.LogException(logger, ErrorCode.PersistentStreamPullingAgent_26,
String.Format("Failed to add subscription for stream {0}." , streamId))
.Ignore();
return TaskDone.Done;
}
public Task RegisterConsumer(GuidId subscriptionId, StreamId streamId, string streamProvider, IStreamConsumerExtension streamConsumer, IStreamFilterPredicateWrapper filter)
{
var streamRendezvous = GetRendezvousGrain(streamId);
return streamRendezvous.RegisterConsumer(subscriptionId, streamId, streamConsumer, filter);
}
public Task AddSubscriber(
GuidId subscriptionId,
StreamId streamId,
IStreamConsumerExtension streamConsumer,
StreamSequenceToken token,
IStreamFilterPredicateWrapper filter)
{
if (logger.IsVerbose) logger.Verbose((int)ErrorCode.PersistentStreamPullingAgent_09, "AddSubscriber: Stream={0} Subscriber={1} Token={2}.", streamId, streamConsumer, token);
AddSubscriber_Impl(subscriptionId, streamId, streamConsumer, token, filter);
return TaskDone.Done;
}
public Task<List<GuidId>> GetAllSubscriptions(StreamId streamId, IStreamConsumerExtension streamConsumer)
{
var streamRendezvous = GetRendezvousGrain(streamId);
return streamRendezvous.GetAllSubscriptions(streamId, streamConsumer);
}
// Called by rendezvous when new remote subscriber subscribes to this stream.
private async Task AddSubscriber_Impl(
GuidId subscriptionId,
StreamId streamId,
IStreamConsumerExtension streamConsumer,
StreamSequenceToken token,
IStreamFilterPredicateWrapper filter)
{
IQueueCacheCursor cursor = null;
StreamSequenceToken requestedToken = null;
// if not cache, then we can't get cursor and there is no reason to ask consumer for token.
if (queueCache != null)
{
DataNotAvailableException errorOccured = null;
try
{
requestedToken = await streamConsumer.GetSequenceToken(subscriptionId);
// Set cursor if not cursor is set, or if subscription provides new token
requestedToken = requestedToken ?? token;
if (requestedToken != null)
{
cursor = queueCache.GetCacheCursor(streamId.Guid, streamId.Namespace, requestedToken);
}
}
catch (DataNotAvailableException dataNotAvailableException)
{
errorOccured = dataNotAvailableException;
}
if (errorOccured != null)
{
// notify consumer that the data is not available, if we can.
await OrleansTaskExtentions.ExecuteAndIgnoreException(() => streamConsumer.ErrorInStream(subscriptionId, errorOccured));
}
}
AddSubscriberToSubscriptionCache(subscriptionId, streamId, streamConsumer, cursor, requestedToken, filter);
}
public GuidId CreateSubscriptionId(StreamId streamId, IStreamConsumerExtension streamConsumer)
{
Guid subscriptionId = SubscriptionMarker.MarkAsExplicitSubscriptionId(Guid.NewGuid());
return GuidId.GetGuidId(subscriptionId);
}
public GuidId CreateSubscriptionId(StreamId streamId, IStreamConsumerExtension streamConsumer)
{
return implicitPubSub.IsImplicitSubscriber(streamConsumer, streamId)
? implicitPubSub.CreateSubscriptionId(streamId, streamConsumer)
: explicitPubSub.CreateSubscriptionId(streamId, streamConsumer);
}
// Called by rendezvous when new remote subsriber subscribes to this stream.
public Task AddSubscriber(GuidId subscriptionId, StreamId streamId, IStreamConsumerExtension streamConsumer, IStreamFilterPredicateWrapper filter)
{
if (logger.IsVerbose)
{
logger.Verbose("{0} AddSubscriber {1} for stream {2}", providerRuntime.ExecutingEntityIdentity(), streamConsumer, streamId);
}
StreamConsumerExtensionCollection consumers;
if (remoteConsumers.TryGetValue(streamId, out consumers))
{
consumers.AddRemoteSubscriber(subscriptionId, streamConsumer, filter);
}
else
{
// We got an item when we don't think we're the subscriber. This is a normal race condition.
// We can drop the item on the floor, or pass it to the rendezvous, or log a warning.
}
return TaskDone.Done;
}
private async Task DeliverToRemote(IStreamConsumerExtension remoteConsumer, StreamId streamId, GuidId subscriptionId, object item, bool optimizeForImmutableData)
{
try
{
if (optimizeForImmutableData)
await remoteConsumer.DeliverImmutable(subscriptionId, new Immutable<object>(item), null, null);
else
await remoteConsumer.DeliverMutable(subscriptionId, item, null, null);
}
catch (ClientNotAvailableException)
{
Tuple<IStreamConsumerExtension, IStreamFilterPredicateWrapper> discard;
if (consumers.TryRemove(subscriptionId, out discard))
{
streamPubSub.UnregisterConsumer(subscriptionId, streamId, streamId.ProviderName).Ignore();
logger.Warn(ErrorCode.Stream_ConsumerIsDead,
"Consumer {0} on stream {1} is no longer active - permanently removing Consumer.", remoteConsumer, streamId);
}
}
}
public Task RegisterConsumer(GuidId subscriptionId, StreamId streamId, string streamProvider, IStreamConsumerExtension streamConsumer, StreamSequenceToken token, IStreamFilterPredicateWrapper filter)
{
return IsImplicitSubscriber(streamConsumer, streamId)
? TaskDone.Done
: explicitPubSub.RegisterConsumer(subscriptionId, streamId, streamProvider, streamConsumer, token, filter);
}
