public async Task <StreamHandshakeToken> DeliverMutable(GuidId subscriptionId, InternalStreamId streamId, object item, StreamSequenceToken currentToken, StreamHandshakeToken handshakeToken)
{
if (logger.IsEnabled(LogLevel.Trace))
{
var itemString = item.ToString();
itemString = (itemString.Length > MAXIMUM_ITEM_STRING_LOG_LENGTH) ? itemString.Substring(0, MAXIMUM_ITEM_STRING_LOG_LENGTH) + "..." : itemString;
logger.Trace("DeliverItem {0} for subscription {1}", itemString, subscriptionId);
}
IStreamSubscriptionHandle observer;
if (allStreamObservers.TryGetValue(subscriptionId, out observer))
{
return(await observer.DeliverItem(item, currentToken, handshakeToken));
}
else if (this.streamSubscriptionObserver != null)
{
var streamProvider = this.providerRuntime.ServiceProvider.GetServiceByName <IStreamProvider>(streamId.ProviderName);
if (streamProvider != null)
{
var subscriptionHandlerFactory = new StreamSubscriptionHandlerFactory(streamProvider, streamId, streamId.ProviderName, subscriptionId);
await this.streamSubscriptionObserver.OnSubscribed(subscriptionHandlerFactory);
//check if an observer were attached after handling the new subscription, deliver on it if attached
if (allStreamObservers.TryGetValue(subscriptionId, out observer))
{
return(await observer.DeliverItem(item, currentToken, handshakeToken));
}
}
}
logger.Warn((int)(ErrorCode.StreamProvider_NoStreamForItem), "{0} got an item for subscription {1}, but I don't have any subscriber for that stream. Dropping on the floor.",
providerRuntime.ExecutingEntityIdentity(), subscriptionId);
// We got an item when we don't think we're the subscriber. This is a normal race condition.
// We can drop the item on the floor, or pass it to the rendezvous, or ...
return(default(StreamHandshakeToken));
}
public Task <int> ConsumerCount(InternalStreamId streamId)
{
var streamRendezvous = GetRendezvousGrain(streamId);
return(streamRendezvous.ConsumerCount(streamId));
}
private IPubSubRendezvousGrain GetRendezvousGrain(InternalStreamId streamId)
{
return(grainFactory.GetGrain <IPubSubRendezvousGrain>(streamId.ToString()));
}
public Task <ISet <PubSubSubscriptionState> > RegisterProducer(InternalStreamId streamId, IStreamProducerExtension streamProducer)
{
var streamRendezvous = GetRendezvousGrain(streamId);
return(streamRendezvous.RegisterProducer(streamId, streamProducer));
}
public Task RegisterConsumer(GuidId subscriptionId, InternalStreamId streamId, IStreamConsumerExtension streamConsumer, string filterData)
{
var streamRendezvous = GetRendezvousGrain(streamId);
return(streamRendezvous.RegisterConsumer(subscriptionId, streamId, streamConsumer, filterData));
}
public Task <int> ConsumerCount(InternalStreamId streamId)
{
return(Task.FromResult(GetConsumersForStream(streamId).Length));
}
private PubSubSubscriptionState[] GetConsumersForStream(InternalStreamId streamId)
{
return(State.Consumers.Where(c => !c.IsFaulted && c.Stream.Equals(streamId)).ToArray());
}
private static async Task <ISet <PubSubSubscriptionState> > PubsubRegisterProducer(IStreamPubSub pubSub, InternalStreamId streamId,
IStreamProducerExtension meAsStreamProducer, ILogger logger)
{
try
{
var streamData = await pubSub.RegisterProducer(streamId, meAsStreamProducer);
return(streamData);
}
catch (Exception e)
{
logger.Error(ErrorCode.PersistentStreamPullingAgent_17, $"RegisterAsStreamProducer failed due to {e}", e);
throw e;
}
}
internal FaultedSubscriptionException(GuidId subscriptionId, InternalStreamId streamId)
: base(string.Format(ErrorStringFormat, subscriptionId.Guid, streamId))
{
}
public Task RemoveSubscriber(GuidId subscriptionId, InternalStreamId streamId)
{
RemoveSubscriber_Impl(subscriptionId, streamId);
return(Task.CompletedTask);
}
/// <summary>
/// Read from queue.
/// Returns true, if data was read, false if it was not
/// </summary>
/// <param name="myQueueId"></param>
/// <param name="rcvr"></param>
/// <param name="maxCacheAddCount"></param>
/// <returns></returns>
private async Task <bool> ReadFromQueue(QueueId myQueueId, IQueueAdapterReceiver rcvr, int maxCacheAddCount)
{
if (rcvr == null)
{
return(false);
}
var now = DateTime.UtcNow;
// Try to cleanup the pubsub cache at the cadence of 10 times in the configurable StreamInactivityPeriod.
if ((now - lastTimeCleanedPubSubCache) >= this.options.StreamInactivityPeriod.Divide(StreamInactivityCheckFrequency))
{
lastTimeCleanedPubSubCache = now;
CleanupPubSubCache(now);
}
if (queueCache != null)
{
IList <IBatchContainer> purgedItems;
if (queueCache.TryPurgeFromCache(out purgedItems))
{
try
{
await rcvr.MessagesDeliveredAsync(purgedItems);
}
catch (Exception exc)
{
logger.Warn(ErrorCode.PersistentStreamPullingAgent_27,
$"Exception calling MessagesDeliveredAsync on queue {myQueueId}. Ignoring.", exc);
}
}
}
if (queueCache != null && queueCache.IsUnderPressure())
{
// Under back pressure. Exit the loop. Will attempt again in the next timer callback.
logger.Info((int)ErrorCode.PersistentStreamPullingAgent_24, "Stream cache is under pressure. Backing off.");
return(false);
}
// Retrieve one multiBatch from the queue. Every multiBatch has an IEnumerable of IBatchContainers, each IBatchContainer may have multiple events.
IList <IBatchContainer> multiBatch = await rcvr.GetQueueMessagesAsync(maxCacheAddCount);
if (multiBatch == null || multiBatch.Count == 0)
{
return(false); // queue is empty. Exit the loop. Will attempt again in the next timer callback.
}
queueCache?.AddToCache(multiBatch);
numMessages += multiBatch.Count;
numReadMessagesCounter.IncrementBy(multiBatch.Count);
if (logger.IsEnabled(LogLevel.Trace))
{
logger.Trace(ErrorCode.PersistentStreamPullingAgent_11, "Got {0} messages from queue {1}. So far {2} msgs from this queue.",
multiBatch.Count, myQueueId.ToStringWithHashCode(), numMessages);
}
foreach (var group in
multiBatch
.Where(m => m != null)
.GroupBy(container => container.StreamId))
{
var streamId = new InternalStreamId(queueAdapter.Name, group.Key);
StreamSequenceToken startToken = group.First().SequenceToken;
StreamConsumerCollection streamData;
if (pubSubCache.TryGetValue(streamId, out streamData))
{
streamData.RefreshActivity(now);
if (streamData.StreamRegistered)
{
StartInactiveCursors(streamData,
startToken); // if this is an existing stream, start any inactive cursors
}
else
{
if (this.logger.IsEnabled(LogLevel.Debug))
{
this.logger.LogDebug($"Pulled new messages in stream {streamId} from the queue, but pulling agent haven't succeeded in" +
$"RegisterStream yet, will start deliver on this stream after RegisterStream succeeded");
}
}
}
else
{
RegisterStream(streamId, startToken, now).Ignore(); // if this is a new stream register as producer of stream in pub sub system
}
}
return(true);
}
public Task AddSubscriber(GuidId subscriptionId, InternalStreamId streamId, IStreamConsumerExtension streamConsumer, string filterData)
{
return(Task.CompletedTask);
}
public Task <IEnumerable <StreamSubscription> > GetSubscriptions(string streamProviderName, StreamId streamId)
{
var internalStreamId = new InternalStreamId(streamProviderName, streamId);
return(streamPubSub.GetAllSubscriptions(internalStreamId).ContinueWith(subs => subs.Result.AsEnumerable()));
}
public async Task RemoveSubscription(string streamProviderName, StreamId streamId, Guid subscriptionId)
{
var internalStreamId = new InternalStreamId(streamProviderName, streamId);
await streamPubSub.UnregisterConsumer(GuidId.GetGuidId(subscriptionId), internalStreamId);
}
public async Task RegisterConsumer(
GuidId subscriptionId,
InternalStreamId streamId,
IStreamConsumerExtension streamConsumer,
string filterData)
{
counterConsumersAdded.Increment();
PubSubSubscriptionState pubSubState = State.Consumers.FirstOrDefault(s => s.Equals(subscriptionId));
if (pubSubState != null && pubSubState.IsFaulted)
{
throw new FaultedSubscriptionException(subscriptionId, streamId);
}
try
{
if (pubSubState == null)
{
pubSubState = new PubSubSubscriptionState(subscriptionId, streamId, streamConsumer);
State.Consumers.Add(pubSubState);
}
if (!string.IsNullOrWhiteSpace(filterData))
{
pubSubState.AddFilter(filterData);
}
LogPubSubCounts("RegisterConsumer {0}", streamConsumer);
await WriteStateAsync();
counterConsumersTotal.Increment();
}
catch (Exception exc)
{
logger.Error(ErrorCode.Stream_RegisterConsumerFailed,
$"Failed to register a stream consumer. Stream: {streamId}, SubscriptionId {subscriptionId}, Consumer: {streamConsumer}", exc);
// Corrupted state, deactivate grain.
DeactivateOnIdle();
throw;
}
int numProducers = State.Producers.Count;
if (numProducers <= 0)
{
return;
}
if (logger.IsEnabled(LogLevel.Debug))
{
logger.Debug("Notifying {0} existing producer(s) about new consumer {1}. Producers={2}",
numProducers, streamConsumer, Utils.EnumerableToString(State.Producers));
}
// Notify producers about a new streamConsumer.
var tasks = new List <Task>();
var producers = State.Producers.ToList();
int initialProducerCount = producers.Count;
try
{
foreach (PubSubPublisherState producerState in producers)
{
tasks.Add(ExecuteProducerTask(producerState, producerState.Producer.AddSubscriber(subscriptionId, streamId, streamConsumer, filterData)));
}
Exception exception = null;
try
{
await Task.WhenAll(tasks);
}
catch (Exception exc)
{
exception = exc;
}
// if the number of producers has been changed, resave state.
if (State.Producers.Count != initialProducerCount)
{
await WriteStateAsync();
counterConsumersTotal.DecrementBy(initialProducerCount - State.Producers.Count);
}
if (exception != null)
{
throw exception;
}
}
catch (Exception exc)
{
logger.Error(ErrorCode.Stream_RegisterConsumerFailed,
$"Failed to update producers while register a stream consumer. Stream: {streamId}, SubscriptionId {subscriptionId}, Consumer: {streamConsumer}", exc);
// Corrupted state, deactivate grain.
DeactivateOnIdle();
throw;
}
}
private async Task DeliverToRemote(IStreamConsumerExtension remoteConsumer, InternalStreamId streamId, GuidId subscriptionId, object item, bool optimizeForImmutableData, bool fireAndForgetDelivery)
{
try
{
if (optimizeForImmutableData)
{
await remoteConsumer.DeliverImmutable(subscriptionId, streamId, new Immutable <object>(item), null, null);
}
else
{
await remoteConsumer.DeliverMutable(subscriptionId, streamId, item, null, null);
}
}
catch (ClientNotAvailableException)
{
if (consumers.TryRemove(subscriptionId, out _))
{
streamPubSub.UnregisterConsumer(subscriptionId, streamId).Ignore();
logger.Warn(ErrorCode.Stream_ConsumerIsDead,
"Consumer {0} on stream {1} is no longer active - permanently removing Consumer.", remoteConsumer, streamId);
}
}
catch (Exception ex)
{
if (!fireAndForgetDelivery)
{
throw;
}
this.logger.LogWarning(ex, "Failed to deliver message to consumer on {SubscriptionId} for stream {StreamId}.", subscriptionId, streamId);
}
}
public Task <int> ProducerCount(InternalStreamId streamId)
{
return(Task.FromResult(State.Producers.Count));
}
private async Task NotifyComplete(IStreamConsumerExtension remoteConsumer, GuidId subscriptionId, InternalStreamId streamId, bool fireAndForgetDelivery)
{
try
{
await remoteConsumer.CompleteStream(subscriptionId);
} catch (Exception ex)
{
if (!fireAndForgetDelivery)
{
throw;
}
this.logger.LogWarning(ex, "Failed to notify consumer of stream completion on {SubscriptionId} for stream {StreamId}.", subscriptionId, streamId);
}
}
public Task <PubSubSubscriptionState[]> DiagGetConsumers(InternalStreamId streamId)
{
return(Task.FromResult(GetConsumersForStream(streamId)));
}
private async Task NotifyError(IStreamConsumerExtension remoteConsumer, GuidId subscriptionId, Exception exc, InternalStreamId streamId, bool fireAndForgetDelivery)
{
try
{
await remoteConsumer.ErrorInStream(subscriptionId, exc);
}
catch (Exception ex)
{
if (!fireAndForgetDelivery)
{
throw;
}
this.logger.LogWarning(ex, "Failed to notify consumer of stream error on {SubscriptionId} for stream {StreamId}. Error: {ErrorException}", subscriptionId, streamId, exc);
}
}
private async Task NotifyProducersOfRemovedSubscription(GuidId subscriptionId, InternalStreamId streamId)
{
int numProducersBeforeNotify = State.Producers.Count;
if (numProducersBeforeNotify > 0)
{
if (logger.IsEnabled(LogLevel.Debug))
{
logger.Debug("Notifying {0} existing producers about unregistered consumer.", numProducersBeforeNotify);
}
// Notify producers about unregistered consumer.
List <Task> tasks = State.Producers
.Select(producerState => ExecuteProducerTask(producerState, producerState.Producer.RemoveSubscriber(subscriptionId, streamId)))
.ToList();
await Task.WhenAll(tasks);
//if producers got removed
if (State.Producers.Count < numProducersBeforeNotify)
{
await this.WriteStateAsync();
}
}
}
internal void RemoveStream(InternalStreamId streamId)
{
remoteConsumers.Remove(streamId);
}
public Task UnregisterProducer(InternalStreamId streamId, IStreamProducerExtension streamProducer)
{
var streamRendezvous = GetRendezvousGrain(streamId);
return(streamRendezvous.UnregisterProducer(streamId, streamProducer));
}
internal StreamEventDeliveryFailureException(InternalStreamId streamId)
: base(string.Format(ErrorStringFormat, streamId.GetNamespace(), streamId.StreamId))
{
}
public Task UnregisterConsumer(GuidId subscriptionId, InternalStreamId streamId)
{
var streamRendezvous = GetRendezvousGrain(streamId);
return(streamRendezvous.UnregisterConsumer(subscriptionId, streamId));
}
internal bool SameStreamId(InternalStreamId streamId)
{
return(IsValid && streamImpl.InternalStreamId.Equals(streamId));
}
public Task <List <StreamSubscription> > GetAllSubscriptions(InternalStreamId streamId, IStreamConsumerExtension streamConsumer = null)
{
var streamRendezvous = GetRendezvousGrain(streamId);
return(streamRendezvous.GetAllSubscriptions(streamId, streamConsumer));
}
private async Task <double> TestOneStream(Guid streamId, string streamProviderName,
int numProducers, int numConsumers, int numMessages,
bool useFanOut = true)
{
output.WriteLine("Testing Stream {0} with Producers={1} Consumers={2} x {3} messages",
streamId, numProducers, numConsumers, numMessages);
Stopwatch sw = Stopwatch.StartNew();
List <IStreamLifecycleConsumerGrain> consumers = new List <IStreamLifecycleConsumerGrain>();
List <IStreamLifecycleProducerGrain> producers = new List <IStreamLifecycleProducerGrain>();
await InitializeTopology(streamId, this.StreamNamespace, streamProviderName,
numProducers, numConsumers,
producers, consumers, useFanOut);
var promises = new List <Task>();
// Producers send M message each
int item = 1;
AsyncPipeline pipeline = new AsyncPipeline(MessagePipelineSize);
foreach (var grain in producers)
{
for (int m = 0; m < numMessages; m++)
{
Task promise = grain.SendItem(item++);
if (useFanOut)
{
pipeline.Add(promise);
promises.Add(promise);
}
else
{
await promise;
}
}
}
if (useFanOut)
{
//output.WriteLine("Test: Waiting for {0} producers to finish sending {1} messages", producers.Count, promises.Count);
await Task.WhenAll(promises);
promises.Clear();
}
var pubSub = StreamTestUtils.GetStreamPubSub(this.InternalClient);
// Check Consumer counts
var streamId1 = new InternalStreamId(streamProviderName, StreamId.Create(StreamNamespace, streamId));
int consumerCount = await pubSub.ConsumerCount(streamId1);
Assert.Equal(numConsumers, consumerCount); // "ConsumerCount for Stream {0}", streamId
// Check Producer counts
int producerCount = await pubSub.ProducerCount(streamId1);
Assert.Equal(numProducers, producerCount); // "ProducerCount for Stream {0}", streamId
// Check message counts received by consumers
int totalMessages = (numMessages + 1) * numProducers;
foreach (var grain in consumers)
{
int count = await grain.GetReceivedCount();
Assert.Equal(totalMessages, count); // "ReceivedCount for Consumer grain {0}", grain.GetPrimaryKey());
}
double rps = totalMessages / sw.Elapsed.TotalSeconds;
//output.WriteLine("Sent {0} messages total from {1} Producers to {2} Consumers in {3} at {4} RPS",
// totalMessages, numProducers, numConsumers,
// sw.Elapsed, rps);
return(rps);
}
public GuidId CreateSubscriptionId(InternalStreamId streamId, IStreamConsumerExtension streamConsumer)
{
Guid subscriptionId = SubscriptionMarker.MarkAsExplicitSubscriptionId(Guid.NewGuid());
return(GuidId.GetGuidId(subscriptionId));
}
public Task <StreamHandshakeToken> DeliverImmutable(GuidId subscriptionId, InternalStreamId streamId, Immutable <object> item, StreamSequenceToken currentToken, StreamHandshakeToken handshakeToken)
{
return(DeliverMutable(subscriptionId, streamId, item.Value, currentToken, handshakeToken));
}
