public async Task <StreamHandshakeToken> DeliverMutable(GuidId subscriptionId, InternalStreamId streamId, object item, StreamSequenceToken currentToken, StreamHandshakeToken handshakeToken) { if (logger.IsEnabled(LogLevel.Trace)) { var itemString = item.ToString(); itemString = (itemString.Length > MAXIMUM_ITEM_STRING_LOG_LENGTH) ? itemString.Substring(0, MAXIMUM_ITEM_STRING_LOG_LENGTH) + "..." : itemString; logger.Trace("DeliverItem {0} for subscription {1}", itemString, subscriptionId); } IStreamSubscriptionHandle observer; if (allStreamObservers.TryGetValue(subscriptionId, out observer)) { return(await observer.DeliverItem(item, currentToken, handshakeToken)); } else if (this.streamSubscriptionObserver != null) { var streamProvider = this.providerRuntime.ServiceProvider.GetServiceByName <IStreamProvider>(streamId.ProviderName); if (streamProvider != null) { var subscriptionHandlerFactory = new StreamSubscriptionHandlerFactory(streamProvider, streamId, streamId.ProviderName, subscriptionId); await this.streamSubscriptionObserver.OnSubscribed(subscriptionHandlerFactory); //check if an observer were attached after handling the new subscription, deliver on it if attached if (allStreamObservers.TryGetValue(subscriptionId, out observer)) { return(await observer.DeliverItem(item, currentToken, handshakeToken)); } } } logger.Warn((int)(ErrorCode.StreamProvider_NoStreamForItem), "{0} got an item for subscription {1}, but I don't have any subscriber for that stream. Dropping on the floor.", providerRuntime.ExecutingEntityIdentity(), subscriptionId); // We got an item when we don't think we're the subscriber. This is a normal race condition. // We can drop the item on the floor, or pass it to the rendezvous, or ... return(default(StreamHandshakeToken)); }
public Task <int> ConsumerCount(InternalStreamId streamId) { var streamRendezvous = GetRendezvousGrain(streamId); return(streamRendezvous.ConsumerCount(streamId)); }
private IPubSubRendezvousGrain GetRendezvousGrain(InternalStreamId streamId) { return(grainFactory.GetGrain <IPubSubRendezvousGrain>(streamId.ToString())); }
public Task <ISet <PubSubSubscriptionState> > RegisterProducer(InternalStreamId streamId, IStreamProducerExtension streamProducer) { var streamRendezvous = GetRendezvousGrain(streamId); return(streamRendezvous.RegisterProducer(streamId, streamProducer)); }
public Task RegisterConsumer(GuidId subscriptionId, InternalStreamId streamId, IStreamConsumerExtension streamConsumer, string filterData) { var streamRendezvous = GetRendezvousGrain(streamId); return(streamRendezvous.RegisterConsumer(subscriptionId, streamId, streamConsumer, filterData)); }
public Task <int> ConsumerCount(InternalStreamId streamId) { return(Task.FromResult(GetConsumersForStream(streamId).Length)); }
private PubSubSubscriptionState[] GetConsumersForStream(InternalStreamId streamId) { return(State.Consumers.Where(c => !c.IsFaulted && c.Stream.Equals(streamId)).ToArray()); }
private static async Task <ISet <PubSubSubscriptionState> > PubsubRegisterProducer(IStreamPubSub pubSub, InternalStreamId streamId, IStreamProducerExtension meAsStreamProducer, ILogger logger) { try { var streamData = await pubSub.RegisterProducer(streamId, meAsStreamProducer); return(streamData); } catch (Exception e) { logger.Error(ErrorCode.PersistentStreamPullingAgent_17, $"RegisterAsStreamProducer failed due to {e}", e); throw e; } }
internal FaultedSubscriptionException(GuidId subscriptionId, InternalStreamId streamId) : base(string.Format(ErrorStringFormat, subscriptionId.Guid, streamId)) { }
public Task RemoveSubscriber(GuidId subscriptionId, InternalStreamId streamId) { RemoveSubscriber_Impl(subscriptionId, streamId); return(Task.CompletedTask); }
/// <summary> /// Read from queue. /// Returns true, if data was read, false if it was not /// </summary> /// <param name="myQueueId"></param> /// <param name="rcvr"></param> /// <param name="maxCacheAddCount"></param> /// <returns></returns> private async Task <bool> ReadFromQueue(QueueId myQueueId, IQueueAdapterReceiver rcvr, int maxCacheAddCount) { if (rcvr == null) { return(false); } var now = DateTime.UtcNow; // Try to cleanup the pubsub cache at the cadence of 10 times in the configurable StreamInactivityPeriod. if ((now - lastTimeCleanedPubSubCache) >= this.options.StreamInactivityPeriod.Divide(StreamInactivityCheckFrequency)) { lastTimeCleanedPubSubCache = now; CleanupPubSubCache(now); } if (queueCache != null) { IList <IBatchContainer> purgedItems; if (queueCache.TryPurgeFromCache(out purgedItems)) { try { await rcvr.MessagesDeliveredAsync(purgedItems); } catch (Exception exc) { logger.Warn(ErrorCode.PersistentStreamPullingAgent_27, $"Exception calling MessagesDeliveredAsync on queue {myQueueId}. Ignoring.", exc); } } } if (queueCache != null && queueCache.IsUnderPressure()) { // Under back pressure. Exit the loop. Will attempt again in the next timer callback. logger.Info((int)ErrorCode.PersistentStreamPullingAgent_24, "Stream cache is under pressure. Backing off."); return(false); } // Retrieve one multiBatch from the queue. Every multiBatch has an IEnumerable of IBatchContainers, each IBatchContainer may have multiple events. IList <IBatchContainer> multiBatch = await rcvr.GetQueueMessagesAsync(maxCacheAddCount); if (multiBatch == null || multiBatch.Count == 0) { return(false); // queue is empty. Exit the loop. Will attempt again in the next timer callback. } queueCache?.AddToCache(multiBatch); numMessages += multiBatch.Count; numReadMessagesCounter.IncrementBy(multiBatch.Count); if (logger.IsEnabled(LogLevel.Trace)) { logger.Trace(ErrorCode.PersistentStreamPullingAgent_11, "Got {0} messages from queue {1}. So far {2} msgs from this queue.", multiBatch.Count, myQueueId.ToStringWithHashCode(), numMessages); } foreach (var group in multiBatch .Where(m => m != null) .GroupBy(container => container.StreamId)) { var streamId = new InternalStreamId(queueAdapter.Name, group.Key); StreamSequenceToken startToken = group.First().SequenceToken; StreamConsumerCollection streamData; if (pubSubCache.TryGetValue(streamId, out streamData)) { streamData.RefreshActivity(now); if (streamData.StreamRegistered) { StartInactiveCursors(streamData, startToken); // if this is an existing stream, start any inactive cursors } else { if (this.logger.IsEnabled(LogLevel.Debug)) { this.logger.LogDebug($"Pulled new messages in stream {streamId} from the queue, but pulling agent haven't succeeded in" + $"RegisterStream yet, will start deliver on this stream after RegisterStream succeeded"); } } } else { RegisterStream(streamId, startToken, now).Ignore(); // if this is a new stream register as producer of stream in pub sub system } } return(true); }
public Task AddSubscriber(GuidId subscriptionId, InternalStreamId streamId, IStreamConsumerExtension streamConsumer, string filterData) { return(Task.CompletedTask); }
public Task <IEnumerable <StreamSubscription> > GetSubscriptions(string streamProviderName, StreamId streamId) { var internalStreamId = new InternalStreamId(streamProviderName, streamId); return(streamPubSub.GetAllSubscriptions(internalStreamId).ContinueWith(subs => subs.Result.AsEnumerable())); }
public async Task RemoveSubscription(string streamProviderName, StreamId streamId, Guid subscriptionId) { var internalStreamId = new InternalStreamId(streamProviderName, streamId); await streamPubSub.UnregisterConsumer(GuidId.GetGuidId(subscriptionId), internalStreamId); }
public async Task RegisterConsumer( GuidId subscriptionId, InternalStreamId streamId, IStreamConsumerExtension streamConsumer, string filterData) { counterConsumersAdded.Increment(); PubSubSubscriptionState pubSubState = State.Consumers.FirstOrDefault(s => s.Equals(subscriptionId)); if (pubSubState != null && pubSubState.IsFaulted) { throw new FaultedSubscriptionException(subscriptionId, streamId); } try { if (pubSubState == null) { pubSubState = new PubSubSubscriptionState(subscriptionId, streamId, streamConsumer); State.Consumers.Add(pubSubState); } if (!string.IsNullOrWhiteSpace(filterData)) { pubSubState.AddFilter(filterData); } LogPubSubCounts("RegisterConsumer {0}", streamConsumer); await WriteStateAsync(); counterConsumersTotal.Increment(); } catch (Exception exc) { logger.Error(ErrorCode.Stream_RegisterConsumerFailed, $"Failed to register a stream consumer. Stream: {streamId}, SubscriptionId {subscriptionId}, Consumer: {streamConsumer}", exc); // Corrupted state, deactivate grain. DeactivateOnIdle(); throw; } int numProducers = State.Producers.Count; if (numProducers <= 0) { return; } if (logger.IsEnabled(LogLevel.Debug)) { logger.Debug("Notifying {0} existing producer(s) about new consumer {1}. Producers={2}", numProducers, streamConsumer, Utils.EnumerableToString(State.Producers)); } // Notify producers about a new streamConsumer. var tasks = new List <Task>(); var producers = State.Producers.ToList(); int initialProducerCount = producers.Count; try { foreach (PubSubPublisherState producerState in producers) { tasks.Add(ExecuteProducerTask(producerState, producerState.Producer.AddSubscriber(subscriptionId, streamId, streamConsumer, filterData))); } Exception exception = null; try { await Task.WhenAll(tasks); } catch (Exception exc) { exception = exc; } // if the number of producers has been changed, resave state. if (State.Producers.Count != initialProducerCount) { await WriteStateAsync(); counterConsumersTotal.DecrementBy(initialProducerCount - State.Producers.Count); } if (exception != null) { throw exception; } } catch (Exception exc) { logger.Error(ErrorCode.Stream_RegisterConsumerFailed, $"Failed to update producers while register a stream consumer. Stream: {streamId}, SubscriptionId {subscriptionId}, Consumer: {streamConsumer}", exc); // Corrupted state, deactivate grain. DeactivateOnIdle(); throw; } }
private async Task DeliverToRemote(IStreamConsumerExtension remoteConsumer, InternalStreamId streamId, GuidId subscriptionId, object item, bool optimizeForImmutableData, bool fireAndForgetDelivery) { try { if (optimizeForImmutableData) { await remoteConsumer.DeliverImmutable(subscriptionId, streamId, new Immutable <object>(item), null, null); } else { await remoteConsumer.DeliverMutable(subscriptionId, streamId, item, null, null); } } catch (ClientNotAvailableException) { if (consumers.TryRemove(subscriptionId, out _)) { streamPubSub.UnregisterConsumer(subscriptionId, streamId).Ignore(); logger.Warn(ErrorCode.Stream_ConsumerIsDead, "Consumer {0} on stream {1} is no longer active - permanently removing Consumer.", remoteConsumer, streamId); } } catch (Exception ex) { if (!fireAndForgetDelivery) { throw; } this.logger.LogWarning(ex, "Failed to deliver message to consumer on {SubscriptionId} for stream {StreamId}.", subscriptionId, streamId); } }
public Task <int> ProducerCount(InternalStreamId streamId) { return(Task.FromResult(State.Producers.Count)); }
private async Task NotifyComplete(IStreamConsumerExtension remoteConsumer, GuidId subscriptionId, InternalStreamId streamId, bool fireAndForgetDelivery) { try { await remoteConsumer.CompleteStream(subscriptionId); } catch (Exception ex) { if (!fireAndForgetDelivery) { throw; } this.logger.LogWarning(ex, "Failed to notify consumer of stream completion on {SubscriptionId} for stream {StreamId}.", subscriptionId, streamId); } }
public Task <PubSubSubscriptionState[]> DiagGetConsumers(InternalStreamId streamId) { return(Task.FromResult(GetConsumersForStream(streamId))); }
private async Task NotifyError(IStreamConsumerExtension remoteConsumer, GuidId subscriptionId, Exception exc, InternalStreamId streamId, bool fireAndForgetDelivery) { try { await remoteConsumer.ErrorInStream(subscriptionId, exc); } catch (Exception ex) { if (!fireAndForgetDelivery) { throw; } this.logger.LogWarning(ex, "Failed to notify consumer of stream error on {SubscriptionId} for stream {StreamId}. Error: {ErrorException}", subscriptionId, streamId, exc); } }
private async Task NotifyProducersOfRemovedSubscription(GuidId subscriptionId, InternalStreamId streamId) { int numProducersBeforeNotify = State.Producers.Count; if (numProducersBeforeNotify > 0) { if (logger.IsEnabled(LogLevel.Debug)) { logger.Debug("Notifying {0} existing producers about unregistered consumer.", numProducersBeforeNotify); } // Notify producers about unregistered consumer. List <Task> tasks = State.Producers .Select(producerState => ExecuteProducerTask(producerState, producerState.Producer.RemoveSubscriber(subscriptionId, streamId))) .ToList(); await Task.WhenAll(tasks); //if producers got removed if (State.Producers.Count < numProducersBeforeNotify) { await this.WriteStateAsync(); } } }
internal void RemoveStream(InternalStreamId streamId) { remoteConsumers.Remove(streamId); }
public Task UnregisterProducer(InternalStreamId streamId, IStreamProducerExtension streamProducer) { var streamRendezvous = GetRendezvousGrain(streamId); return(streamRendezvous.UnregisterProducer(streamId, streamProducer)); }
internal StreamEventDeliveryFailureException(InternalStreamId streamId) : base(string.Format(ErrorStringFormat, streamId.GetNamespace(), streamId.StreamId)) { }
public Task UnregisterConsumer(GuidId subscriptionId, InternalStreamId streamId) { var streamRendezvous = GetRendezvousGrain(streamId); return(streamRendezvous.UnregisterConsumer(subscriptionId, streamId)); }
internal bool SameStreamId(InternalStreamId streamId) { return(IsValid && streamImpl.InternalStreamId.Equals(streamId)); }
public Task <List <StreamSubscription> > GetAllSubscriptions(InternalStreamId streamId, IStreamConsumerExtension streamConsumer = null) { var streamRendezvous = GetRendezvousGrain(streamId); return(streamRendezvous.GetAllSubscriptions(streamId, streamConsumer)); }
private async Task <double> TestOneStream(Guid streamId, string streamProviderName, int numProducers, int numConsumers, int numMessages, bool useFanOut = true) { output.WriteLine("Testing Stream {0} with Producers={1} Consumers={2} x {3} messages", streamId, numProducers, numConsumers, numMessages); Stopwatch sw = Stopwatch.StartNew(); List <IStreamLifecycleConsumerGrain> consumers = new List <IStreamLifecycleConsumerGrain>(); List <IStreamLifecycleProducerGrain> producers = new List <IStreamLifecycleProducerGrain>(); await InitializeTopology(streamId, this.StreamNamespace, streamProviderName, numProducers, numConsumers, producers, consumers, useFanOut); var promises = new List <Task>(); // Producers send M message each int item = 1; AsyncPipeline pipeline = new AsyncPipeline(MessagePipelineSize); foreach (var grain in producers) { for (int m = 0; m < numMessages; m++) { Task promise = grain.SendItem(item++); if (useFanOut) { pipeline.Add(promise); promises.Add(promise); } else { await promise; } } } if (useFanOut) { //output.WriteLine("Test: Waiting for {0} producers to finish sending {1} messages", producers.Count, promises.Count); await Task.WhenAll(promises); promises.Clear(); } var pubSub = StreamTestUtils.GetStreamPubSub(this.InternalClient); // Check Consumer counts var streamId1 = new InternalStreamId(streamProviderName, StreamId.Create(StreamNamespace, streamId)); int consumerCount = await pubSub.ConsumerCount(streamId1); Assert.Equal(numConsumers, consumerCount); // "ConsumerCount for Stream {0}", streamId // Check Producer counts int producerCount = await pubSub.ProducerCount(streamId1); Assert.Equal(numProducers, producerCount); // "ProducerCount for Stream {0}", streamId // Check message counts received by consumers int totalMessages = (numMessages + 1) * numProducers; foreach (var grain in consumers) { int count = await grain.GetReceivedCount(); Assert.Equal(totalMessages, count); // "ReceivedCount for Consumer grain {0}", grain.GetPrimaryKey()); } double rps = totalMessages / sw.Elapsed.TotalSeconds; //output.WriteLine("Sent {0} messages total from {1} Producers to {2} Consumers in {3} at {4} RPS", // totalMessages, numProducers, numConsumers, // sw.Elapsed, rps); return(rps); }
public GuidId CreateSubscriptionId(InternalStreamId streamId, IStreamConsumerExtension streamConsumer) { Guid subscriptionId = SubscriptionMarker.MarkAsExplicitSubscriptionId(Guid.NewGuid()); return(GuidId.GetGuidId(subscriptionId)); }
public Task <StreamHandshakeToken> DeliverImmutable(GuidId subscriptionId, InternalStreamId streamId, Immutable <object> item, StreamSequenceToken currentToken, StreamHandshakeToken handshakeToken) { return(DeliverMutable(subscriptionId, streamId, item.Value, currentToken, handshakeToken)); }