public async Task<ISet<PubSubSubscriptionState>> RegisterProducer(StreamId streamId, string streamProvider, IStreamProducerExtension streamProducer) { ISet<PubSubSubscriptionState> explicitRes = await explicitPubSub.RegisterProducer(streamId, streamProvider, streamProducer); ISet<PubSubSubscriptionState> implicitRes = await implicitPubSub.RegisterProducer(streamId, streamProvider, streamProducer); explicitRes.UnionWith(implicitRes); return explicitRes; }
public StreamConsumerData AddConsumer(GuidId subscriptionId, StreamId streamId, IStreamConsumerExtension streamConsumer, IStreamFilterPredicateWrapper filter) { var consumerData = new StreamConsumerData(subscriptionId, streamId, streamConsumer, filter); queueData.Add(subscriptionId, consumerData); lastActivityTime = DateTime.UtcNow; return consumerData; }
/// <summary> /// Retrieve a map of implicit subscriptionsIds to implicit subscribers, given a stream ID. This method throws an exception if there's no namespace associated with the stream ID. /// </summary> /// <param name="streamId">A stream ID.</param> /// <returns>A set of references to implicitly subscribed grains. They are expected to support the streaming consumer extension.</returns> /// <exception cref="System.ArgumentException">The stream ID doesn't have an associated namespace.</exception> /// <exception cref="System.InvalidOperationException">Internal invariant violation.</exception> internal IDictionary<Guid, IStreamConsumerExtension> GetImplicitSubscribers(StreamId streamId) { if (String.IsNullOrWhiteSpace(streamId.Namespace)) { throw new ArgumentException("The stream ID doesn't have an associated namespace.", "streamId"); } HashSet<int> entry; var result = new Dictionary<Guid, IStreamConsumerExtension>(); if (table.TryGetValue(streamId.Namespace, out entry)) { foreach (var i in entry) { IStreamConsumerExtension consumer = MakeConsumerReference(streamId.Guid, i); Guid subscriptionGuid = MakeSubscriptionGuid(i, streamId); if (result.ContainsKey(subscriptionGuid)) { throw new InvalidOperationException(string.Format("Internal invariant violation: generated duplicate subscriber reference: {0}, subscriptionId: {1}", consumer, subscriptionGuid)); } result.Add(subscriptionGuid, consumer); } return result; } return result; }
public StreamConsumerData(GuidId subscriptionId, StreamId streamId, IStreamConsumerExtension streamConsumer, IStreamFilterPredicateWrapper filter) { SubscriptionId = subscriptionId; StreamId = streamId; StreamConsumer = streamConsumer; Filter = filter; }
public Task<List<GuidId>> GetAllSubscriptions(StreamId streamId, IStreamConsumerExtension streamConsumer) { if (!IsImplicitSubscriber(streamConsumer, streamId)) { throw new ArgumentOutOfRangeException(streamId.ToString(), "Only implicit subscriptions are supported."); } return Task.FromResult(new List<GuidId> { GuidId.GetGuidId(streamConsumer.GetPrimaryKey()) }); }
public Task RegisterConsumer(GuidId subscriptionId, StreamId streamId, string streamProvider, IStreamConsumerExtension streamConsumer, IStreamFilterPredicateWrapper filter) { if (!IsImplicitSubscriber(streamConsumer, streamId)) { throw new ArgumentOutOfRangeException(streamId.ToString(), "Only implicit subscriptions are supported."); } return TaskDone.Done; }
public int sceMpegGetAtracAu(SceMpegPointer* Mpeg, StreamId StreamId, SceMpegAu* MpegAccessUnit, void* Atrac3PlusPointer) { CheckEnabledMpeg(); //Mpeg->SceMpegData. throw (new SceKernelException(SceKernelErrors.ERROR_MPEG_NO_DATA)); }
public int sceMpegGetAtracAu(SceMpegPointer* SceMpegPointer, StreamId StreamId, out SceMpegAu MpegAccessUnit, out PspPointer Atrac3PlusPointer) { var Mpeg = GetMpeg(SceMpegPointer); if (!Mpeg.HasData) throw (new SceKernelException(SceKernelErrors.ERROR_MPEG_NO_DATA)); MpegAccessUnit = Mpeg.GetAtracAu(StreamId); Atrac3PlusPointer.Address = 0; return 0; }
internal void AddStream(StreamId streamId) { StreamConsumerExtensionCollection obs; // no need to lock on _remoteConsumers, since on the client we have one extension per stream (per StreamProducer) // so this call is only made once, when StreamProducer is created. if (remoteConsumers.TryGetValue(streamId, out obs)) return; obs = new StreamConsumerExtensionCollection(); remoteConsumers.Add(streamId, obs); }
public GuidId CreateSubscriptionId(StreamId streamId, IStreamConsumerExtension streamConsumer) { GrainId grainId = GrainExtensions.GetGrainId(streamConsumer); Guid subscriptionGuid; if (!implicitTable.TryGetImplicitSubscriptionGuid(grainId, streamId, out subscriptionGuid)) { throw new ArgumentOutOfRangeException(streamId.ToString(), "Only implicit subscriptions are supported."); } return GuidId.GetGuidId(subscriptionGuid); }
public Task<ISet<PubSubSubscriptionState>> RegisterProducer(StreamId streamId, string streamProvider, IStreamProducerExtension streamProducer) { ISet<PubSubSubscriptionState> result = new HashSet<PubSubSubscriptionState>(); if (String.IsNullOrWhiteSpace(streamId.Namespace)) return Task.FromResult(result); IDictionary<Guid, IStreamConsumerExtension> implicitSubscriptions = implicitTable.GetImplicitSubscribers(streamId); foreach (var kvp in implicitSubscriptions) { GuidId subscriptionId = GuidId.GetGuidId(kvp.Key); result.Add(new PubSubSubscriptionState(subscriptionId, streamId, kvp.Value)); } return Task.FromResult(result); }
public int sceMpegGetAvcAu(SceMpegPointer* Mpeg, StreamId StreamId, SceMpegAu* MpegAccessUnit, int* DataAttributes) { CheckEnabledMpeg(); if (DataAttributes != null) { *DataAttributes = 1; } throw(new SceKernelException(SceKernelErrors.ERROR_MPEG_NO_DATA)); //throw(new NotImplementedException()); //return 0; }
//public SceMpegStream* sceMpegRegistStream(SceMpeg* Mpeg, int iStreamID, int iUnk) public int sceMpegRegistStream(SceMpegPointer* Mpeg, StreamId StreamId, int StreamIndex) { CheckEnabledMpeg(); var StreamInfoId = RegisteredStreams.Create(new StreamInfo() { StreamId = StreamId, StreamIndex = StreamIndex, }); //Console.WriteLine(iStreamID); //return 0; //var SceMpegData = GetSceMpegData(Mpeg); //throw(new NotImplementedException()); return StreamInfoId; }
async Task<ISet<PubSubSubscriptionState>> IStreamPubSub.RegisterProducer(StreamId streamId, string streamProvider, IStreamProducerExtension streamProducer) { var matches = new PubSubSubscriptionState[0]; if (ShouldMatch(streamProvider)) { matches = (from StreamPubSubMatch m in matcher(new StreamIdentity(streamId)) let subId = GuidId.GetNewGuidId() select new PubSubSubscriptionState(subId, streamId, new PushExtension(m), null)) .ToArray(); } var registered = await registry.RegisterProducer(streamId, streamProvider, streamProducer); registered.UnionWith(matches); return registered; }
internal Task DeliverItem(StreamId streamId, object item) { StreamConsumerExtensionCollection consumers; if (remoteConsumers.TryGetValue(streamId, out consumers)) { // Note: This is the main hot code path, // and the caller immediately does await on the Task // returned from this method, so we can just direct return here // without incurring overhead of additional await. return consumers.DeliverItem(streamId, item, fireAndForgetDelivery); } else { // We got an item when we don't think we're the subscriber. This is a normal race condition. // We can drop the item on the floor, or pass it to the rendezvous, or log a warning. } return TaskDone.Done; }
internal void AddSubscribers(StreamId streamId, ICollection<PubSubSubscriptionState> newSubscribers) { if (logger.IsVerbose) logger.Verbose("{0} AddSubscribers {1} for stream {2}", providerRuntime.ExecutingEntityIdentity(), Utils.EnumerableToString(newSubscribers), streamId); StreamConsumerExtensionCollection consumers; if (remoteConsumers.TryGetValue(streamId, out consumers)) { foreach (var newSubscriber in newSubscribers) { consumers.AddRemoteSubscriber(newSubscriber.SubscriptionId, newSubscriber.Consumer, newSubscriber.Filter); } } else { // We got an item when we don't think we're the subscriber. This is a normal race condition. // We can drop the item on the floor, or pass it to the rendezvous, or log a warning. } }
public static EventStreamId FromStreamId(StreamId id) => new(
GuidId IStreamPubSub.CreateSubscriptionId(StreamId streamId, IStreamConsumerExtension streamConsumer) { return(registry.CreateSubscriptionId(streamId, streamConsumer)); }
public Task<List<GuidId>> GetAllSubscriptions(StreamId streamId, IStreamConsumerExtension streamConsumer) { return Task.FromResult(new List<GuidId> { CreateSubscriptionId(streamId, streamConsumer) }); }
public IQueueCacheCursor GetCacheCursor(StreamId streamId, StreamSequenceToken token) { return(new Cursor(this.cache, streamId, token)); }
Task IStreamPubSub.RegisterConsumer(GuidId subscriptionId, StreamId streamId, string streamProvider, IStreamConsumerExtension streamConsumer, IStreamFilterPredicateWrapper filter) { return(registry.RegisterConsumer(subscriptionId, streamId, streamProvider, streamConsumer, filter)); }
internal static void WritingEventToPublisStream(this ILogger logger, EventLogSequenceNumber eventLogSequenceNumber, StreamId stream) => _writingEventToPublisStream(logger, eventLogSequenceNumber, stream, null);
private Task Test_Stream_Churn_NumStreams_FewPublishers( string streamProviderName, int pipelineSize, int numStreams, int numConsumers = 9, int numProducers = 4, bool warmUpPubSub = true, bool warmUpProducers = false, bool normalSubscribeCalls = true) { output.WriteLine("Testing churn with {0} Streams on {1} Producers with {2} Consumers per Stream", numStreams, numProducers, numConsumers); AsyncPipeline pipeline = new AsyncPipeline(pipelineSize); // Create streamId Guids StreamId[] streamIds = new StreamId[numStreams]; for (int i = 0; i < numStreams; i++) { streamIds[i] = StreamId.Create(this.StreamNamespace, Guid.NewGuid()); } int activeConsumerGrains = ActiveGrainCount(typeof(StreamLifecycleConsumerGrain)); Assert.Equal(0, activeConsumerGrains); // "Initial Consumer count should be zero" int activeProducerGrains = ActiveGrainCount(typeof(StreamLifecycleProducerGrain)); Assert.Equal(0, activeProducerGrains); // "Initial Producer count should be zero" if (warmUpPubSub) { WarmUpPubSub(streamProviderName, streamIds, pipeline); pipeline.Wait(); int activePubSubGrains = ActiveGrainCount(typeof(PubSubRendezvousGrain)); Assert.Equal(streamIds.Length, activePubSubGrains); // "Initial PubSub count -- should all be warmed up" } Guid[] producerIds = new Guid[numProducers]; if (numProducers > 0 && warmUpProducers) { // Warm up Producers to pre-create grains for (int i = 0; i < numProducers; i++) { producerIds[i] = Guid.NewGuid(); var grain = this.GrainFactory.GetGrain <IStreamLifecycleProducerGrain>(producerIds[i]); Task promise = grain.Ping(); pipeline.Add(promise); } pipeline.Wait(); int activePublisherGrains = this.ActiveGrainCount(typeof(StreamLifecycleProducerGrain)); Assert.Equal(numProducers, activePublisherGrains); // "Initial Publisher count -- should all be warmed up" } var promises = new List <Task>(); Stopwatch sw = Stopwatch.StartNew(); if (numProducers > 0) { // Producers for (int i = 0; i < numStreams; i++) { StreamId streamId = streamIds[i]; Guid producerId = producerIds[i % numProducers]; var grain = this.GrainFactory.GetGrain <IStreamLifecycleProducerGrain>(producerId); Task promise = grain.BecomeProducer(streamId, streamProviderName); promises.Add(promise); pipeline.Add(promise); } pipeline.Wait(); promises.Clear(); } // Consumers for (int i = 0; i < numStreams; i++) { StreamId streamId = streamIds[i]; Task promise = SetupOneStream(streamId, streamProviderName, pipeline, numConsumers, 0, normalSubscribeCalls); promises.Add(promise); } pipeline.Wait(); Task.WhenAll(promises).Wait(); sw.Stop(); int consumerCount = ActiveGrainCount(typeof(StreamLifecycleConsumerGrain)); Assert.Equal(activeConsumerGrains + (numStreams * numConsumers), consumerCount); // "The right number of Consumer grains are active" int producerCount = ActiveGrainCount(typeof(StreamLifecycleProducerGrain)); Assert.Equal(activeProducerGrains + (numStreams * numProducers), producerCount); // "The right number of Producer grains are active" int pubSubCount = ActiveGrainCount(typeof(PubSubRendezvousGrain)); Assert.Equal(streamIds.Length, pubSubCount); // "Final PubSub count -- no more started" TimeSpan elapsed = sw.Elapsed; int totalSubscriptions = numStreams * numConsumers; double rps = totalSubscriptions / elapsed.TotalSeconds; output.WriteLine("Subscriptions-per-second = {0} during period {1}", rps, elapsed); Assert.NotEqual(0.0, rps); // "RPS greater than zero" return(Task.CompletedTask); }
public async Task RemoveSubscription(string streamProviderName, StreamId streamId, Guid subscriptionId) { var internalStreamId = new InternalStreamId(streamProviderName, streamId); await streamPubSub.UnregisterConsumer(GuidId.GetGuidId(subscriptionId), internalStreamId); }
public Task <IEnumerable <StreamSubscription> > GetSubscriptions(string streamProviderName, StreamId streamId) { var internalStreamId = new InternalStreamId(streamProviderName, streamId); return(streamPubSub.GetAllSubscriptions(internalStreamId).ContinueWith(subs => subs.Result.AsEnumerable())); }
public async Task <StreamSubscription> AddSubscription(string streamProviderName, StreamId streamId, GrainReference grainRef) { var consumer = grainRef.AsReference <IStreamConsumerExtension>(); var internalStreamId = new InternalStreamId(streamProviderName, streamId); var subscriptionId = streamPubSub.CreateSubscriptionId(internalStreamId, consumer); await streamPubSub.RegisterConsumer(subscriptionId, internalStreamId, consumer, null); var newSub = new StreamSubscription(subscriptionId.Guid, streamProviderName, streamId, grainRef.GrainId); return(newSub); }
/// <summary> /// Initializes a new instance of the <see cref="StreamDefinitionDoesNotExist" /> class. /// </summary> /// <param name="stream">The stream id.</param> /// <param name="scope">The scope id.</param> public StreamDefinitionDoesNotExist(StreamId stream, ScopeId scope) : base($"Stream definition for stream {stream.Value} in scope {scope.Value} does not exist") { }
Task IStreamPubSub.UnregisterConsumer(GuidId subscriptionId, StreamId streamId, string streamProvider) { return(registry.UnregisterConsumer(subscriptionId, streamId, streamProvider)); }
public static string ReadStreamBackwards( StreamId streamId, int fromVersionInclusive, int maxCount, bool prefetchJsonData) => ReadStream(streamId, fromVersionInclusive, maxCount, prefetchJsonData, Constants.Direction.Backwards);
private static ReadStreamPage ReadStreamForwardsInternal( IHalClient client, StreamId streamId, int fromVersionInclusive, bool prefetchJsonData) { var resource = client.Current.First(); if (client.StatusCode == HttpStatusCode.NotFound) { return(new ReadStreamPage( streamId, PageReadStatus.StreamNotFound, fromVersionInclusive, -1, -1, -1, ReadDirection.Forward, true)); } var pageInfo = resource.Data <HalReadPage>(); var streamMessages = Convert( resource.Embedded .Where(r => r.Rel == Constants.Relations.Message) .Reverse() .ToArray(), client, prefetchJsonData); var readStreamPage = new ReadStreamPage( streamId, PageReadStatus.Success, pageInfo.FromStreamVersion, pageInfo.NextStreamVersion, pageInfo.LastStreamVersion, pageInfo.LastStreamPosition, ReadDirection.Forward, pageInfo.IsEnd, ReadNextStreamPage, streamMessages); return(readStreamPage); async Task <ReadStreamPage> ReadNextStreamPage(int nextVersion, CancellationToken ct) => resource.Links.Any(link => link.Rel == Constants.Relations.Next) ? ReadStreamForwardsInternal( await client.GetAsync(resource, Constants.Relations.Next), streamId, pageInfo.LastStreamVersion, prefetchJsonData) : new ReadStreamPage( streamId, PageReadStatus.Success, pageInfo.LastStreamVersion, nextVersion, pageInfo.LastStreamVersion, pageInfo.LastStreamPosition, ReadDirection.Forward, true, ReadNextStreamPage); }
internal bool IsImplicitSubscriber(IAddressable addressable, StreamId streamId) { return implicitTable.IsImplicitSubscriber(GrainExtensions.GetGrainId(addressable), streamId); }
Task <bool> IStreamPubSub.FaultSubscription(StreamId streamId, GuidId subscriptionId) { return(registry.FaultSubscription(streamId, subscriptionId)); }
public static string StreamByMessageId(StreamId streamId, Guid messageId) => $"{Stream(streamId)}/{messageId}";
public static string Stream(StreamId streamId) => $"/streams/{streamId}";
public IEnumerable<StreamConsumerData> AllConsumersForStream(StreamId streamId) { return queueData.Values.Where(consumer => consumer.StreamId.Equals(streamId)); }
Task IStreamPubSub.UnregisterProducer(StreamId streamId, string streamProvider, IStreamProducerExtension streamProducer) { return(registry.UnregisterProducer(streamId, streamProvider, streamProducer)); }
// Called by rendezvous when new remote subscriber subscribes to this stream. private async Task AddSubscriber_Impl( GuidId subscriptionId, StreamId streamId, IStreamConsumerExtension streamConsumer, StreamSequenceToken cacheToken, IStreamFilterPredicateWrapper filter) { if (IsShutdown) return; StreamConsumerCollection streamDataCollection; if (!pubSubCache.TryGetValue(streamId, out streamDataCollection)) { streamDataCollection = new StreamConsumerCollection(DateTime.UtcNow); pubSubCache.Add(streamId, streamDataCollection); } StreamConsumerData data; if (!streamDataCollection.TryGetConsumer(subscriptionId, out data)) data = streamDataCollection.AddConsumer(subscriptionId, streamId, streamConsumer, filter ?? DefaultStreamFilter); if (await DoHandshakeWithConsumer(data, cacheToken)) { if (data.State == StreamConsumerDataState.Inactive) RunConsumerCursor(data, data.Filter).Ignore(); // Start delivering events if not actively doing so } }
public void RemoveSubscriber_Impl(GuidId subscriptionId, StreamId streamId) { if (IsShutdown) return; StreamConsumerCollection streamData; if (!pubSubCache.TryGetValue(streamId, out streamData)) return; // remove consumer bool removed = streamData.RemoveConsumer(subscriptionId, logger); if (removed && logger.IsVerbose) logger.Verbose((int)ErrorCode.PersistentStreamPullingAgent_10, "Removed Consumer: subscription={0}, for stream {1}.", subscriptionId, streamId); if (streamData.Count == 0) pubSubCache.Remove(streamId); }
/// <summary> /// For test purpose. ConfigureDataGeneratorForStream will configure a data generator for the stream /// </summary> /// <param name="streamId"></param> internal void ConfigureDataGeneratorForStream(StreamId streamId) { (this.receiver as EventHubPartitionGeneratorReceiver)?.ConfigureDataGeneratorForStream(streamId); }
private async Task RegisterAsStreamProducer(StreamId streamId, StreamSequenceToken streamStartToken) { try { if (pubSub == null) throw new NullReferenceException("Found pubSub reference not set up correctly in RetreaveNewStream"); IStreamProducerExtension meAsStreamProducer = this.AsReference<IStreamProducerExtension>(); ISet<PubSubSubscriptionState> streamData = await pubSub.RegisterProducer(streamId, streamProviderName, meAsStreamProducer); if (logger.IsVerbose) logger.Verbose((int)ErrorCode.PersistentStreamPullingAgent_16, "Got back {0} Subscribers for stream {1}.", streamData.Count, streamId); var addSubscriptionTasks = new List<Task>(streamData.Count); foreach (PubSubSubscriptionState item in streamData) { addSubscriptionTasks.Add(AddSubscriber_Impl(item.SubscriptionId, item.Stream, item.Consumer, streamStartToken, item.Filter)); } await Task.WhenAll(addSubscriptionTasks); } catch (Exception exc) { // RegisterAsStreamProducer is fired with .Ignore so we should log if anything goes wrong, because there is no one to catch the exception logger.Error((int)ErrorCode.PersistentStreamPullingAgent_17, "Ignored RegisterAsStreamProducer Error", exc); throw; } }
private async Task <double> TestOneStream(Guid streamId, string streamProviderName, int numProducers, int numConsumers, int numMessages, bool useFanOut = true) { output.WriteLine("Testing Stream {0} with Producers={1} Consumers={2} x {3} messages", streamId, numProducers, numConsumers, numMessages); Stopwatch sw = Stopwatch.StartNew(); List <IStreamLifecycleConsumerGrain> consumers = new List <IStreamLifecycleConsumerGrain>(); List <IStreamLifecycleProducerGrain> producers = new List <IStreamLifecycleProducerGrain>(); await InitializeTopology(streamId, this.StreamNamespace, streamProviderName, numProducers, numConsumers, producers, consumers, useFanOut); var promises = new List <Task>(); // Producers send M message each int item = 1; AsyncPipeline pipeline = new AsyncPipeline(MessagePipelineSize); foreach (var grain in producers) { for (int m = 0; m < numMessages; m++) { Task promise = grain.SendItem(item++); if (useFanOut) { pipeline.Add(promise); promises.Add(promise); } else { await promise; } } } if (useFanOut) { //output.WriteLine("Test: Waiting for {0} producers to finish sending {1} messages", producers.Count, promises.Count); await Task.WhenAll(promises); promises.Clear(); } var pubSub = StreamTestUtils.GetStreamPubSub(this.InternalClient); // Check Consumer counts var streamId1 = new InternalStreamId(streamProviderName, StreamId.Create(StreamNamespace, streamId)); int consumerCount = await pubSub.ConsumerCount(streamId1); Assert.Equal(numConsumers, consumerCount); // "ConsumerCount for Stream {0}", streamId // Check Producer counts int producerCount = await pubSub.ProducerCount(streamId1); Assert.Equal(numProducers, producerCount); // "ProducerCount for Stream {0}", streamId // Check message counts received by consumers int totalMessages = (numMessages + 1) * numProducers; foreach (var grain in consumers) { int count = await grain.GetReceivedCount(); Assert.Equal(totalMessages, count); // "ReceivedCount for Consumer grain {0}", grain.GetPrimaryKey()); } double rps = totalMessages / sw.Elapsed.TotalSeconds; //output.WriteLine("Sent {0} messages total from {1} Producers to {2} Consumers in {3} at {4} RPS", // totalMessages, numProducers, numConsumers, // sw.Elapsed, rps); return(rps); }
internal bool IsImplicitSubscriber(GuidId subscriptionId, StreamId streamId) { return SubscriptionMarker.IsImplicitSubscription(subscriptionId.Guid); }
public Task AddSubscriber(GuidId subscriptionId, StreamId streamId, IStreamConsumerExtension streamConsumer, IStreamFilterPredicateWrapper filter) { return(TaskDone.Done); }
Task <List <StreamSubscription> > IStreamPubSub.GetAllSubscriptions(StreamId streamId, IStreamConsumerExtension streamConsumer) { return(registry.GetAllSubscriptions(streamId, streamConsumer)); }
public Task AddSubscriber( GuidId subscriptionId, StreamId streamId, IStreamConsumerExtension streamConsumer, IStreamFilterPredicateWrapper filter) { if (logger.IsVerbose) logger.Verbose((int)ErrorCode.PersistentStreamPullingAgent_09, "AddSubscriber: Stream={0} Subscriber={1}.", streamId, streamConsumer); // cannot await here because explicit consumers trigger this call, so it could cause a deadlock. AddSubscriber_Impl(subscriptionId, streamId, streamConsumer, null, filter) .LogException(logger, ErrorCode.PersistentStreamPullingAgent_26, String.Format("Failed to add subscription for stream {0}." , streamId)) .Ignore(); return TaskDone.Done; }
public Task RemoveSubscriber(GuidId subscriptionId, StreamId streamId) { return(TaskDone.Done); }
public Task RemoveSubscriber(GuidId subscriptionId, StreamId streamId) { RemoveSubscriber_Impl(subscriptionId, streamId); return TaskDone.Done; }
private async Task SendAndReceiveFromQueueAdapter(IQueueAdapterFactory adapterFactory) { IQueueAdapter adapter = await adapterFactory.CreateAdapter(); IQueueAdapterCache cache = adapterFactory.GetQueueAdapterCache(); // Create receiver per queue IStreamQueueMapper mapper = adapterFactory.GetStreamQueueMapper(); Dictionary <QueueId, IQueueAdapterReceiver> receivers = mapper.GetAllQueues().ToDictionary(queueId => queueId, adapter.CreateReceiver); Dictionary <QueueId, IQueueCache> caches = mapper.GetAllQueues().ToDictionary(queueId => queueId, cache.CreateQueueCache); await Task.WhenAll(receivers.Values.Select(receiver => receiver.Initialize(TimeSpan.FromSeconds(5)))); // test using 2 streams Guid streamId1 = Guid.NewGuid(); Guid streamId2 = Guid.NewGuid(); int receivedBatches = 0; var streamsPerQueue = new ConcurrentDictionary <QueueId, HashSet <StreamId> >(); // reader threads (at most 2 active queues because only two streams) var work = new List <Task>(); foreach (KeyValuePair <QueueId, IQueueAdapterReceiver> receiverKvp in receivers) { QueueId queueId = receiverKvp.Key; var receiver = receiverKvp.Value; var qCache = caches[queueId]; Task task = Task.Factory.StartNew(() => { while (receivedBatches < NumBatches) { var messages = receiver.GetQueueMessagesAsync(SQSStorage.MAX_NUMBER_OF_MESSAGE_TO_PEAK).Result.ToArray(); if (!messages.Any()) { continue; } foreach (var message in messages.Cast <SQSBatchContainer>()) { streamsPerQueue.AddOrUpdate(queueId, id => new HashSet <StreamId> { message.StreamId }, (id, set) => { set.Add(message.StreamId); return(set); }); output.WriteLine("Queue {0} received message on stream {1}", queueId, message.StreamId); Assert.Equal(NumMessagesPerBatch / 2, message.GetEvents <int>().Count()); // "Half the events were ints" Assert.Equal(NumMessagesPerBatch / 2, message.GetEvents <string>().Count()); // "Half the events were strings" } Interlocked.Add(ref receivedBatches, messages.Length); qCache.AddToCache(messages); } }); work.Add(task); } // send events List <object> events = CreateEvents(NumMessagesPerBatch); work.Add(Task.Factory.StartNew(() => Enumerable.Range(0, NumBatches) .Select(i => i % 2 == 0 ? streamId1 : streamId2) .ToList() .ForEach(streamId => adapter.QueueMessageBatchAsync(StreamId.Create(streamId.ToString(), streamId), events.Take(NumMessagesPerBatch).ToArray(), null, RequestContextExtensions.Export(this.fixture.SerializationManager)).Wait()))); await Task.WhenAll(work); // Make sure we got back everything we sent Assert.Equal(NumBatches, receivedBatches); // check to see if all the events are in the cache and we can enumerate through them StreamSequenceToken firstInCache = new EventSequenceTokenV2(0); foreach (KeyValuePair <QueueId, HashSet <StreamId> > kvp in streamsPerQueue) { var receiver = receivers[kvp.Key]; var qCache = caches[kvp.Key]; foreach (StreamId streamGuid in kvp.Value) { // read all messages in cache for stream IQueueCacheCursor cursor = qCache.GetCacheCursor(streamGuid, firstInCache); int messageCount = 0; StreamSequenceToken tenthInCache = null; StreamSequenceToken lastToken = firstInCache; while (cursor.MoveNext()) { Exception ex; messageCount++; IBatchContainer batch = cursor.GetCurrent(out ex); output.WriteLine("Token: {0}", batch.SequenceToken); Assert.True(batch.SequenceToken.CompareTo(lastToken) >= 0, $"order check for event {messageCount}"); lastToken = batch.SequenceToken; if (messageCount == 10) { tenthInCache = batch.SequenceToken; } } output.WriteLine("On Queue {0} we received a total of {1} message on stream {2}", kvp.Key, messageCount, streamGuid); Assert.Equal(NumBatches / 2, messageCount); Assert.NotNull(tenthInCache); // read all messages from the 10th cursor = qCache.GetCacheCursor(streamGuid, tenthInCache); messageCount = 0; while (cursor.MoveNext()) { messageCount++; } output.WriteLine("On Queue {0} we received a total of {1} message on stream {2}", kvp.Key, messageCount, streamGuid); const int expected = NumBatches / 2 - 10 + 1; // all except the first 10, including the 10th (10 + 1) Assert.Equal(expected, messageCount); } } }
private async Task RegisterStream(StreamId streamId, StreamSequenceToken firstToken, DateTime now) { var streamData = new StreamConsumerCollection(now); pubSubCache.Add(streamId, streamData); // Create a fake cursor to point into a cache. // That way we will not purge the event from the cache, until we talk to pub sub. // This will help ensure the "casual consistency" between pre-existing subscripton (of a potentially new already subscribed consumer) // and later production. var pinCursor = queueCache.GetCacheCursor(streamId.Guid, streamId.Namespace, firstToken); try { await RegisterAsStreamProducer(streamId, firstToken); }finally { // Cleanup the fake pinning cursor. pinCursor.Dispose(); } }
/// <inheritdoc/> public async Task <Try <IFilterDefinition> > TryGetFromStream(ScopeId scopeId, StreamId streamId, CancellationToken cancellationToken) { var tryGetStream = await _streamDefinitions.TryGet(scopeId, streamId, cancellationToken).ConfigureAwait(false); return(tryGetStream.Success, tryGetStream.Result?.FilterDefinition); }
internal FaultedSubscriptionException(GuidId subscriptionId, StreamId streamId) : base(string.Format(ErrorStringFormat, subscriptionId.Guid, streamId)) { }
/// <summary> /// 接收消息 /// </summary> /// <param name="content">消息内容</param> /// <param name="token">StreamSequenceToken</param> protected override Task OnReceive(IsOperationPoint content, StreamSequenceToken token) { UserMessage.Send(StreamId.ToString(), Name, Utilities.JsonSerialize(content)); return(Task.CompletedTask); }
public Task<bool> FaultSubscription(StreamId streamId, GuidId subscriptionId) { return Task.FromResult(false); }
private async Task DeliverToRemote(IStreamConsumerExtension remoteConsumer, StreamId streamId, GuidId subscriptionId, object item, bool optimizeForImmutableData) { try { if (optimizeForImmutableData) { await remoteConsumer.DeliverImmutable(subscriptionId, streamId, new Immutable <object>(item), null, null); } else { await remoteConsumer.DeliverMutable(subscriptionId, streamId, item, null, null); } } catch (ClientNotAvailableException) { Tuple <IStreamConsumerExtension, IStreamFilterPredicateWrapper> discard; if (consumers.TryRemove(subscriptionId, out discard)) { streamPubSub.UnregisterConsumer(subscriptionId, streamId, streamId.ProviderName).Ignore(); logger.Warn(ErrorCode.Stream_ConsumerIsDead, "Consumer {0} on stream {1} is no longer active - permanently removing Consumer.", remoteConsumer, streamId); } } }
public Task UnregisterProducer(StreamId streamId, string streamProvider, IStreamProducerExtension streamProducer) { return TaskDone.Done; }
/// <inheritdoc/> public Task Write(CommittedEvent @event, ScopeId scope, StreamId streamId, PartitionId partitionId, CancellationToken cancellationToken) => Write(@event, streamId, partitionId, cancellationToken);
public Task UnregisterConsumer(GuidId subscriptionId, StreamId streamId, string streamProvider) { return TaskDone.Done; }
internal void StopProducingOnStream(StreamId streamId) { (this.receiver as EventHubPartitionGeneratorReceiver)?.StopProducingOnStream(streamId); }
internal void RemoveStream(StreamId streamId) { remoteConsumers.Remove(streamId); }
public Cursor(IEventHubQueueCache cache, StreamId streamId, StreamSequenceToken token) { this.cache = cache; this.cursor = cache.GetCursor(streamId, token); }