public async Task<ISet<PubSubSubscriptionState>> RegisterProducer(StreamId streamId, string streamProvider, IStreamProducerExtension streamProducer)
{
ISet<PubSubSubscriptionState> explicitRes = await explicitPubSub.RegisterProducer(streamId, streamProvider, streamProducer);
ISet<PubSubSubscriptionState> implicitRes = await implicitPubSub.RegisterProducer(streamId, streamProvider, streamProducer);
explicitRes.UnionWith(implicitRes);
return explicitRes;
}
public StreamConsumerData AddConsumer(GuidId subscriptionId, StreamId streamId, IStreamConsumerExtension streamConsumer, IStreamFilterPredicateWrapper filter)
{
var consumerData = new StreamConsumerData(subscriptionId, streamId, streamConsumer, filter);
queueData.Add(subscriptionId, consumerData);
lastActivityTime = DateTime.UtcNow;
return consumerData;
}
/// <summary>
/// Retrieve a map of implicit subscriptionsIds to implicit subscribers, given a stream ID. This method throws an exception if there's no namespace associated with the stream ID.
/// </summary>
/// <param name="streamId">A stream ID.</param>
/// <returns>A set of references to implicitly subscribed grains. They are expected to support the streaming consumer extension.</returns>
/// <exception cref="System.ArgumentException">The stream ID doesn't have an associated namespace.</exception>
/// <exception cref="System.InvalidOperationException">Internal invariant violation.</exception>
internal IDictionary<Guid, IStreamConsumerExtension> GetImplicitSubscribers(StreamId streamId)
{
if (String.IsNullOrWhiteSpace(streamId.Namespace))
{
throw new ArgumentException("The stream ID doesn't have an associated namespace.", "streamId");
}
HashSet<int> entry;
var result = new Dictionary<Guid, IStreamConsumerExtension>();
if (table.TryGetValue(streamId.Namespace, out entry))
{
foreach (var i in entry)
{
IStreamConsumerExtension consumer = MakeConsumerReference(streamId.Guid, i);
Guid subscriptionGuid = MakeSubscriptionGuid(i, streamId);
if (result.ContainsKey(subscriptionGuid))
{
throw new InvalidOperationException(string.Format("Internal invariant violation: generated duplicate subscriber reference: {0}, subscriptionId: {1}", consumer, subscriptionGuid));
}
result.Add(subscriptionGuid, consumer);
}
return result;
}
return result;
}
public StreamConsumerData(GuidId subscriptionId, StreamId streamId, IStreamConsumerExtension streamConsumer, IStreamFilterPredicateWrapper filter)
{
SubscriptionId = subscriptionId;
StreamId = streamId;
StreamConsumer = streamConsumer;
Filter = filter;
}
public Task<List<GuidId>> GetAllSubscriptions(StreamId streamId, IStreamConsumerExtension streamConsumer)
{
if (!IsImplicitSubscriber(streamConsumer, streamId))
{
throw new ArgumentOutOfRangeException(streamId.ToString(), "Only implicit subscriptions are supported.");
}
return Task.FromResult(new List<GuidId> { GuidId.GetGuidId(streamConsumer.GetPrimaryKey()) });
}
public Task RegisterConsumer(GuidId subscriptionId, StreamId streamId, string streamProvider, IStreamConsumerExtension streamConsumer, IStreamFilterPredicateWrapper filter)
{
if (!IsImplicitSubscriber(streamConsumer, streamId))
{
throw new ArgumentOutOfRangeException(streamId.ToString(), "Only implicit subscriptions are supported.");
}
return TaskDone.Done;
}
public int sceMpegGetAtracAu(SceMpegPointer* Mpeg, StreamId StreamId, SceMpegAu* MpegAccessUnit, void* Atrac3PlusPointer)
{
CheckEnabledMpeg();
//Mpeg->SceMpegData.
throw (new SceKernelException(SceKernelErrors.ERROR_MPEG_NO_DATA));
}
public int sceMpegGetAtracAu(SceMpegPointer* SceMpegPointer, StreamId StreamId, out SceMpegAu MpegAccessUnit, out PspPointer Atrac3PlusPointer)
{
var Mpeg = GetMpeg(SceMpegPointer);
if (!Mpeg.HasData) throw (new SceKernelException(SceKernelErrors.ERROR_MPEG_NO_DATA));
MpegAccessUnit = Mpeg.GetAtracAu(StreamId);
Atrac3PlusPointer.Address = 0;
return 0;
}
internal void AddStream(StreamId streamId)
{
StreamConsumerExtensionCollection obs;
// no need to lock on _remoteConsumers, since on the client we have one extension per stream (per StreamProducer)
// so this call is only made once, when StreamProducer is created.
if (remoteConsumers.TryGetValue(streamId, out obs)) return;
obs = new StreamConsumerExtensionCollection();
remoteConsumers.Add(streamId, obs);
}
public GuidId CreateSubscriptionId(StreamId streamId, IStreamConsumerExtension streamConsumer)
{
GrainId grainId = GrainExtensions.GetGrainId(streamConsumer);
Guid subscriptionGuid;
if (!implicitTable.TryGetImplicitSubscriptionGuid(grainId, streamId, out subscriptionGuid))
{
throw new ArgumentOutOfRangeException(streamId.ToString(), "Only implicit subscriptions are supported.");
}
return GuidId.GetGuidId(subscriptionGuid);
}
public Task<ISet<PubSubSubscriptionState>> RegisterProducer(StreamId streamId, string streamProvider, IStreamProducerExtension streamProducer)
{
ISet<PubSubSubscriptionState> result = new HashSet<PubSubSubscriptionState>();
if (String.IsNullOrWhiteSpace(streamId.Namespace)) return Task.FromResult(result);
IDictionary<Guid, IStreamConsumerExtension> implicitSubscriptions = implicitTable.GetImplicitSubscribers(streamId);
foreach (var kvp in implicitSubscriptions)
{
GuidId subscriptionId = GuidId.GetGuidId(kvp.Key);
result.Add(new PubSubSubscriptionState(subscriptionId, streamId, kvp.Value));
}
return Task.FromResult(result);
}
public int sceMpegGetAvcAu(SceMpegPointer* Mpeg, StreamId StreamId, SceMpegAu* MpegAccessUnit, int* DataAttributes)
{
CheckEnabledMpeg();
if (DataAttributes != null)
{
*DataAttributes = 1;
}
throw(new SceKernelException(SceKernelErrors.ERROR_MPEG_NO_DATA));
//throw(new NotImplementedException());
//return 0;
}
//public SceMpegStream* sceMpegRegistStream(SceMpeg* Mpeg, int iStreamID, int iUnk)
public int sceMpegRegistStream(SceMpegPointer* Mpeg, StreamId StreamId, int StreamIndex)
{
CheckEnabledMpeg();
var StreamInfoId = RegisteredStreams.Create(new StreamInfo()
{
StreamId = StreamId,
StreamIndex = StreamIndex,
});
//Console.WriteLine(iStreamID);
//return 0;
//var SceMpegData = GetSceMpegData(Mpeg);
//throw(new NotImplementedException());
return StreamInfoId;
}
async Task<ISet<PubSubSubscriptionState>> IStreamPubSub.RegisterProducer(StreamId streamId, string streamProvider, IStreamProducerExtension streamProducer)
{
var matches = new PubSubSubscriptionState[0];
if (ShouldMatch(streamProvider))
{
matches = (from StreamPubSubMatch m in matcher(new StreamIdentity(streamId))
let subId = GuidId.GetNewGuidId()
select new PubSubSubscriptionState(subId, streamId, new PushExtension(m), null))
.ToArray();
}
var registered = await registry.RegisterProducer(streamId, streamProvider, streamProducer);
registered.UnionWith(matches);
return registered;
}
internal Task DeliverItem(StreamId streamId, object item)
{
StreamConsumerExtensionCollection consumers;
if (remoteConsumers.TryGetValue(streamId, out consumers))
{
// Note: This is the main hot code path,
// and the caller immediately does await on the Task
// returned from this method, so we can just direct return here
// without incurring overhead of additional await.
return consumers.DeliverItem(streamId, item, fireAndForgetDelivery);
}
else
{
// We got an item when we don't think we're the subscriber. This is a normal race condition.
// We can drop the item on the floor, or pass it to the rendezvous, or log a warning.
}
return TaskDone.Done;
}
internal void AddSubscribers(StreamId streamId, ICollection<PubSubSubscriptionState> newSubscribers)
{
if (logger.IsVerbose) logger.Verbose("{0} AddSubscribers {1} for stream {2}", providerRuntime.ExecutingEntityIdentity(), Utils.EnumerableToString(newSubscribers), streamId);
StreamConsumerExtensionCollection consumers;
if (remoteConsumers.TryGetValue(streamId, out consumers))
{
foreach (var newSubscriber in newSubscribers)
{
consumers.AddRemoteSubscriber(newSubscriber.SubscriptionId, newSubscriber.Consumer, newSubscriber.Filter);
}
}
else
{
// We got an item when we don't think we're the subscriber. This is a normal race condition.
// We can drop the item on the floor, or pass it to the rendezvous, or log a warning.
}
}
public static EventStreamId FromStreamId(StreamId id) => new(
GuidId IStreamPubSub.CreateSubscriptionId(StreamId streamId, IStreamConsumerExtension streamConsumer)
{
return(registry.CreateSubscriptionId(streamId, streamConsumer));
}
public Task<List<GuidId>> GetAllSubscriptions(StreamId streamId, IStreamConsumerExtension streamConsumer)
{
return Task.FromResult(new List<GuidId> { CreateSubscriptionId(streamId, streamConsumer) });
}
public IQueueCacheCursor GetCacheCursor(StreamId streamId, StreamSequenceToken token)
{
return(new Cursor(this.cache, streamId, token));
}
Task IStreamPubSub.RegisterConsumer(GuidId subscriptionId, StreamId streamId, string streamProvider, IStreamConsumerExtension streamConsumer, IStreamFilterPredicateWrapper filter)
{
return(registry.RegisterConsumer(subscriptionId, streamId, streamProvider, streamConsumer, filter));
}
internal static void WritingEventToPublisStream(this ILogger logger, EventLogSequenceNumber eventLogSequenceNumber, StreamId stream) => _writingEventToPublisStream(logger, eventLogSequenceNumber, stream, null);
private Task Test_Stream_Churn_NumStreams_FewPublishers(
string streamProviderName,
int pipelineSize,
int numStreams,
int numConsumers = 9,
int numProducers = 4,
bool warmUpPubSub = true,
bool warmUpProducers = false,
bool normalSubscribeCalls = true)
{
output.WriteLine("Testing churn with {0} Streams on {1} Producers with {2} Consumers per Stream",
numStreams, numProducers, numConsumers);
AsyncPipeline pipeline = new AsyncPipeline(pipelineSize);
// Create streamId Guids
StreamId[] streamIds = new StreamId[numStreams];
for (int i = 0; i < numStreams; i++)
{
streamIds[i] = StreamId.Create(this.StreamNamespace, Guid.NewGuid());
}
int activeConsumerGrains = ActiveGrainCount(typeof(StreamLifecycleConsumerGrain));
Assert.Equal(0, activeConsumerGrains); // "Initial Consumer count should be zero"
int activeProducerGrains = ActiveGrainCount(typeof(StreamLifecycleProducerGrain));
Assert.Equal(0, activeProducerGrains); // "Initial Producer count should be zero"
if (warmUpPubSub)
{
WarmUpPubSub(streamProviderName, streamIds, pipeline);
pipeline.Wait();
int activePubSubGrains = ActiveGrainCount(typeof(PubSubRendezvousGrain));
Assert.Equal(streamIds.Length, activePubSubGrains); // "Initial PubSub count -- should all be warmed up"
}
Guid[] producerIds = new Guid[numProducers];
if (numProducers > 0 && warmUpProducers)
{
// Warm up Producers to pre-create grains
for (int i = 0; i < numProducers; i++)
{
producerIds[i] = Guid.NewGuid();
var grain = this.GrainFactory.GetGrain <IStreamLifecycleProducerGrain>(producerIds[i]);
Task promise = grain.Ping();
pipeline.Add(promise);
}
pipeline.Wait();
int activePublisherGrains = this.ActiveGrainCount(typeof(StreamLifecycleProducerGrain));
Assert.Equal(numProducers, activePublisherGrains); // "Initial Publisher count -- should all be warmed up"
}
var promises = new List <Task>();
Stopwatch sw = Stopwatch.StartNew();
if (numProducers > 0)
{
// Producers
for (int i = 0; i < numStreams; i++)
{
StreamId streamId = streamIds[i];
Guid producerId = producerIds[i % numProducers];
var grain = this.GrainFactory.GetGrain <IStreamLifecycleProducerGrain>(producerId);
Task promise = grain.BecomeProducer(streamId, streamProviderName);
promises.Add(promise);
pipeline.Add(promise);
}
pipeline.Wait();
promises.Clear();
}
// Consumers
for (int i = 0; i < numStreams; i++)
{
StreamId streamId = streamIds[i];
Task promise = SetupOneStream(streamId, streamProviderName, pipeline, numConsumers, 0, normalSubscribeCalls);
promises.Add(promise);
}
pipeline.Wait();
Task.WhenAll(promises).Wait();
sw.Stop();
int consumerCount = ActiveGrainCount(typeof(StreamLifecycleConsumerGrain));
Assert.Equal(activeConsumerGrains + (numStreams * numConsumers), consumerCount); // "The right number of Consumer grains are active"
int producerCount = ActiveGrainCount(typeof(StreamLifecycleProducerGrain));
Assert.Equal(activeProducerGrains + (numStreams * numProducers), producerCount); // "The right number of Producer grains are active"
int pubSubCount = ActiveGrainCount(typeof(PubSubRendezvousGrain));
Assert.Equal(streamIds.Length, pubSubCount); // "Final PubSub count -- no more started"
TimeSpan elapsed = sw.Elapsed;
int totalSubscriptions = numStreams * numConsumers;
double rps = totalSubscriptions / elapsed.TotalSeconds;
output.WriteLine("Subscriptions-per-second = {0} during period {1}", rps, elapsed);
Assert.NotEqual(0.0, rps); // "RPS greater than zero"
return(Task.CompletedTask);
}
public async Task RemoveSubscription(string streamProviderName, StreamId streamId, Guid subscriptionId)
{
var internalStreamId = new InternalStreamId(streamProviderName, streamId);
await streamPubSub.UnregisterConsumer(GuidId.GetGuidId(subscriptionId), internalStreamId);
}
public Task <IEnumerable <StreamSubscription> > GetSubscriptions(string streamProviderName, StreamId streamId)
{
var internalStreamId = new InternalStreamId(streamProviderName, streamId);
return(streamPubSub.GetAllSubscriptions(internalStreamId).ContinueWith(subs => subs.Result.AsEnumerable()));
}
public async Task <StreamSubscription> AddSubscription(string streamProviderName, StreamId streamId, GrainReference grainRef)
{
var consumer = grainRef.AsReference <IStreamConsumerExtension>();
var internalStreamId = new InternalStreamId(streamProviderName, streamId);
var subscriptionId = streamPubSub.CreateSubscriptionId(internalStreamId, consumer);
await streamPubSub.RegisterConsumer(subscriptionId, internalStreamId, consumer, null);
var newSub = new StreamSubscription(subscriptionId.Guid, streamProviderName, streamId, grainRef.GrainId);
return(newSub);
}
/// <summary>
/// Initializes a new instance of the <see cref="StreamDefinitionDoesNotExist" /> class.
/// </summary>
/// <param name="stream">The stream id.</param>
/// <param name="scope">The scope id.</param>
public StreamDefinitionDoesNotExist(StreamId stream, ScopeId scope)
: base($"Stream definition for stream {stream.Value} in scope {scope.Value} does not exist")
{
}
Task IStreamPubSub.UnregisterConsumer(GuidId subscriptionId, StreamId streamId, string streamProvider)
{
return(registry.UnregisterConsumer(subscriptionId, streamId, streamProvider));
}
public static string ReadStreamBackwards(
StreamId streamId,
int fromVersionInclusive,
int maxCount,
bool prefetchJsonData)
=> ReadStream(streamId, fromVersionInclusive, maxCount, prefetchJsonData, Constants.Direction.Backwards);
private static ReadStreamPage ReadStreamForwardsInternal(
IHalClient client,
StreamId streamId,
int fromVersionInclusive,
bool prefetchJsonData)
{
var resource = client.Current.First();
if (client.StatusCode == HttpStatusCode.NotFound)
{
return(new ReadStreamPage(
streamId,
PageReadStatus.StreamNotFound,
fromVersionInclusive,
-1,
-1,
-1,
ReadDirection.Forward,
true));
}
var pageInfo = resource.Data <HalReadPage>();
var streamMessages = Convert(
resource.Embedded
.Where(r => r.Rel == Constants.Relations.Message)
.Reverse()
.ToArray(),
client,
prefetchJsonData);
var readStreamPage = new ReadStreamPage(
streamId,
PageReadStatus.Success,
pageInfo.FromStreamVersion,
pageInfo.NextStreamVersion,
pageInfo.LastStreamVersion,
pageInfo.LastStreamPosition,
ReadDirection.Forward,
pageInfo.IsEnd,
ReadNextStreamPage,
streamMessages);
return(readStreamPage);
async Task <ReadStreamPage> ReadNextStreamPage(int nextVersion, CancellationToken ct)
=> resource.Links.Any(link => link.Rel == Constants.Relations.Next)
? ReadStreamForwardsInternal(
await client.GetAsync(resource, Constants.Relations.Next),
streamId,
pageInfo.LastStreamVersion,
prefetchJsonData)
: new ReadStreamPage(
streamId,
PageReadStatus.Success,
pageInfo.LastStreamVersion,
nextVersion,
pageInfo.LastStreamVersion,
pageInfo.LastStreamPosition,
ReadDirection.Forward,
true,
ReadNextStreamPage);
}
internal bool IsImplicitSubscriber(IAddressable addressable, StreamId streamId)
{
return implicitTable.IsImplicitSubscriber(GrainExtensions.GetGrainId(addressable), streamId);
}
Task <bool> IStreamPubSub.FaultSubscription(StreamId streamId, GuidId subscriptionId)
{
return(registry.FaultSubscription(streamId, subscriptionId));
}
public static string StreamByMessageId(StreamId streamId, Guid messageId)
=> $"{Stream(streamId)}/{messageId}";
public static string Stream(StreamId streamId)
=> $"/streams/{streamId}";
public IEnumerable<StreamConsumerData> AllConsumersForStream(StreamId streamId)
{
return queueData.Values.Where(consumer => consumer.StreamId.Equals(streamId));
}
Task IStreamPubSub.UnregisterProducer(StreamId streamId, string streamProvider, IStreamProducerExtension streamProducer)
{
return(registry.UnregisterProducer(streamId, streamProvider, streamProducer));
}
// Called by rendezvous when new remote subscriber subscribes to this stream.
private async Task AddSubscriber_Impl(
GuidId subscriptionId,
StreamId streamId,
IStreamConsumerExtension streamConsumer,
StreamSequenceToken cacheToken,
IStreamFilterPredicateWrapper filter)
{
if (IsShutdown) return;
StreamConsumerCollection streamDataCollection;
if (!pubSubCache.TryGetValue(streamId, out streamDataCollection))
{
streamDataCollection = new StreamConsumerCollection(DateTime.UtcNow);
pubSubCache.Add(streamId, streamDataCollection);
}
StreamConsumerData data;
if (!streamDataCollection.TryGetConsumer(subscriptionId, out data))
data = streamDataCollection.AddConsumer(subscriptionId, streamId, streamConsumer, filter ?? DefaultStreamFilter);
if (await DoHandshakeWithConsumer(data, cacheToken))
{
if (data.State == StreamConsumerDataState.Inactive)
RunConsumerCursor(data, data.Filter).Ignore(); // Start delivering events if not actively doing so
}
}
public void RemoveSubscriber_Impl(GuidId subscriptionId, StreamId streamId)
{
if (IsShutdown) return;
StreamConsumerCollection streamData;
if (!pubSubCache.TryGetValue(streamId, out streamData)) return;
// remove consumer
bool removed = streamData.RemoveConsumer(subscriptionId, logger);
if (removed && logger.IsVerbose) logger.Verbose((int)ErrorCode.PersistentStreamPullingAgent_10, "Removed Consumer: subscription={0}, for stream {1}.", subscriptionId, streamId);
if (streamData.Count == 0)
pubSubCache.Remove(streamId);
}
/// <summary>
/// For test purpose. ConfigureDataGeneratorForStream will configure a data generator for the stream
/// </summary>
/// <param name="streamId"></param>
internal void ConfigureDataGeneratorForStream(StreamId streamId)
{
(this.receiver as EventHubPartitionGeneratorReceiver)?.ConfigureDataGeneratorForStream(streamId);
}
private async Task RegisterAsStreamProducer(StreamId streamId, StreamSequenceToken streamStartToken)
{
try
{
if (pubSub == null) throw new NullReferenceException("Found pubSub reference not set up correctly in RetreaveNewStream");
IStreamProducerExtension meAsStreamProducer = this.AsReference<IStreamProducerExtension>();
ISet<PubSubSubscriptionState> streamData = await pubSub.RegisterProducer(streamId, streamProviderName, meAsStreamProducer);
if (logger.IsVerbose) logger.Verbose((int)ErrorCode.PersistentStreamPullingAgent_16, "Got back {0} Subscribers for stream {1}.", streamData.Count, streamId);
var addSubscriptionTasks = new List<Task>(streamData.Count);
foreach (PubSubSubscriptionState item in streamData)
{
addSubscriptionTasks.Add(AddSubscriber_Impl(item.SubscriptionId, item.Stream, item.Consumer, streamStartToken, item.Filter));
}
await Task.WhenAll(addSubscriptionTasks);
}
catch (Exception exc)
{
// RegisterAsStreamProducer is fired with .Ignore so we should log if anything goes wrong, because there is no one to catch the exception
logger.Error((int)ErrorCode.PersistentStreamPullingAgent_17, "Ignored RegisterAsStreamProducer Error", exc);
throw;
}
}
private async Task <double> TestOneStream(Guid streamId, string streamProviderName,
int numProducers, int numConsumers, int numMessages,
bool useFanOut = true)
{
output.WriteLine("Testing Stream {0} with Producers={1} Consumers={2} x {3} messages",
streamId, numProducers, numConsumers, numMessages);
Stopwatch sw = Stopwatch.StartNew();
List <IStreamLifecycleConsumerGrain> consumers = new List <IStreamLifecycleConsumerGrain>();
List <IStreamLifecycleProducerGrain> producers = new List <IStreamLifecycleProducerGrain>();
await InitializeTopology(streamId, this.StreamNamespace, streamProviderName,
numProducers, numConsumers,
producers, consumers, useFanOut);
var promises = new List <Task>();
// Producers send M message each
int item = 1;
AsyncPipeline pipeline = new AsyncPipeline(MessagePipelineSize);
foreach (var grain in producers)
{
for (int m = 0; m < numMessages; m++)
{
Task promise = grain.SendItem(item++);
if (useFanOut)
{
pipeline.Add(promise);
promises.Add(promise);
}
else
{
await promise;
}
}
}
if (useFanOut)
{
//output.WriteLine("Test: Waiting for {0} producers to finish sending {1} messages", producers.Count, promises.Count);
await Task.WhenAll(promises);
promises.Clear();
}
var pubSub = StreamTestUtils.GetStreamPubSub(this.InternalClient);
// Check Consumer counts
var streamId1 = new InternalStreamId(streamProviderName, StreamId.Create(StreamNamespace, streamId));
int consumerCount = await pubSub.ConsumerCount(streamId1);
Assert.Equal(numConsumers, consumerCount); // "ConsumerCount for Stream {0}", streamId
// Check Producer counts
int producerCount = await pubSub.ProducerCount(streamId1);
Assert.Equal(numProducers, producerCount); // "ProducerCount for Stream {0}", streamId
// Check message counts received by consumers
int totalMessages = (numMessages + 1) * numProducers;
foreach (var grain in consumers)
{
int count = await grain.GetReceivedCount();
Assert.Equal(totalMessages, count); // "ReceivedCount for Consumer grain {0}", grain.GetPrimaryKey());
}
double rps = totalMessages / sw.Elapsed.TotalSeconds;
//output.WriteLine("Sent {0} messages total from {1} Producers to {2} Consumers in {3} at {4} RPS",
// totalMessages, numProducers, numConsumers,
// sw.Elapsed, rps);
return(rps);
}
internal bool IsImplicitSubscriber(GuidId subscriptionId, StreamId streamId)
{
return SubscriptionMarker.IsImplicitSubscription(subscriptionId.Guid);
}
public Task AddSubscriber(GuidId subscriptionId, StreamId streamId, IStreamConsumerExtension streamConsumer,
IStreamFilterPredicateWrapper filter)
{
return(TaskDone.Done);
}
Task <List <StreamSubscription> > IStreamPubSub.GetAllSubscriptions(StreamId streamId, IStreamConsumerExtension streamConsumer)
{
return(registry.GetAllSubscriptions(streamId, streamConsumer));
}
public Task AddSubscriber(
GuidId subscriptionId,
StreamId streamId,
IStreamConsumerExtension streamConsumer,
IStreamFilterPredicateWrapper filter)
{
if (logger.IsVerbose) logger.Verbose((int)ErrorCode.PersistentStreamPullingAgent_09, "AddSubscriber: Stream={0} Subscriber={1}.", streamId, streamConsumer);
// cannot await here because explicit consumers trigger this call, so it could cause a deadlock.
AddSubscriber_Impl(subscriptionId, streamId, streamConsumer, null, filter)
.LogException(logger, ErrorCode.PersistentStreamPullingAgent_26,
String.Format("Failed to add subscription for stream {0}." , streamId))
.Ignore();
return TaskDone.Done;
}
public Task RemoveSubscriber(GuidId subscriptionId, StreamId streamId)
{
return(TaskDone.Done);
}
public Task RemoveSubscriber(GuidId subscriptionId, StreamId streamId)
{
RemoveSubscriber_Impl(subscriptionId, streamId);
return TaskDone.Done;
}
private async Task SendAndReceiveFromQueueAdapter(IQueueAdapterFactory adapterFactory)
{
IQueueAdapter adapter = await adapterFactory.CreateAdapter();
IQueueAdapterCache cache = adapterFactory.GetQueueAdapterCache();
// Create receiver per queue
IStreamQueueMapper mapper = adapterFactory.GetStreamQueueMapper();
Dictionary <QueueId, IQueueAdapterReceiver> receivers = mapper.GetAllQueues().ToDictionary(queueId => queueId, adapter.CreateReceiver);
Dictionary <QueueId, IQueueCache> caches = mapper.GetAllQueues().ToDictionary(queueId => queueId, cache.CreateQueueCache);
await Task.WhenAll(receivers.Values.Select(receiver => receiver.Initialize(TimeSpan.FromSeconds(5))));
// test using 2 streams
Guid streamId1 = Guid.NewGuid();
Guid streamId2 = Guid.NewGuid();
int receivedBatches = 0;
var streamsPerQueue = new ConcurrentDictionary <QueueId, HashSet <StreamId> >();
// reader threads (at most 2 active queues because only two streams)
var work = new List <Task>();
foreach (KeyValuePair <QueueId, IQueueAdapterReceiver> receiverKvp in receivers)
{
QueueId queueId = receiverKvp.Key;
var receiver = receiverKvp.Value;
var qCache = caches[queueId];
Task task = Task.Factory.StartNew(() =>
{
while (receivedBatches < NumBatches)
{
var messages = receiver.GetQueueMessagesAsync(SQSStorage.MAX_NUMBER_OF_MESSAGE_TO_PEAK).Result.ToArray();
if (!messages.Any())
{
continue;
}
foreach (var message in messages.Cast <SQSBatchContainer>())
{
streamsPerQueue.AddOrUpdate(queueId,
id => new HashSet <StreamId> {
message.StreamId
},
(id, set) =>
{
set.Add(message.StreamId);
return(set);
});
output.WriteLine("Queue {0} received message on stream {1}", queueId,
message.StreamId);
Assert.Equal(NumMessagesPerBatch / 2, message.GetEvents <int>().Count()); // "Half the events were ints"
Assert.Equal(NumMessagesPerBatch / 2, message.GetEvents <string>().Count()); // "Half the events were strings"
}
Interlocked.Add(ref receivedBatches, messages.Length);
qCache.AddToCache(messages);
}
});
work.Add(task);
}
// send events
List <object> events = CreateEvents(NumMessagesPerBatch);
work.Add(Task.Factory.StartNew(() => Enumerable.Range(0, NumBatches)
.Select(i => i % 2 == 0 ? streamId1 : streamId2)
.ToList()
.ForEach(streamId =>
adapter.QueueMessageBatchAsync(StreamId.Create(streamId.ToString(), streamId),
events.Take(NumMessagesPerBatch).ToArray(), null, RequestContextExtensions.Export(this.fixture.SerializationManager)).Wait())));
await Task.WhenAll(work);
// Make sure we got back everything we sent
Assert.Equal(NumBatches, receivedBatches);
// check to see if all the events are in the cache and we can enumerate through them
StreamSequenceToken firstInCache = new EventSequenceTokenV2(0);
foreach (KeyValuePair <QueueId, HashSet <StreamId> > kvp in streamsPerQueue)
{
var receiver = receivers[kvp.Key];
var qCache = caches[kvp.Key];
foreach (StreamId streamGuid in kvp.Value)
{
// read all messages in cache for stream
IQueueCacheCursor cursor = qCache.GetCacheCursor(streamGuid, firstInCache);
int messageCount = 0;
StreamSequenceToken tenthInCache = null;
StreamSequenceToken lastToken = firstInCache;
while (cursor.MoveNext())
{
Exception ex;
messageCount++;
IBatchContainer batch = cursor.GetCurrent(out ex);
output.WriteLine("Token: {0}", batch.SequenceToken);
Assert.True(batch.SequenceToken.CompareTo(lastToken) >= 0, $"order check for event {messageCount}");
lastToken = batch.SequenceToken;
if (messageCount == 10)
{
tenthInCache = batch.SequenceToken;
}
}
output.WriteLine("On Queue {0} we received a total of {1} message on stream {2}", kvp.Key, messageCount, streamGuid);
Assert.Equal(NumBatches / 2, messageCount);
Assert.NotNull(tenthInCache);
// read all messages from the 10th
cursor = qCache.GetCacheCursor(streamGuid, tenthInCache);
messageCount = 0;
while (cursor.MoveNext())
{
messageCount++;
}
output.WriteLine("On Queue {0} we received a total of {1} message on stream {2}", kvp.Key, messageCount, streamGuid);
const int expected = NumBatches / 2 - 10 + 1; // all except the first 10, including the 10th (10 + 1)
Assert.Equal(expected, messageCount);
}
}
}
private async Task RegisterStream(StreamId streamId, StreamSequenceToken firstToken, DateTime now)
{
var streamData = new StreamConsumerCollection(now);
pubSubCache.Add(streamId, streamData);
// Create a fake cursor to point into a cache.
// That way we will not purge the event from the cache, until we talk to pub sub.
// This will help ensure the "casual consistency" between pre-existing subscripton (of a potentially new already subscribed consumer)
// and later production.
var pinCursor = queueCache.GetCacheCursor(streamId.Guid, streamId.Namespace, firstToken);
try
{
await RegisterAsStreamProducer(streamId, firstToken);
}finally
{
// Cleanup the fake pinning cursor.
pinCursor.Dispose();
}
}
/// <inheritdoc/>
public async Task <Try <IFilterDefinition> > TryGetFromStream(ScopeId scopeId, StreamId streamId, CancellationToken cancellationToken)
{
var tryGetStream = await _streamDefinitions.TryGet(scopeId, streamId, cancellationToken).ConfigureAwait(false);
return(tryGetStream.Success, tryGetStream.Result?.FilterDefinition);
}
internal FaultedSubscriptionException(GuidId subscriptionId, StreamId streamId)
: base(string.Format(ErrorStringFormat, subscriptionId.Guid, streamId)) { }
/// <summary>
/// 接收消息
/// </summary>
/// <param name="content">消息内容</param>
/// <param name="token">StreamSequenceToken</param>
protected override Task OnReceive(IsOperationPoint content, StreamSequenceToken token)
{
UserMessage.Send(StreamId.ToString(), Name, Utilities.JsonSerialize(content));
return(Task.CompletedTask);
}
public Task<bool> FaultSubscription(StreamId streamId, GuidId subscriptionId)
{
return Task.FromResult(false);
}
private async Task DeliverToRemote(IStreamConsumerExtension remoteConsumer, StreamId streamId, GuidId subscriptionId, object item, bool optimizeForImmutableData)
{
try
{
if (optimizeForImmutableData)
{
await remoteConsumer.DeliverImmutable(subscriptionId, streamId, new Immutable <object>(item), null, null);
}
else
{
await remoteConsumer.DeliverMutable(subscriptionId, streamId, item, null, null);
}
}
catch (ClientNotAvailableException)
{
Tuple <IStreamConsumerExtension, IStreamFilterPredicateWrapper> discard;
if (consumers.TryRemove(subscriptionId, out discard))
{
streamPubSub.UnregisterConsumer(subscriptionId, streamId, streamId.ProviderName).Ignore();
logger.Warn(ErrorCode.Stream_ConsumerIsDead,
"Consumer {0} on stream {1} is no longer active - permanently removing Consumer.", remoteConsumer, streamId);
}
}
}
public Task UnregisterProducer(StreamId streamId, string streamProvider, IStreamProducerExtension streamProducer)
{
return TaskDone.Done;
}
/// <inheritdoc/> public Task Write(CommittedEvent @event, ScopeId scope, StreamId streamId, PartitionId partitionId, CancellationToken cancellationToken) => Write(@event, streamId, partitionId, cancellationToken);
public Task UnregisterConsumer(GuidId subscriptionId, StreamId streamId, string streamProvider)
{
return TaskDone.Done;
}
internal void StopProducingOnStream(StreamId streamId)
{
(this.receiver as EventHubPartitionGeneratorReceiver)?.StopProducingOnStream(streamId);
}
internal void RemoveStream(StreamId streamId)
{
remoteConsumers.Remove(streamId);
}
public Cursor(IEventHubQueueCache cache, StreamId streamId, StreamSequenceToken token)
{
this.cache = cache;
this.cursor = cache.GetCursor(streamId, token);
}
