public async Task StreamingTests_Consumer_Producer_UnSubscribe()
{
var streamId = new FullStreamIdentity(Guid.NewGuid(), "EmptySpace", StreamProviderName);
var subGrain = this.fixture.GrainFactory.GetGrain <ISubscribeGrain>(Guid.NewGuid());
//set up subscription for consumer grains
var subscriptions = await subGrain.SetupStreamingSubscriptionForStream <IPassive_ConsumerGrain>(streamId, 2);
var producer = this.fixture.GrainFactory.GetGrain <ITypedProducerGrainProducingInt>(Guid.NewGuid());
await producer.BecomeProducer(streamId.Guid, streamId.Namespace, streamId.ProviderName);
await producer.StartPeriodicProducing();
int numProduced = 0;
await TestingUtils.WaitUntilAsync(lastTry => ProducerHasProducedSinceLastCheck(numProduced, producer, lastTry), _timeout);
//the subscription to remove
var subscription = subscriptions[0];
// remove subscription
await subGrain.RemoveSubscription(subscription);
var numProducedWhenUnSub = await producer.GetNumberProduced();
var consumerUnSub = this.fixture.GrainFactory.GetGrain <IPassive_ConsumerGrain>(subscription.GrainId.PrimaryKey);
var consumerNormal = this.fixture.GrainFactory.GetGrain <IPassive_ConsumerGrain>(subscriptions[1].GrainId.PrimaryKey);
//assert consumer grain's onAdd func got called.
Assert.True((await consumerUnSub.GetCountOfOnAddFuncCalled()) > 0);
await TestingUtils.WaitUntilAsync(lastTry => ProducerHasProducedSinceLastCheck(numProducedWhenUnSub, producer, lastTry), _timeout);
await producer.StopPeriodicProducing();
//wait for consumers to finish consuming
await Task.Delay(TimeSpan.FromMilliseconds(2000));
//assert normal consumer consumed equal to produced
await TestingUtils.WaitUntilAsync(
lastTry => CheckCounters(new List <ITypedProducerGrain> {
producer
}, consumerNormal, lastTry, this.fixture.Logger), _timeout);
//asert unsubscribed consumer consumed less than produced
numProduced = await producer.GetNumberProduced();
var numConsumed = await consumerUnSub.GetNumberConsumed();
Assert.True(numConsumed <= numProducedWhenUnSub);
Assert.True(numConsumed < numProduced);
// clean up test
await consumerNormal.StopConsuming();
await consumerUnSub.StopConsuming();
}
public async Task StreamingTests_Consumer_Producer_SubscribeToStreamsHandledByDifferentStreamProvider()
{
var subscriptionManager = new SubscriptionManager(this.fixture.HostedCluster);
var streamId = new FullStreamIdentity(Guid.NewGuid(), "EmptySpace", StreamProviderName);
//set up subscription for 10 consumer grains
var subscriptions = await subscriptionManager.SetupStreamingSubscriptionForStream <IPassive_ConsumerGrain>(streamId, 10);
var consumers = subscriptions.Select(sub => this.fixture.GrainFactory.GetGrain <IPassive_ConsumerGrain>(sub.GrainId.PrimaryKey)).ToList();
var producer = this.fixture.GrainFactory.GetGrain <ITypedProducerGrainProducingApple>(Guid.NewGuid());
await producer.BecomeProducer(streamId.Guid, streamId.Namespace, streamId.ProviderName);
await producer.StartPeriodicProducing();
int numProduced = 0;
await TestingUtils.WaitUntilAsync(lastTry => ProducerHasProducedSinceLastCheck(numProduced, producer, lastTry), _timeout);
// set up the new stream to subscribe, which produce strings
var streamId2 = new FullStreamIdentity(Guid.NewGuid(), "EmptySpace2", StreamProviderName2);
var producer2 = this.fixture.GrainFactory.GetGrain <ITypedProducerGrainProducingApple>(Guid.NewGuid());
await producer2.BecomeProducer(streamId2.Guid, streamId2.Namespace, streamId2.ProviderName);
//register the consumer grain to second stream
var tasks = consumers.Select(consumer => subscriptionManager.AddSubscription <IPassive_ConsumerGrain>(streamId2, consumer.GetPrimaryKey())).ToList();
await Task.WhenAll(tasks);
await producer2.StartPeriodicProducing();
await TestingUtils.WaitUntilAsync(lastTry => ProducerHasProducedSinceLastCheck(numProduced, producer2, lastTry), _timeout);
await producer.StopPeriodicProducing();
await producer2.StopPeriodicProducing();
var tasks2 = new List <Task>();
foreach (var consumer in consumers)
{
tasks2.Add(TestingUtils.WaitUntilAsync(lastTry => CheckCounters(new List <ITypedProducerGrain> {
producer, producer2
},
consumer, lastTry, this.fixture.Logger), _timeout));
}
await Task.WhenAll(tasks);
//clean up test
tasks2.Clear();
tasks2 = consumers.Select(consumer => consumer.StopConsuming()).ToList();
await Task.WhenAll(tasks2);
}
public async Task EHStatistics_MonitorCalledAccordingly()
{
var streamId = new FullStreamIdentity(Guid.NewGuid(), StreamNamespace, StreamProviderName);
//set up one slow consumer grain
var slowConsumer = this.fixture.GrainFactory.GetGrain <ISlowConsumingGrain>(Guid.NewGuid());
await slowConsumer.BecomeConsumer(streamId.Guid, StreamNamespace, StreamProviderName);
//set up 30 healthy consumer grain to show how much we favor slow consumer
int healthyConsumerCount = 30;
var healthyConsumers = await EHSlowConsumingTests.SetUpHealthyConsumerGrain(this.fixture.GrainFactory, streamId.Guid, StreamNamespace, StreamProviderName, healthyConsumerCount);
//configure data generator for stream and start producing
var mgmtGrain = this.fixture.GrainFactory.GetGrain <IManagementGrain>(0);
var randomStreamPlacementArg = new EventDataGeneratorStreamProvider.AdapterFactory.StreamRandomPlacementArg(streamId, this.seed.Next(100));
await mgmtGrain.SendControlCommandToProvider(typeof(EHStreamProviderForMonitorTests).FullName, StreamProviderName,
(int)EventDataGeneratorStreamProvider.AdapterFactory.Commands.Randomly_Place_Stream_To_Queue, randomStreamPlacementArg);
//since there's an extreme slow consumer, so the back pressure algorithm should be triggered
await TestingUtils.WaitUntilAsync(lastTry => AssertCacheBackPressureTriggered(true, lastTry), timeout);
//make slow consumer stop consuming
await slowConsumer.StopConsuming();
//assert EventHubReceiverMonitor call counters
var receiverMonitorCounters = await mgmtGrain.SendControlCommandToProvider(typeof(EHStreamProviderForMonitorTests).FullName, StreamProviderName,
(int)EHStreamProviderForMonitorTests.AdapterFactory.QueryCommands.GetReceiverMonitorCallCounters, null);
foreach (var callCounter in receiverMonitorCounters)
{
AssertReceiverMonitorCallCounters(callCounter as EventHubReceiverMonitorCounters);
}
var cacheMonitorCounters = await mgmtGrain.SendControlCommandToProvider(typeof(EHStreamProviderForMonitorTests).FullName, StreamProviderName,
(int)EHStreamProviderForMonitorTests.AdapterFactory.QueryCommands.GetCacheMonitorCallCounters, null);
foreach (var callCounter in cacheMonitorCounters)
{
AssertCacheMonitorCallCounters(callCounter as CacheMonitorCounters);
}
var objectPoolMonitorCounters = await mgmtGrain.SendControlCommandToProvider(typeof(EHStreamProviderForMonitorTests).FullName, StreamProviderName,
(int)EHStreamProviderForMonitorTests.AdapterFactory.QueryCommands.GetObjectPoolMonitorCallCounters, null);
foreach (var callCounter in objectPoolMonitorCounters)
{
AssertObjectPoolMonitorCallCounters(callCounter as ObjectPoolMonitorCounters);
}
}
public async Task EHStatistics_MonitorCalledAccordingly()
{
var streamId = new FullStreamIdentity(Guid.NewGuid(), StreamNamespace, StreamProviderName);
//set up 30 healthy consumer grain to show how much we favor slow consumer
int healthyConsumerCount = 30;
_ = await EHSlowConsumingTests.SetUpHealthyConsumerGrain(this.fixture.GrainFactory, streamId.Guid, StreamNamespace, StreamProviderName, healthyConsumerCount);
//configure data generator for stream and start producing
var mgmtGrain = this.fixture.GrainFactory.GetGrain <IManagementGrain>(0);
var randomStreamPlacementArg = new EHStreamProviderForMonitorTestsAdapterFactory.StreamRandomPlacementArg(streamId, this.seed.Next(100));
await mgmtGrain.SendControlCommandToProvider(typeof(PersistentStreamProvider).FullName, StreamProviderName,
(int)EHStreamProviderForMonitorTestsAdapterFactory.Commands.Randomly_Place_Stream_To_Queue, randomStreamPlacementArg);
// let the test to run for a while to build up some streaming traffic
await Task.Delay(timeout);
//wait sometime after cache pressure changing, for the system to notice it and trigger cache monitor to track it
await mgmtGrain.SendControlCommandToProvider(typeof(PersistentStreamProvider).FullName, StreamProviderName,
(int)EHStreamProviderForMonitorTestsAdapterFactory.QueryCommands.ChangeCachePressure, null);
await Task.Delay(timeout);
//assert EventHubReceiverMonitor call counters
var receiverMonitorCounters = await mgmtGrain.SendControlCommandToProvider(typeof(PersistentStreamProvider).FullName, StreamProviderName,
(int)EHStreamProviderForMonitorTestsAdapterFactory.QueryCommands.GetReceiverMonitorCallCounters, null);
foreach (var callCounter in receiverMonitorCounters)
{
AssertReceiverMonitorCallCounters(callCounter as EventHubReceiverMonitorCounters);
}
var cacheMonitorCounters = await mgmtGrain.SendControlCommandToProvider(typeof(PersistentStreamProvider).FullName, StreamProviderName,
(int)EHStreamProviderForMonitorTestsAdapterFactory.QueryCommands.GetCacheMonitorCallCounters, null);
foreach (var callCounter in cacheMonitorCounters)
{
AssertCacheMonitorCallCounters(callCounter as CacheMonitorCounters);
}
var objectPoolMonitorCounters = await mgmtGrain.SendControlCommandToProvider(typeof(PersistentStreamProvider).FullName, StreamProviderName,
(int)EHStreamProviderForMonitorTestsAdapterFactory.QueryCommands.GetObjectPoolMonitorCallCounters, null);
foreach (var callCounter in objectPoolMonitorCounters)
{
AssertObjectPoolMonitorCallCounters(callCounter as ObjectPoolMonitorCounters);
}
}
public async Task Programmatic_Subscribe_UsingClientSideSubscriptionManager_UsingDynamicProviderConfig()
{
var streamId = new FullStreamIdentity(Guid.NewGuid(), "EmptySpace", StreamProviderName);
var subManager = this.Client.ServiceProvider.GetService <IStreamSubscriptionManagerAdmin>()
.GetStreamSubscriptionManager(StreamSubscriptionManagerType.ExplicitSubscribeOnly);
//set up stream subscriptions for grains
var subscriptions = await SetupStreamingSubscriptionForStream <IPassive_ConsumerGrain>(subManager, this.GrainFactory, streamId, 10);
var consumers = subscriptions.Select(sub => this.GrainFactory.GetGrain <IPassive_ConsumerGrain>(sub.GrainId.PrimaryKey)).ToList();
// configure stream provider after subscriptions set up
await AddSimpleStreamProviderAndUpdate(new List <String>() { StreamProviderName });
//set up producer
var producer = this.GrainFactory.GetGrain <ITypedProducerGrainProducingApple>(Guid.NewGuid());
await producer.BecomeProducer(streamId.Guid, streamId.Namespace, streamId.ProviderName);
await producer.StartPeriodicProducing();
int numProduced = 0;
await TestingUtils.WaitUntilAsync(lastTry => ProgrammaticSubcribeTestsRunner.ProducerHasProducedSinceLastCheck(numProduced, producer, lastTry), _timeout);
await producer.StopPeriodicProducing();
var tasks = new List <Task>();
foreach (var consumer in consumers)
{
tasks.Add(TestingUtils.WaitUntilAsync(lastTry => ProgrammaticSubcribeTestsRunner.CheckCounters(new List <ITypedProducerGrain> {
producer
},
consumer, lastTry, this.Logger), _timeout));
}
await Task.WhenAll(tasks);
//clean up test
tasks.Clear();
tasks = consumers.Select(consumer => consumer.StopConsuming()).ToList();
await Task.WhenAll(tasks);
}
public async Task StreamingTests_Consumer_Producer_Subscribe()
{
var streamId = new FullStreamIdentity(Guid.NewGuid(), ImplicitSubscribeGrain.StreamNameSpace, StreamProviderName);
var producer = this.fixture.HostedCluster.GrainFactory.GetGrain <ITypedProducerGrainProducingApple>(Guid.NewGuid());
await producer.BecomeProducer(streamId.Guid, streamId.Namespace, streamId.ProviderName);
await producer.StartPeriodicProducing();
int numProduced = 0;
await TestingUtils.WaitUntilAsync(lastTry => ProgrammaticSubcribeTestsRunner.ProducerHasProducedSinceLastCheck(numProduced, producer, lastTry), _timeout);
await producer.StopPeriodicProducing();
var implicitConsumer =
this.fixture.HostedCluster.GrainFactory.GetGrain <IImplicitSubscribeGrain>(streamId.Guid);
await TestingUtils.WaitUntilAsync(lastTry => ProgrammaticSubcribeTestsRunner.CheckCounters(new List <ITypedProducerGrain> {
producer
},
implicitConsumer, lastTry, this.fixture.Logger), _timeout);
//clean up test
await implicitConsumer.StopConsuming();
}
public async Task RemoveSubscription(FullStreamIdentity streamId, Guid subscriptionId)
{
await subManager.RemoveSubscription(streamId.ProviderName, streamId, subscriptionId);
}
public Task <IEnumerable <StreamSubscription> > GetSubscriptions(FullStreamIdentity streamIdentity)
{
return(subManager.GetSubscriptions(streamIdentity.ProviderName, streamIdentity));
}
public async Task <List <StreamSubscription> > SetupStreamingSubscriptionForStream <TGrainInterface>(FullStreamIdentity streamIdentity, int grainCount)
where TGrainInterface : IGrainWithGuidKey
{
var subscriptions = new List <StreamSubscription>();
while (grainCount > 0)
{
var grainId = Guid.NewGuid();
var grainRef = this.grainFactory.GetGrain <TGrainInterface>(grainId) as GrainReference;
subscriptions.Add(await subManager.AddSubscription(streamIdentity.ProviderName, streamIdentity, grainRef));
grainCount--;
}
return(subscriptions);
}
private async Task <List <StreamSubscription> > SetupStreamingSubscriptionForGrains <TGrainInterface>(IStreamSubscriptionManager subManager,
FullStreamIdentity streamIdentity, List <TGrainInterface> grains)
where TGrainInterface : IGrainWithGuidKey
{
var subscriptions = new List <StreamSubscription>();
foreach (var grain in grains)
{
var grainRef = grain as GrainReference;
subscriptions.Add(await subManager.AddSubscription(streamIdentity.ProviderName, streamIdentity, grainRef));
}
return(subscriptions);
}
private async Task <List <StreamSubscription> > SetupStreamingSubscriptionForStream <TGrainInterface>(IStreamSubscriptionManager subManager, IGrainFactory grainFactory,
FullStreamIdentity streamIdentity, int grainCount)
where TGrainInterface : IGrainWithGuidKey
{
//generate grain refs
List <TGrainInterface> grains = new List <TGrainInterface>();
while (grainCount > 0)
{
var grainId = Guid.NewGuid();
var grain = grainFactory.GetGrain <TGrainInterface>(grainId);
grains.Add(grain);
grainCount--;
}
return(await SetupStreamingSubscriptionForGrains(subManager, streamIdentity, grains));
}
