public async Task AddAndRemoveSubscriptionTest(Guid streamGuid, string streamNamespace)
{
// get producer and consumer
var producer = GrainClient.GrainFactory.GetGrain <ISampleStreaming_ProducerGrain>(Guid.NewGuid());
var consumer = GrainClient.GrainFactory.GetGrain <IMultipleSubscriptionConsumerGrain>(Guid.NewGuid());
await producer.BecomeProducer(streamGuid, streamNamespace, streamProviderName);
// setup one subscription and send messsages
StreamSubscriptionHandle <int> firstSubscriptionHandle = await consumer.BecomeConsumer(streamGuid, streamNamespace, streamProviderName);
await producer.StartPeriodicProducing();
await Task.Delay(TimeSpan.FromMilliseconds(1000));
await producer.StopPeriodicProducing();
await TestingUtils.WaitUntilAsync(lastTry => CheckCounters(producer, consumer, 1, lastTry), Timeout);
// clear counts
await consumer.ClearNumberConsumed();
await producer.ClearNumberProduced();
// setup second subscription and send messages
StreamSubscriptionHandle <int> secondSubscriptionHandle = await consumer.BecomeConsumer(streamGuid, streamNamespace, streamProviderName);
await producer.StartPeriodicProducing();
await Task.Delay(TimeSpan.FromMilliseconds(1000));
await producer.StopPeriodicProducing();
await TestingUtils.WaitUntilAsync(lastTry => CheckCounters(producer, consumer, 2, lastTry), Timeout);
// clear counts
await consumer.ClearNumberConsumed();
await producer.ClearNumberProduced();
// remove first subscription and send messages
await consumer.StopConsuming(firstSubscriptionHandle);
await producer.StartPeriodicProducing();
await Task.Delay(TimeSpan.FromMilliseconds(1000));
await producer.StopPeriodicProducing();
await TestingUtils.WaitUntilAsync(lastTry => CheckCounters(producer, consumer, 1, lastTry), Timeout);
// remove second subscription
await consumer.StopConsuming(secondSubscriptionHandle);
}
public async Task SubscribingWithTooOldSequenceToken()
{
// get producer and consumer
var firstProducer = GrainClient.GrainFactory.GetGrain <ISampleStreaming_ProducerGrain>(Guid.NewGuid());
var consumer = GrainClient.GrainFactory.GetGrain <ITimedConsumerGrain>(Guid.NewGuid());
Guid firstStream = Guid.NewGuid();
string streamNamespace = "RewindingTestNamespace";
// subscribing to different streams on the same namespace
await consumer.BecomeConsumer(firstStream, streamNamespace, null, TimeSpan.Zero, _streamProviderName);
// time to produce
await firstProducer.BecomeProducer(firstStream, streamNamespace, _streamProviderName);
// Starting to produce!
int numToProduce = 5;
for (int i = 1; i <= numToProduce; i++)
{
await firstProducer.Produce();
}
await
TestingUtils.WaitUntilAsync(lastTry => CheckCounters(numToProduce, consumer, lastTry),
TimeSpan.FromSeconds(30));
await consumer.StopConsuming();
// This will make sure next produce will kick the messages away
await Task.Delay(TimeSpan.FromSeconds(6));
// Getting the old token
var tokens = await consumer.GetReceivedTokens();
var wantedToken = tokens[numToProduce - 2];
// Prodcing the new message
int secnodNumToProduce = 2;
for (int i = 1; i <= secnodNumToProduce; i++)
{
await firstProducer.Produce();
}
// We have to wait a bit for the pulling agent to get the new messages and throw the old ones.
await Task.Delay(TimeSpan.FromSeconds(1));
await
consumer.BecomeConsumer(firstStream, streamNamespace, wantedToken, TimeSpan.Zero, _streamProviderName);
await TestingUtils.WaitUntilAsync(lastTry => CheckCounters(numToProduce + secnodNumToProduce, consumer, lastTry), TimeSpan.FromSeconds(30));
}
public async Task StreamingTests_Consumer_Producer_UnSubscribe()
{
var subscriptionManager = new SubscriptionManager(this.fixture.HostedCluster);
var streamId = new FullStreamIdentity(Guid.NewGuid(), "EmptySpace", StreamProviderName);
//set up subscription for consumer grains
var subscriptions = await subscriptionManager.SetupStreamingSubscriptionForStream <IPassive_ConsumerGrain>(streamId, 2);
var producer = this.fixture.GrainFactory.GetGrain <ITypedProducerGrainProducingApple>(Guid.NewGuid());
await producer.BecomeProducer(streamId.Guid, streamId.Namespace, streamId.ProviderName);
await producer.StartPeriodicProducing();
int numProduced = 0;
await TestingUtils.WaitUntilAsync(lastTry => ProducerHasProducedSinceLastCheck(numProduced, producer, lastTry), _timeout);
//the subscription to remove
var subscription = subscriptions[0];
// remove subscription
await subscriptionManager.RemoveSubscription(streamId, subscription.SubscriptionId);
var numProducedWhenUnSub = await producer.GetNumberProduced();
var consumerUnSub = this.fixture.GrainFactory.GetGrain <IPassive_ConsumerGrain>(subscription.GrainId.PrimaryKey);
var consumerNormal = this.fixture.GrainFactory.GetGrain <IPassive_ConsumerGrain>(subscriptions[1].GrainId.PrimaryKey);
//assert consumer grain's onAdd func got called.
Assert.True((await consumerUnSub.GetCountOfOnAddFuncCalled()) > 0);
await TestingUtils.WaitUntilAsync(lastTry => ProducerHasProducedSinceLastCheck(numProducedWhenUnSub, producer, lastTry), _timeout);
await producer.StopPeriodicProducing();
//wait for consumers to finish consuming
await Task.Delay(TimeSpan.FromMilliseconds(2000));
//assert normal consumer consumed equal to produced
await TestingUtils.WaitUntilAsync(
lastTry => CheckCounters(new List <ITypedProducerGrain> {
producer
}, consumerNormal, lastTry, this.fixture.Logger), _timeout);
//asert unsubscribed consumer consumed less than produced
numProduced = await producer.GetNumberProduced();
var numConsumed = await consumerUnSub.GetNumberConsumed();
Assert.True(numConsumed <= numProducedWhenUnSub);
Assert.True(numConsumed < numProduced);
// clean up test
await consumerNormal.StopConsuming();
await consumerUnSub.StopConsuming();
}
public async Task CanDeserializeJsonMessage()
{
var subscriber = GrainFactory.GetGrain <IStreamSubscriberGrain <Dictionary <string, string> > >(Guid.Empty);
await subscriber.SubscribeTo(Guid.Empty, "$stats-127.0.0.1:2113", ProviderName);
await TestingUtils.WaitUntilAsync(lastTry => subscriber.HasReceivedMessage(), TimeSpan.FromSeconds(30));
var msg = await subscriber.ReceivedMessage();
Assert.IsAssignableFrom <Dictionary <string, string> >(msg);
Assert.True(msg.ContainsKey("proc-id"));
}
public async Task MultipleProducersSameGrainDifferentNamespaceDifferentConsumers()
{
// get producer and consumer
var producer = GrainClient.GrainFactory.GetGrain <IMultipleProducingProducerGrain>(Guid.NewGuid());
var firstConsumer = GrainClient.GrainFactory.GetGrain <IMultipleSubscriptionConsumerGrain>(Guid.NewGuid());
var secondConsumer = GrainClient.GrainFactory.GetGrain <IMultipleSubscriptionConsumerGrain>(Guid.NewGuid());
Guid firstStream = Guid.NewGuid();
Guid secondStream = Guid.NewGuid();
var streamFirstNamespace = "FirstMultipleProducersSameGrainSameNamespaceDifferentConsumers";
var streamSecondNamespace = "SecondMultipleProducersSameGrainSameNamespaceDifferentConsumers";
// subscribing to different streams on the same namespace
StreamSubscriptionHandle <int> firstHandle = await firstConsumer.BecomeConsumer(firstStream, streamFirstNamespace, _streamProviderName);
StreamSubscriptionHandle <int> secondHandle = await secondConsumer.BecomeConsumer(secondStream, streamSecondNamespace, _streamProviderName);
// Becoming a producer
var firstKey = await producer.BecomeProducer(firstStream, streamFirstNamespace, _streamProviderName);
var secondKey = await producer.BecomeProducer(secondStream, streamSecondNamespace, _streamProviderName);
// Starting to produce!
await producer.StartPeriodicProducing(firstKey);
await producer.StartPeriodicProducing(secondKey);
// Waiting a bit..
await Task.Delay(TimeSpan.FromMilliseconds(3000));
// Stopping
await producer.StopPeriodicProducing(firstKey);
await producer.StopPeriodicProducing(secondKey);
var numProduced = await producer.GetNumberProduced();
_logger.Info("Producer number 1 produced {0} messages", numProduced.Values.First());
_logger.Info("Producer number 2 produced {0} messages", numProduced.Values.Last());
// Checking
await
Task.WhenAll(
TestingUtils.WaitUntilAsync(lastTry => CheckCounters(producer, firstConsumer, 2, 1, lastTry), Timeout),
TestingUtils.WaitUntilAsync(lastTry => CheckCounters(producer, secondConsumer, 2, 1, lastTry), Timeout));
// unsubscribe
await firstConsumer.StopConsuming(firstHandle);
await secondConsumer.StopConsuming(secondHandle);
}
public async Task CacheIsUnderPressreDueToLateSubscriber()
{
// get producer and consumer
var firstProducer = GrainClient.GrainFactory.GetGrain <ISampleStreaming_ProducerGrain>(Guid.NewGuid());
var regularConsumer = GrainClient.GrainFactory.GetGrain <ITimedConsumerGrain>(Guid.NewGuid());
var slowConsumer = GrainClient.GrainFactory.GetGrain <ITimedConsumerGrain>(Guid.NewGuid());
Guid firstStream = Guid.NewGuid();
string streamNamespace = "RewindingTestNamespace";
// subscribing to different streams on the same namespace
await regularConsumer.BecomeConsumer(firstStream, streamNamespace, null, TimeSpan.Zero, _streamProviderName);
// time to produce
await firstProducer.BecomeProducer(firstStream, streamNamespace, _streamProviderName);
// Starting to produce!
int numToProduce = 1;
for (int i = 1; i <= numToProduce; i++)
{
await firstProducer.Produce();
}
await
TestingUtils.WaitUntilAsync(lastTry => CheckCounters(numToProduce, regularConsumer, lastTry),
TimeSpan.FromSeconds(30));
var wantedToken = (await regularConsumer.GetReceivedTokens()).Values.First();
int secondNumToProduce = 4;
for (int i = 1; i <= secondNumToProduce; i++)
{
await firstProducer.Produce();
}
await slowConsumer.BecomeConsumer(firstStream, streamNamespace, wantedToken, TimeSpan.FromSeconds(2), _streamProviderName);
int thirdNumToProduce = 5;
for (int i = 1; i <= thirdNumToProduce; i++)
{
await firstProducer.Produce();
}
await
Task.WhenAll(TestingUtils.WaitUntilAsync(lastTry => CheckCounters(numToProduce + secondNumToProduce + thirdNumToProduce, regularConsumer, lastTry), TimeSpan.FromSeconds(30)),
TestingUtils.WaitUntilAsync(lastTry => CheckCounters(numToProduce + secondNumToProduce + thirdNumToProduce, slowConsumer, lastTry), TimeSpan.FromSeconds(30)));
await Task.WhenAll(slowConsumer.StopConsuming(), regularConsumer.StopConsuming());
}
//For distributedTM
public virtual async Task TransactionWillRecoverAfterRandomSiloFailure(TransactionTestConstants.TransactionGrainStates transactionTestGrainClassName)
{
const int grainCount = 100;
var txGrains = Enumerable.Range(0, grainCount)
.Select(i => RandomTestGrain(transactionTestGrainClassName))
.ToList();
var txSucceedBeforeInterruption = await AllTxSucceed(txGrains);
this.logger.LogInformation($"Tx succeed before interruption : {txSucceedBeforeInterruption}");
//randomly ungraceful shut down one silo
this.testCluster.KillSilo(this.testCluster.Silos[this.seed.Next(this.testCluster.Silos.Count)]);
await TestingUtils.WaitUntilAsync(lastTry => CheckTxResult(txGrains, lastTry), RecoveryTimeout);
}
public async Task StreamingTests_Consumer_Producer_SubscribeToStreamsHandledByDifferentStreamProvider()
{
var subscriptionManager = new SubscriptionManager(this.fixture.HostedCluster);
var streamId = new FullStreamIdentity(Guid.NewGuid(), "EmptySpace", StreamProviderName);
//set up subscription for 10 consumer grains
var subscriptions = await subscriptionManager.SetupStreamingSubscriptionForStream <IPassive_ConsumerGrain>(streamId, 10);
var consumers = subscriptions.Select(sub => this.fixture.GrainFactory.GetGrain <IPassive_ConsumerGrain>(sub.GrainId.PrimaryKey)).ToList();
var producer = this.fixture.GrainFactory.GetGrain <ITypedProducerGrainProducingApple>(Guid.NewGuid());
await producer.BecomeProducer(streamId.Guid, streamId.Namespace, streamId.ProviderName);
await producer.StartPeriodicProducing();
int numProduced = 0;
await TestingUtils.WaitUntilAsync(lastTry => ProducerHasProducedSinceLastCheck(numProduced, producer, lastTry), _timeout);
// set up the new stream to subscribe, which produce strings
var streamId2 = new FullStreamIdentity(Guid.NewGuid(), "EmptySpace2", StreamProviderName2);
var producer2 = this.fixture.GrainFactory.GetGrain <ITypedProducerGrainProducingApple>(Guid.NewGuid());
await producer2.BecomeProducer(streamId2.Guid, streamId2.Namespace, streamId2.ProviderName);
//register the consumer grain to second stream
var tasks = consumers.Select(consumer => subscriptionManager.AddSubscription <IPassive_ConsumerGrain>(streamId2, consumer.GetPrimaryKey())).ToList();
await Task.WhenAll(tasks);
await producer2.StartPeriodicProducing();
await TestingUtils.WaitUntilAsync(lastTry => ProducerHasProducedSinceLastCheck(numProduced, producer2, lastTry), _timeout);
await producer.StopPeriodicProducing();
await producer2.StopPeriodicProducing();
var tasks2 = new List <Task>();
foreach (var consumer in consumers)
{
tasks2.Add(TestingUtils.WaitUntilAsync(lastTry => CheckCounters(new List <ITypedProducerGrain> {
producer, producer2
},
consumer, lastTry, this.fixture.Logger), _timeout));
}
await Task.WhenAll(tasks);
//clean up test
tasks2.Clear();
tasks2 = consumers.Select(consumer => consumer.StopConsuming()).ToList();
await Task.WhenAll(tasks2);
}
public async Task Recoverable100EventStreamsWithTransientErrors(Func <string, int, int, Task> generateFn, string streamNamespace, int streamCount, int eventsInStream)
{
try
{
await generateFn(streamNamespace, streamCount, eventsInStream);
await TestingUtils.WaitUntilAsync(assertIsTrue => this.CheckCounters(streamNamespace, streamCount, eventsInStream, assertIsTrue), TimeSpan.FromSeconds(30));
}
finally
{
var reporter = this.grainFactory.GetGrain <IGeneratedEventReporterGrain>(GeneratedStreamTestConstants.ReporterId);
reporter.Reset().Ignore();
}
}
public async Task TwoIntermitentStreamTest(Guid streamGuid)
{
const string streamNamespace1 = "streamNamespace1";
const string streamNamespace2 = "streamNamespace2";
// send events on first stream /////////////////////////////
var producer = GrainClient.GrainFactory.GetGrain <ISampleStreaming_ProducerGrain>(Guid.NewGuid());
var consumer = GrainClient.GrainFactory.GetGrain <IMultipleSubscriptionConsumerGrain>(Guid.NewGuid());
await producer.BecomeProducer(streamGuid, streamNamespace1, streamProviderName);
StreamSubscriptionHandle <int> handle = await consumer.BecomeConsumer(streamGuid, streamNamespace1, streamProviderName);
await producer.StartPeriodicProducing();
await Task.Delay(TimeSpan.FromMilliseconds(1000));
await producer.StopPeriodicProducing();
await TestingUtils.WaitUntilAsync(lastTry => CheckCounters(producer, consumer, 1, lastTry), Timeout);
// send some events on second stream /////////////////////////////
var producer2 = GrainClient.GrainFactory.GetGrain <ISampleStreaming_ProducerGrain>(Guid.NewGuid());
var consumer2 = GrainClient.GrainFactory.GetGrain <IMultipleSubscriptionConsumerGrain>(Guid.NewGuid());
await producer2.BecomeProducer(streamGuid, streamNamespace2, streamProviderName);
StreamSubscriptionHandle <int> handle2 = await consumer2.BecomeConsumer(streamGuid, streamNamespace2, streamProviderName);
await producer2.StartPeriodicProducing();
await Task.Delay(TimeSpan.FromMilliseconds(1000));
await producer2.StopPeriodicProducing();
await TestingUtils.WaitUntilAsync(lastTry => CheckCounters(producer2, consumer2, 1, lastTry), Timeout);
// send some events on first stream again /////////////////////////////
await producer.StartPeriodicProducing();
await Task.Delay(TimeSpan.FromMilliseconds(1000));
await producer.StopPeriodicProducing();
await TestingUtils.WaitUntilAsync(lastTry => CheckCounters(producer, consumer, 1, lastTry), Timeout);
await consumer.StopConsuming(handle);
await consumer2.StopConsuming(handle2);
}
private async Task ProduceEventsFromClient(string streamProviderName, Guid streamGuid, string streamNamespace, int eventsProduced)
{
// get reference to a consumer
var consumer = this.testHost.GrainFactory.GetGrain <ISampleStreaming_ConsumerGrain>(Guid.NewGuid());
// subscribe
await consumer.BecomeConsumer(streamGuid, streamNamespace, streamProviderName);
// generate events
await GenerateEvents(streamProviderName, streamGuid, streamNamespace, eventsProduced);
// make sure all went well
await TestingUtils.WaitUntilAsync(lastTry => CheckCounters(() => consumer.GetNumberConsumed(), () => Task.FromResult(eventsProduced), lastTry), _timeout);
}
public async Task Recoverable100EventStreamsWith1NonTransientError(Func <string, int, int, Task> generateFn, string streamNamespace, int streamCount, int eventsInStream)
{
try
{
await generateFn(streamNamespace, streamCount, eventsInStream);
// should eventually skip the faulted event, so event count should be one (faulted event) less that number of events in stream.
await TestingUtils.WaitUntilAsync(assertIsTrue => this.CheckCounters(streamNamespace, streamCount, eventsInStream - 1, assertIsTrue), TimeSpan.FromSeconds(90));
}
finally
{
var reporter = this.grainFactory.GetGrain <IGeneratedEventReporterGrain>(GeneratedStreamTestConstants.ReporterId);
reporter.Reset().Ignore();
}
}
public async Task SingleSendBatchConsume()
{
const int ExpectedConsumed = 30;
Guid streamGuid = Guid.NewGuid();
IStreamProvider provider = this.fixture.Client.GetStreamProvider(StreamBatchingTestConst.ProviderName);
IAsyncStream <string> stream = provider.GetStream <string>(streamGuid, StreamBatchingTestConst.BatchingNameSpace);
for (int i = 0; i < ExpectedConsumed; i++)
{
await stream.OnNextAsync(i.ToString());
}
var consumer = this.fixture.GrainFactory.GetGrain <IStreamBatchingTestConsumerGrain>(streamGuid);
await TestingUtils.WaitUntilAsync(lastTry => CheckCounters(consumer, ExpectedConsumed, 2, lastTry), Timeout);
}
public async Task EHStatistics_MonitorCalledAccordingly()
{
var streamId = new FullStreamIdentity(Guid.NewGuid(), StreamNamespace, StreamProviderName);
//set up one slow consumer grain
var slowConsumer = this.fixture.GrainFactory.GetGrain <ISlowConsumingGrain>(Guid.NewGuid());
await slowConsumer.BecomeConsumer(streamId.Guid, StreamNamespace, StreamProviderName);
//set up 30 healthy consumer grain to show how much we favor slow consumer
int healthyConsumerCount = 30;
var healthyConsumers = await EHSlowConsumingTests.SetUpHealthyConsumerGrain(this.fixture.GrainFactory, streamId.Guid, StreamNamespace, StreamProviderName, healthyConsumerCount);
//configure data generator for stream and start producing
var mgmtGrain = this.fixture.GrainFactory.GetGrain <IManagementGrain>(0);
var randomStreamPlacementArg = new EventDataGeneratorStreamProvider.AdapterFactory.StreamRandomPlacementArg(streamId, this.seed.Next(100));
await mgmtGrain.SendControlCommandToProvider(typeof(EHStreamProviderForMonitorTests).FullName, StreamProviderName,
(int)EventDataGeneratorStreamProvider.AdapterFactory.Commands.Randomly_Place_Stream_To_Queue, randomStreamPlacementArg);
//since there's an extreme slow consumer, so the back pressure algorithm should be triggered
await TestingUtils.WaitUntilAsync(lastTry => AssertCacheBackPressureTriggered(true, lastTry), timeout);
//make slow consumer stop consuming
await slowConsumer.StopConsuming();
//assert EventHubReceiverMonitor call counters
var receiverMonitorCounters = await mgmtGrain.SendControlCommandToProvider(typeof(EHStreamProviderForMonitorTests).FullName, StreamProviderName,
(int)EHStreamProviderForMonitorTests.AdapterFactory.QueryCommands.GetReceiverMonitorCallCounters, null);
foreach (var callCounter in receiverMonitorCounters)
{
AssertReceiverMonitorCallCounters(callCounter as EventHubReceiverMonitorCounters);
}
var cacheMonitorCounters = await mgmtGrain.SendControlCommandToProvider(typeof(EHStreamProviderForMonitorTests).FullName, StreamProviderName,
(int)EHStreamProviderForMonitorTests.AdapterFactory.QueryCommands.GetCacheMonitorCallCounters, null);
foreach (var callCounter in cacheMonitorCounters)
{
AssertCacheMonitorCallCounters(callCounter as CacheMonitorCounters);
}
var objectPoolMonitorCounters = await mgmtGrain.SendControlCommandToProvider(typeof(EHStreamProviderForMonitorTests).FullName, StreamProviderName,
(int)EHStreamProviderForMonitorTests.AdapterFactory.QueryCommands.GetObjectPoolMonitorCallCounters, null);
foreach (var callCounter in objectPoolMonitorCounters)
{
AssertObjectPoolMonitorCallCounters(callCounter as ObjectPoolMonitorCounters);
}
}
public async Task BatchTestMultipleProducers(int numProducers, int batchSize)
{
List <IBatchProducerGrain> producersList = new List <IBatchProducerGrain>(numProducers);
for (int i = 0; i < numProducers; i++)
{
producersList.Add(GrainClient.GrainFactory.GetGrain <IBatchProducerGrain>(Guid.NewGuid()));
}
var consumer = GrainClient.GrainFactory.GetGrain <ISampleStreaming_ConsumerGrain>(Guid.NewGuid());
var streamGuid = Guid.NewGuid();
var streamNamespace = "BatchTestStream";
await consumer.BecomeConsumer(streamGuid, streamNamespace, _streamProviderName);
foreach (var producer in producersList)
{
await producer.BecomeProducer(streamGuid, streamNamespace, _streamProviderName);
}
foreach (var producer in producersList)
{
await producer.StartPeriodicBatchProducing(batchSize);
}
_logger.Info("***************Starting to Produce!*****************");
await Task.Delay(TimeSpan.FromSeconds(1));
var sum = 0;
foreach (var producer in producersList)
{
await producer.StopPeriodicBatchProducing();
_logger.Info("Producer produced {0} messages", await producer.GetNumberProduced());
var numProduced = await producer.GetNumberProduced();
sum += numProduced;
}
await TestingUtils.WaitUntilAsync(lastTry => CheckCounter(sum, consumer, lastTry), Timeout);
await consumer.StopConsuming();
}
public async Task BatchSendBatchConsume()
{
const int BatchesSent = 3;
const int ItemsPerBatch = 10;
const int ExpectedConsumed = BatchesSent * ItemsPerBatch;
Guid streamGuid = Guid.NewGuid();
IStreamProvider provider = this.fixture.Client.GetStreamProvider(StreamBatchingTestConst.ProviderName);
IAsyncStream <string> stream = provider.GetStream <string>(streamGuid, StreamBatchingTestConst.BatchingNameSpace);
for (int i = 0; i < BatchesSent; i++)
{
await stream.OnNextBatchAsync(Enumerable.Range(i, ItemsPerBatch).Select(v => v.ToString()));
}
var consumer = this.fixture.GrainFactory.GetGrain <IStreamBatchingTestConsumerGrain>(streamGuid);
await TestingUtils.WaitUntilAsync(lastTry => CheckCounters(consumer, ExpectedConsumed, ItemsPerBatch, lastTry), Timeout);
}
private async Task ProducerConsumerTest(Guid producerGuid, Guid consumerGuid)
{
// producer joins first, consumer later
IProducerEventCountingGrain producer = this.fixture.GrainFactory.GetGrain <IProducerEventCountingGrain>(producerGuid);
await producer.BecomeProducer(_streamId, _streamProvider);
IConsumerEventCountingGrain consumer = this.fixture.GrainFactory.GetGrain <IConsumerEventCountingGrain>(consumerGuid);
await consumer.BecomeConsumer(_streamId, _streamProvider);
await producer.SendEvent();
await Task.Delay(1000);
await TestingUtils.WaitUntilAsync(lastTry => CheckCounters(producer, consumer), Timeout);
await consumer.StopConsuming();
}
public async Task LeaseBalancedQueueBalancer_SupportAutoScaleScenario()
{
var mgmtGrain = this.GrainFactory.GetGrain <IManagementGrain>(0);
//6 queue and 4 silo, then each agent manager should own queues/agents in range of [1, 2]
await TestingUtils.WaitUntilAsync(lastTry => AgentManagerOwnCorrectAmountOfAgents(1, 2, mgmtGrain, lastTry), TimeOut);
//stop one silo, 6 queues, 3 silo, then each agent manager should own 2 queues
this.HostedCluster.StopSilo(this.HostedCluster.SecondarySilos[0]);
await TestingUtils.WaitUntilAsync(lastTry => AgentManagerOwnCorrectAmountOfAgents(2, 2, mgmtGrain, lastTry), TimeOut);
//stop another silo, 6 queues, 2 silo, then each agent manager should own 3 queues
this.HostedCluster.StopSilo(this.HostedCluster.SecondarySilos[0]);
await TestingUtils.WaitUntilAsync(lastTry => AgentManagerOwnCorrectAmountOfAgents(3, 3, mgmtGrain, lastTry), TimeOut);
//start one silo, 6 queues, 3 silo, then each agent manager should own 2 queues
this.HostedCluster.StartAdditionalSilo();
await TestingUtils.WaitUntilAsync(lastTry => AgentManagerOwnCorrectAmountOfAgents(2, 2, mgmtGrain, lastTry), TimeOut);
}
public async Task ResubscriptionAfterDeactivationTest(Guid streamGuid, string streamNamespace)
{
// get producer and consumer
var producer = GrainClient.GrainFactory.GetGrain <ISampleStreaming_ProducerGrain>(Guid.NewGuid());
var consumer = GrainClient.GrainFactory.GetGrain <IMultipleSubscriptionConsumerGrain>(Guid.NewGuid());
await producer.BecomeProducer(streamGuid, streamNamespace, streamProviderName);
// setup one subscription and send messsages
StreamSubscriptionHandle <int> firstSubscriptionHandle = await consumer.BecomeConsumer(streamGuid, streamNamespace, streamProviderName);
await producer.StartPeriodicProducing();
await Task.Delay(TimeSpan.FromMilliseconds(1000));
await producer.StopPeriodicProducing();
await TestingUtils.WaitUntilAsync(lastTry => CheckCounters(producer, consumer, 1, lastTry), Timeout);
// Deactivate grain
await consumer.Deactivate();
// make sure grain has time to deactivate.
await Task.Delay(TimeSpan.FromMilliseconds(100));
// clear producer counts
await producer.ClearNumberProduced();
// Resume
StreamSubscriptionHandle <int> resumeHandle = await consumer.Resume(firstSubscriptionHandle);
Assert.AreEqual(firstSubscriptionHandle, resumeHandle, "Handle matches");
await producer.StartPeriodicProducing();
await Task.Delay(TimeSpan.FromMilliseconds(1000));
await producer.StopPeriodicProducing();
await TestingUtils.WaitUntilAsync(lastTry => CheckCounters(producer, consumer, 1, lastTry), Timeout);
// remove subscription
await consumer.StopConsuming(resumeHandle);
}
public async Task SubscribingWithOldSequenceTokenAfterBatchSending()
{
// get producer and consumer
var firstProducer = GrainClient.GrainFactory.GetGrain <IBatchProducerGrain>(Guid.NewGuid());
var consumer = GrainClient.GrainFactory.GetGrain <ITimedConsumerGrain>(Guid.NewGuid());
Guid firstStream = Guid.NewGuid();
string streamNamespace = "RewindingTestNamespace";
// subscribing to different streams on the same namespace
await consumer.BecomeConsumer(firstStream, streamNamespace, null, TimeSpan.Zero, _streamProviderName);
// time to produce
await firstProducer.BecomeProducer(firstStream, streamNamespace, _streamProviderName);
// Starting to produce!
int numToProduce = 1;
int batchToProduce = 5;
for (int i = 1; i <= numToProduce; i++)
{
await firstProducer.Produce(batchToProduce);
}
await
TestingUtils.WaitUntilAsync(lastTry => CheckCounters(numToProduce *batchToProduce, consumer, lastTry),
TimeSpan.FromSeconds(30));
await consumer.StopConsuming();
// Resubscribing with token
var tokens = await consumer.GetReceivedTokens();
var wantedToken = tokens[batchToProduce - 2];
// Producing just one more tiny message
await firstProducer.Produce(1);
await
consumer.BecomeConsumer(firstStream, streamNamespace, wantedToken, TimeSpan.Zero, _streamProviderName);
await TestingUtils.WaitUntilAsync(lastTry => CheckCounters((numToProduce * batchToProduce * 2) + 1, consumer, lastTry), TimeSpan.FromSeconds(30));
}
private async Task StreamingTests_Consumer_Producer(Guid streamId, string streamProvider)
{
// consumer joins first, producer later
ISampleStreaming_ConsumerGrain consumer = SampleStreaming_ConsumerGrainFactory.GetGrain(Guid.NewGuid());
await consumer.BecomeConsumer(streamId, StreamNamespace, streamProvider);
ISampleStreaming_ProducerGrain producer = SampleStreaming_ProducerGrainFactory.GetGrain(Guid.NewGuid());
await producer.BecomeProducer(streamId, StreamNamespace, streamProvider);
await producer.StartPeriodicProducing();
Thread.Sleep(1000);
await producer.StopPeriodicProducing();
await TestingUtils.WaitUntilAsync(lastTry => CheckCounters(producer, consumer, lastTry), _timeout);
await consumer.StopConsuming();
}
private async Task ReloadFromCheckpointTestRunner(string streamNamespace, int streamCount, int eventsInStream)
{
List<Guid> streamGuids = Enumerable.Range(0, streamCount).Select(_ => Guid.NewGuid()).ToList();
try
{
await GenerateEvents(streamNamespace, streamGuids, eventsInStream, 4096);
await TestingUtils.WaitUntilAsync(assertIsTrue => CheckCounters(streamNamespace, streamCount, eventsInStream, assertIsTrue), TimeSpan.FromSeconds(60));
await RestartAgents();
await GenerateEvents(streamNamespace, streamGuids, eventsInStream, 4096);
await TestingUtils.WaitUntilAsync(assertIsTrue => CheckCounters(streamNamespace, streamCount, eventsInStream * 2, assertIsTrue), TimeSpan.FromSeconds(90));
}
finally
{
var reporter = this.GrainFactory.GetGrain<IGeneratedEventReporterGrain>(GeneratedStreamTestConstants.ReporterId);
reporter.Reset().Ignore();
}
}
//For singleTM , only in singleTM we have the need of test two scenarios, whic is kill silo which runs the TM and ramdom silo which doesn't
public virtual async Task TransactionWillRecoverAfterRandomSiloFailure(string transactionTestGrainClassName, bool killSiloWhichRunsTm)
{
const int grainCount = 100;
var txGrains = Enumerable.Range(0, grainCount)
.Select(i => RandomTestGrain(transactionTestGrainClassName))
.ToList();
var txSucceedBeforeInterruption = await AllTxSucceed(txGrains);
this.logger.LogInformation($"Tx succeed before interruption : {txSucceedBeforeInterruption}");
if (killSiloWhichRunsTm)
{
await KillSilo(location => location.ActiveInCurrentSilo);
}
else
{
await KillSilo(location => !location.ActiveInCurrentSilo);
}
await TestingUtils.WaitUntilAsync(lastTry => CheckTxResult(txGrains, lastTry), RecoveryTimeout);
}
protected virtual async Task TransactionWillRecoverAfterRandomSiloFailure(string transactionTestGrainClassName, bool gracefulShutdown)
{
const int grainCount = 100;
var txGrains = Enumerable.Range(0, grainCount)
.Select(i => RandomTestGrain(transactionTestGrainClassName))
.ToList();
var txSucceedBeforeInterruption = await AllTxSucceed(txGrains);
this.logger.LogInformation($"Tx succeed before interruption : {txSucceedBeforeInterruption}");
if (gracefulShutdown)
{
this.testCluster.StopSilo(this.testCluster.Silos[this.seed.Next(this.testCluster.Silos.Count)]);
}
else
{
this.testCluster.KillSilo(this.testCluster.Silos[this.seed.Next(this.testCluster.Silos.Count)]);
}
await TestingUtils.WaitUntilAsync(lastTry => CheckTxResult(txGrains, lastTry), RecoveryTimeout);
}
public async Task StreamingTests_Consumer_Producer(Guid streamId)
{
// consumer joins first, producer later
var consumer = GrainClient.GrainFactory.GetGrain <ISampleStreaming_ConsumerGrain>(Guid.NewGuid());
await consumer.BecomeConsumer(streamId, StreamNamespace, streamProvider);
var producer = GrainClient.GrainFactory.GetGrain <ISampleStreaming_ProducerGrain>(Guid.NewGuid());
await producer.BecomeProducer(streamId, StreamNamespace, streamProvider);
await producer.StartPeriodicProducing();
await Task.Delay(TimeSpan.FromMilliseconds(1000));
await producer.StopPeriodicProducing();
await TestingUtils.WaitUntilAsync(lastTry => CheckCounters(producer, consumer, lastTry), _timeout);
await consumer.StopConsuming();
}
public async Task BatchTestLoadTestingWithImplicitSubscription()
{
var guid = Guid.NewGuid();
var producer = GrainClient.GrainFactory.GetGrain <IBatchProducerGrain>(guid);
var streamGuid = Guid.NewGuid();
await producer.BecomeProducer(streamGuid, ImplicitConsumerGrain.StreamNamespace, _streamProviderName);
var batchSize = 10;
await producer.StartPeriodicBatchProducing(batchSize);
await Task.Delay(TimeSpan.FromSeconds(3));
await producer.StopPeriodicBatchProducing();
_logger.Info("Producer produced {0} messages", await producer.GetNumberProduced());
var consumer = GrainClient.GrainFactory.GetGrain <IImplicitConsumerGrain>(streamGuid);
await TestingUtils.WaitUntilAsync(lastTry => CheckCounter(producer, consumer, lastTry), Timeout);
}
private async Task StreamingTests_Producer_InlineConsumer(Guid streamId, string streamProvider)
{
// producer joins first, consumer later
ISampleStreaming_ProducerGrain producer = SampleStreaming_ProducerGrainFactory.GetGrain(Guid.NewGuid());
await producer.BecomeProducer(streamId, StreamNamespace, streamProvider);
ISampleStreaming_InlineConsumerGrain consumer = SampleStreaming_InlineConsumerGrainFactory.GetGrain(Guid.NewGuid());
await consumer.BecomeConsumer(streamId, StreamNamespace, streamProvider);
await producer.StartPeriodicProducing();
await Task.Delay(TimeSpan.FromMilliseconds(1000));
await producer.StopPeriodicProducing();
//int numProduced = await producer.NumberProduced;
await TestingUtils.WaitUntilAsync(lastTry => CheckCounters(producer, consumer, lastTry), _timeout);
await consumer.StopConsuming();
}
protected virtual async Task TransactionWillRecoverAfterRandomSiloFailure(string transactionTestGrainClassName, bool gracefulShutdown)
{
const int grainCount = 100;
var index = new[] { 0 };
Func <int> getIndex = () => index[0]++;
List <ExpectedGrainActivity> txGrains = Enumerable.Range(0, grainCount)
.Select(i => new ExpectedGrainActivity {
Grain = RandomTestGrain <ITransactionalBitArrayGrain>(transactionTestGrainClassName)
})
.ToList();
var txSucceedBeforeInterruption = await AllTxSucceed(txGrains, getIndex());
Assert.True(txSucceedBeforeInterruption);
await ValidateResults(txGrains);
// have transactions in flight when silo goes down
Task succeeding = RunWhileSucceeding(txGrains, getIndex);
var siloToTerminate = this.testCluster.Silos[this.random.Next(this.testCluster.Silos.Count)];
this.Log($"Warmup transaction succeeded. {(gracefulShutdown ? "Stopping" : "Killing")} silo {siloToTerminate.SiloAddress} ({siloToTerminate.Name}) and continuing");
if (gracefulShutdown)
{
this.testCluster.StopSilo(siloToTerminate);
}
else
{
this.testCluster.KillSilo(siloToTerminate);
}
this.Log("Waiting for transactions to stop completing successfully");
await succeeding;
this.Log($"Waiting for system to recover. Performed {index[0]} transactions on each group.");
await TestingUtils.WaitUntilAsync(lastTry => CheckTxResult(txGrains, getIndex, lastTry), RecoveryTimeout);
this.Log($"Recovery completed. Performed {index[0]} transactions on each group. Validating results.");
await ValidateResults(txGrains);
}
public async Task MultipleStreamsSameNamespaceTest(string streamNamespace)
{
// get producer and consumer
var firstProducer = GrainClient.GrainFactory.GetGrain <ISampleStreaming_ProducerGrain>(Guid.NewGuid());
var secondProducer = GrainClient.GrainFactory.GetGrain <ISampleStreaming_ProducerGrain>(Guid.NewGuid());
var consumer = GrainClient.GrainFactory.GetGrain <IMultipleSubscriptionConsumerGrain>(Guid.NewGuid());
Guid firstStream = Guid.NewGuid();
Guid secondStream = Guid.NewGuid();
// subscribing to different streams on the same namespace
StreamSubscriptionHandle <int> firstHandle = await consumer.BecomeConsumer(firstStream, streamNamespace, _streamProviderName);
StreamSubscriptionHandle <int> secondHandle = await consumer.BecomeConsumer(secondStream, streamNamespace, _streamProviderName);
// time to produce
await firstProducer.BecomeProducer(firstStream, streamNamespace, _streamProviderName);
await secondProducer.BecomeProducer(secondStream, streamNamespace, _streamProviderName);
// Starting to produce!
await Task.WhenAll(firstProducer.StartPeriodicProducing(), secondProducer.StartPeriodicProducing());
// Waiting a bit..
await Task.Delay(TimeSpan.FromMilliseconds(1000));
// Stopping
await Task.WhenAll(firstProducer.StopPeriodicProducing(), secondProducer.StopPeriodicProducing());
// Checking
await
Task.WhenAll(
TestingUtils.WaitUntilAsync(lastTry => CheckCounters(firstProducer, consumer, 2, firstHandle, lastTry), Timeout),
TestingUtils.WaitUntilAsync(lastTry => CheckCounters(secondProducer, consumer, 2, secondHandle, lastTry), Timeout));
// unsubscribe
await consumer.StopConsuming(firstHandle);
await consumer.StopConsuming(secondHandle);
}
public async Task Programmatic_Subscribe_UsingClientSideSubscriptionManager_UsingDynamicProviderConfig()
{
var streamId = new FullStreamIdentity(Guid.NewGuid(), "EmptySpace", StreamProviderName);
var subManager = this.Client.ServiceProvider.GetService <IStreamSubscriptionManagerAdmin>()
.GetStreamSubscriptionManager(StreamSubscriptionManagerType.ExplicitSubscribeOnly);
//set up stream subscriptions for grains
var subscriptions = await SetupStreamingSubscriptionForStream <IPassive_ConsumerGrain>(subManager, this.GrainFactory, streamId, 10);
var consumers = subscriptions.Select(sub => this.GrainFactory.GetGrain <IPassive_ConsumerGrain>(sub.GrainId.PrimaryKey)).ToList();
// configure stream provider after subscriptions set up
await AddSimpleStreamProviderAndUpdate(new List <String>() { StreamProviderName });
//set up producer
var producer = this.GrainFactory.GetGrain <ITypedProducerGrainProducingApple>(Guid.NewGuid());
await producer.BecomeProducer(streamId.Guid, streamId.Namespace, streamId.ProviderName);
await producer.StartPeriodicProducing();
int numProduced = 0;
await TestingUtils.WaitUntilAsync(lastTry => ProgrammaticSubcribeTestsRunner.ProducerHasProducedSinceLastCheck(numProduced, producer, lastTry), _timeout);
await producer.StopPeriodicProducing();
var tasks = new List <Task>();
foreach (var consumer in consumers)
{
tasks.Add(TestingUtils.WaitUntilAsync(lastTry => ProgrammaticSubcribeTestsRunner.CheckCounters(new List <ITypedProducerGrain> {
producer
},
consumer, lastTry, this.Logger), _timeout));
}
await Task.WhenAll(tasks);
//clean up test
tasks.Clear();
tasks = consumers.Select(consumer => consumer.StopConsuming()).ToList();
await Task.WhenAll(tasks);
}
