private async Task <double> TestOneStream(Guid streamId, string streamProviderName,
int numProducers, int numConsumers, int numMessages,
bool useFanOut = true)
{
output.WriteLine("Testing Stream {0} with Producers={1} Consumers={2} x {3} messages",
streamId, numProducers, numConsumers, numMessages);
Stopwatch sw = Stopwatch.StartNew();
List <IStreamLifecycleConsumerGrain> consumers = new List <IStreamLifecycleConsumerGrain>();
List <IStreamLifecycleProducerGrain> producers = new List <IStreamLifecycleProducerGrain>();
await InitializeTopology(streamId, this.StreamNamespace, streamProviderName,
numProducers, numConsumers,
producers, consumers, useFanOut);
var promises = new List <Task>();
// Producers send M message each
int item = 1;
AsyncPipeline pipeline = new AsyncPipeline(MessagePipelineSize);
foreach (var grain in producers)
{
for (int m = 0; m < numMessages; m++)
{
Task promise = grain.SendItem(item++);
if (useFanOut)
{
pipeline.Add(promise);
promises.Add(promise);
}
else
{
await promise;
}
}
}
if (useFanOut)
{
//output.WriteLine("Test: Waiting for {0} producers to finish sending {1} messages", producers.Count, promises.Count);
await Task.WhenAll(promises);
promises.Clear();
}
var pubSub = StreamTestUtils.GetStreamPubSub(this.InternalClient);
// Check Consumer counts
int consumerCount = await pubSub.ConsumerCount(streamId, streamProviderName, StreamNamespace);
Assert.Equal(numConsumers, consumerCount); // "ConsumerCount for Stream {0}", streamId
// Check Producer counts
int producerCount = await pubSub.ProducerCount(streamId, streamProviderName, StreamNamespace);
Assert.Equal(numProducers, producerCount); // "ProducerCount for Stream {0}", streamId
// Check message counts received by consumers
int totalMessages = (numMessages + 1) * numProducers;
foreach (var grain in consumers)
{
int count = await grain.GetReceivedCount();
Assert.Equal(totalMessages, count); // "ReceivedCount for Consumer grain {0}", grain.GetPrimaryKey());
}
double rps = totalMessages / sw.Elapsed.TotalSeconds;
//output.WriteLine("Sent {0} messages total from {1} Producers to {2} Consumers in {3} at {4} RPS",
// totalMessages, numProducers, numConsumers,
// sw.Elapsed, rps);
return(rps);
}
// ---------- Test support methods ----------
private async Task DoStreamCleanupTest_Deactivate(bool uncleanShutdown, bool useStreamAfterDeactivate, [CallerMemberName] string testName = null)
{
StreamTestUtils.LogStartTest(testName, StreamId, StreamProviderName, logger, HostedCluster);
var producer1 = GrainClient.GrainFactory.GetGrain <IStreamLifecycleProducerInternalGrain>(Guid.NewGuid());
var producer2 = GrainClient.GrainFactory.GetGrain <IStreamLifecycleProducerInternalGrain>(Guid.NewGuid());
var consumer1 = GrainClient.GrainFactory.GetGrain <IStreamLifecycleConsumerInternalGrain>(Guid.NewGuid());
var consumer2 = GrainClient.GrainFactory.GetGrain <IStreamLifecycleConsumerInternalGrain>(Guid.NewGuid());
await consumer1.BecomeConsumer(StreamId, StreamNamespace, StreamProviderName);
await producer1.BecomeProducer(StreamId, StreamNamespace, StreamProviderName);
await StreamTestUtils.CheckPubSubCounts(output, "after first producer added", 1, 1,
StreamId, StreamProviderName, StreamNamespace);
Assert.AreEqual(1, await producer1.GetSendCount(), "SendCount after first send");
var activations = await watcher.GetActivateCalls();
var deactivations = await watcher.GetDeactivateCalls();
Assert.AreEqual(2, activations.Count(), "Number of activations");
Assert.AreEqual(0, deactivations.Count(), "Number of deactivations");
int expectedNumProducers;
if (uncleanShutdown)
{
expectedNumProducers = 1; // Will not cleanup yet
await producer1.DoBadDeactivateNoClose();
}
else
{
expectedNumProducers = 0; // Should immediately cleanup on Deactivate
await producer1.DoDeactivateNoClose();
}
await WaitForDeactivation();
deactivations = await watcher.GetDeactivateCalls();
Assert.AreEqual(1, deactivations.Count(), "Number of deactivations");
// Test grains that did unclean shutdown will not have cleaned up yet, so PubSub counts are unchanged here for them
await StreamTestUtils.CheckPubSubCounts(output, "after deactivate first producer", expectedNumProducers, 1,
StreamId, StreamProviderName, StreamNamespace);
// Add another consumer - which forces cleanup of dead producers and PubSub counts should now always be correct
await consumer2.BecomeConsumer(StreamId, StreamNamespace, StreamProviderName);
// Runtime should have cleaned up after next consumer added
await StreamTestUtils.CheckPubSubCounts(output, "after add second consumer", 0, 2,
StreamId, StreamProviderName, StreamNamespace);
if (useStreamAfterDeactivate)
{
// Add new producer
await producer2.BecomeProducer(StreamId, StreamNamespace, StreamProviderName);
// These Producer test grains always send first message when they BecomeProducer, so should be registered with PubSub
await StreamTestUtils.CheckPubSubCounts(output, "after add second producer", 1, 2,
StreamId, StreamProviderName, StreamNamespace);
Assert.AreEqual(1, await producer1.GetSendCount(), "SendCount (Producer#1) after second publisher added");
Assert.AreEqual(1, await producer2.GetSendCount(), "SendCount (Producer#2) after second publisher added");
Assert.AreEqual(2, await consumer1.GetReceivedCount(), "ReceivedCount (Consumer#1) after second publisher added");
Assert.AreEqual(1, await consumer2.GetReceivedCount(), "ReceivedCount (Consumer#2) after second publisher added");
await producer2.SendItem(3);
await StreamTestUtils.CheckPubSubCounts(output, "after second producer send", 1, 2,
StreamId, StreamProviderName, StreamNamespace);
Assert.AreEqual(3, await consumer1.GetReceivedCount(), "ReceivedCount (Consumer#1) after second publisher send");
Assert.AreEqual(2, await consumer2.GetReceivedCount(), "ReceivedCount (Consumer#2) after second publisher send");
}
StreamTestUtils.LogEndTest(testName, logger);
}
public async Task Stream_Lifecycle_AddRemoveProducers()
{
string testName = "Stream_Lifecycle_AddRemoveProducers";
StreamTestUtils.LogStartTest(testName, StreamId, StreamProviderName, logger, HostedCluster);
int numProducers = 10;
var consumer = this.GrainFactory.GetGrain <IStreamLifecycleConsumerInternalGrain>(Guid.NewGuid());
await consumer.BecomeConsumer(StreamId, StreamNamespace, StreamProviderName);
var producers = new IStreamLifecycleProducerInternalGrain[numProducers];
for (int i = 1; i <= producers.Length; i++)
{
var producer = this.GrainFactory.GetGrain <IStreamLifecycleProducerInternalGrain>(Guid.NewGuid());
producers[i - 1] = producer;
}
int expectedReceived = 0;
string when = "round 1";
await IncrementalAddProducers(producers, when);
expectedReceived += numProducers;
Assert.Equal(expectedReceived, await consumer.GetReceivedCount());
for (int i = producers.Length; i > 0; i--)
{
var producer = producers[i - 1];
// Force Remove
await producer.TestInternalRemoveProducer(StreamId, StreamProviderName);
await StreamTestUtils.CheckPubSubCounts(this.InternalClient, output, "producer #" + i + " remove", (i - 1), 1,
StreamId, StreamProviderName, StreamNamespace);
}
when = "round 2";
await IncrementalAddProducers(producers, when);
expectedReceived += numProducers;
Assert.Equal(expectedReceived, await consumer.GetReceivedCount());
List <Task> promises = new List <Task>();
for (int i = producers.Length; i > 0; i--)
{
var producer = producers[i - 1];
// Remove when Deactivate
promises.Add(producer.DoDeactivateNoClose());
}
await Task.WhenAll(promises);
await WaitForDeactivation();
await StreamTestUtils.CheckPubSubCounts(this.InternalClient, output, "all producers deactivated", 0, 1,
StreamId, StreamProviderName, StreamNamespace);
when = "round 3";
await IncrementalAddProducers(producers, when);
expectedReceived += numProducers;
Assert.Equal(expectedReceived, await consumer.GetReceivedCount());
}