public async Task AQ_Baseline_StreamRel()
{
// This test case is just a sanity-check that the AzureQueue test config is OK.
const string testName = "AQ_Baseline_StreamRel";
_streamId = Guid.NewGuid();
_streamProviderName = AZURE_QUEUE_STREAM_PROVIDER_NAME;
StreamTestUtils.LogStartTest(testName, _streamId, _streamProviderName, logger, HostedCluster);
long consumerGrainId = random.Next();
long producerGrainId = random.Next();
await Do_BaselineTest(consumerGrainId, producerGrainId);
StreamTestUtils.LogEndTest(testName, logger);
}
public void Baseline_StreamRel_RestartSilos()
{
// This test case is just a sanity-check that the silo test config is OK.
const string testName = "Baseline_StreamRel_RestartSilos";
StreamTestUtils.LogStartTest(testName, _streamId, _streamProviderName, logger, HostedCluster);
CheckSilosRunning("Before Restart", numExpectedSilos);
var silos = this.HostedCluster.Silos;
RestartAllSilos();
CheckSilosRunning("After Restart", numExpectedSilos);
Assert.NotEqual(silos, this.HostedCluster.Silos); // Should be different silos after restart
StreamTestUtils.LogEndTest(testName, logger);
}
public async Task SMS_Baseline_StreamRel()
{
// This test case is just a sanity-check that the SMS test config is OK.
const string testName = "SMS_Baseline_StreamRel";
_streamId = Guid.NewGuid();
_streamProviderName = SMS_STREAM_PROVIDER_NAME;
StreamTestUtils.LogStartTest(testName, _streamId, _streamProviderName, logger);
// Grain Producer -> Grain Consumer
long consumerGrainId = random.Next();
long producerGrainId = random.Next();
await Do_BaselineTest(consumerGrainId, producerGrainId);
StreamTestUtils.LogEndTest(testName, logger);
}
private async Task Test_SiloDies_Consumer(string testName, string streamProviderName)
{
_streamId = Guid.NewGuid();
_streamProviderName = streamProviderName;
string when;
StreamTestUtils.LogStartTest(testName, _streamId, _streamProviderName, logger);
long consumerGrainId = random.Next();
long producerGrainId = random.Next();
var producerGrain = await Do_BaselineTest(consumerGrainId, producerGrainId);
when = "Before kill one silo";
CheckSilosRunning(when, numExpectedSilos);
bool sameSilo = await CheckGrainCounts();
// Find which silo the consumer grain is located on
var consumerGrain = GetGrain(consumerGrainId);
SiloAddress siloAddress = await consumerGrain.GetLocation();
Console.WriteLine("Consumer grain is located on silo {0} ; Producer on same silo = {1}", siloAddress, sameSilo);
// Kill the silo containing the consumer grain
bool isPrimary = siloAddress.Equals(this.HostedCluster.Primary.Silo.SiloAddress);
SiloHandle siloToKill = isPrimary ? this.HostedCluster.Primary : this.HostedCluster.Secondary;
StopSilo(siloToKill, true, false);
// Note: Don't restart failed silo for this test case
// Note: Don't reinitialize client
when = "After kill one silo";
CheckSilosRunning(when, numExpectedSilos - 1);
when = "SendItem";
await producerGrain.SendItem(1);
await CheckConsumerProducerStatus(when, producerGrainId, consumerGrainId, true, true);
StreamTestUtils.LogEndTest(testName, logger);
}
public void Baseline_StreamRel_RestartSilos()
{
// This test case is just a sanity-check that the silo test config is OK.
const string testName = "Baseline_StreamRel_RestartSilos";
StreamTestUtils.LogStartTest(testName, _streamId, _streamProviderName, logger);
CheckSilosRunning("Before Restart", numExpectedSilos);
SiloHandle prim1 = this.HostedCluster.Primary;
SiloHandle sec1 = this.HostedCluster.Secondary;
RestartAllSilos();
CheckSilosRunning("After Restart", numExpectedSilos);
Assert.AreNotEqual(prim1, this.HostedCluster.Primary, "Should be different Primary silos after restart");
Assert.AreNotEqual(sec1, this.HostedCluster.Secondary, "Should be different Secondary silos after restart");
StreamTestUtils.LogEndTest(testName, logger);
}
private async Task Test_AllSilosRestart_PubSubCounts(string testName, string streamProviderName)
{
_streamId = Guid.NewGuid();
_streamProviderName = streamProviderName;
StreamTestUtils.LogStartTest(testName, _streamId, _streamProviderName, logger, HostedCluster);
long consumerGrainId = random.Next();
long producerGrainId = random.Next();
#if USE_GENERICS
IStreamReliabilityTestGrain <int> producerGrain =
#else
IStreamReliabilityTestGrain producerGrain =
#endif
await Do_BaselineTest(consumerGrainId, producerGrainId);
string when = "Before restart all silos";
await StreamTestUtils.CheckPubSubCounts(this.InternalClient, output, when, 1, 1, _streamId, _streamProviderName, StreamTestsConstants.StreamReliabilityNamespace);
// Restart silos
//RestartDefaultSilosButKeepCurrentClient(testName);
RestartAllSilos();
when = "After restart all silos";
CheckSilosRunning(when, numExpectedSilos);
// Note: It is not guaranteed that the list of producers will not get modified / cleaned up during silo shutdown, so can't assume count will be 1 here.
// Expected == -1 means don't care.
await StreamTestUtils.CheckPubSubCounts(this.InternalClient, output, when, -1, 1, _streamId, _streamProviderName, StreamTestsConstants.StreamReliabilityNamespace);
await producerGrain.SendItem(1);
when = "After SendItem";
await StreamTestUtils.CheckPubSubCounts(this.InternalClient, output, when, 1, 1, _streamId, _streamProviderName, StreamTestsConstants.StreamReliabilityNamespace);
var consumerGrain = GetGrain(consumerGrainId);
await CheckReceivedCounts(when, consumerGrain, 1, 0);
StreamTestUtils.LogEndTest(testName, logger);
}
private async Task Test_SiloRestarts_Consumer(string testName, string streamProviderName)
{
_streamId = Guid.NewGuid();
_streamProviderName = streamProviderName;
string when;
StreamTestUtils.LogStartTest(testName, _streamId, _streamProviderName, logger, HostedCluster);
long consumerGrainId = random.Next();
long producerGrainId = random.Next();
var producerGrain = await Do_BaselineTest(consumerGrainId, producerGrainId);
when = "Before restart one silo";
CheckSilosRunning(when, numExpectedSilos);
bool sameSilo = await CheckGrainCounts();
// Find which silo the consumer grain is located on
var consumerGrain = GetGrain(consumerGrainId);
SiloAddress siloAddress = await consumerGrain.GetLocation();
output.WriteLine("Consumer grain is located on silo {0} ; Producer on same silo = {1}", siloAddress, sameSilo);
// Restart the silo containing the consumer grain
SiloHandle siloToKill = this.HostedCluster.Silos.First(s => s.SiloAddress.Equals(siloAddress));
StopSilo(siloToKill, true, true);
// Note: Don't reinitialize client
when = "After restart one silo";
CheckSilosRunning(when, numExpectedSilos);
when = "SendItem";
await producerGrain.SendItem(1);
await CheckConsumerProducerStatus(when, producerGrainId, consumerGrainId, true, true);
StreamTestUtils.LogEndTest(testName, logger);
}
private async Task Test_PubSub_MultiConsumerSameGrain(string testName, string streamProviderName)
{
_streamId = Guid.NewGuid();
_streamProviderName = streamProviderName;
StreamTestUtils.LogStartTest(testName, _streamId, _streamProviderName, logger, HostedCluster);
// Grain Producer -> Grain 2 x Consumer
long consumerGrainId = random.Next();
long producerGrainId = random.Next();
string when;
logger.Info("Initializing: ConsumerGrain={0} ProducerGrain={1}", consumerGrainId, producerGrainId);
var consumerGrain = GetGrain(consumerGrainId);
var producerGrain = GetGrain(producerGrainId);
logger.Info("BecomeProducer: StreamId={0} Provider={1}", _streamId, _streamProviderName);
await producerGrain.BecomeProducer(_streamId, _streamProviderName);
when = "After BecomeProducer";
// Note: Only semantics guarenteed for producer is that they will have been registered by time that first msg is sent.
await producerGrain.SendItem(0);
await StreamTestUtils.CheckPubSubCounts(this.InternalClient, output, when, 1, 0, _streamId, _streamProviderName, StreamTestsConstants.StreamReliabilityNamespace);
logger.Info("AddConsumer x 2 : StreamId={0} Provider={1}", _streamId, _streamProviderName);
await consumerGrain.AddConsumer(_streamId, _streamProviderName);
when = "After first AddConsumer";
await StreamTestUtils.CheckPubSubCounts(this.InternalClient, output, when, 1, 1, _streamId, _streamProviderName, StreamTestsConstants.StreamReliabilityNamespace);
await consumerGrain.AddConsumer(_streamId, _streamProviderName);
when = "After second AddConsumer";
await StreamTestUtils.CheckPubSubCounts(this.InternalClient, output, when, 1, 2, _streamId, _streamProviderName, StreamTestsConstants.StreamReliabilityNamespace);
StreamTestUtils.LogEndTest(testName, logger);
}
private async Task Test_SiloJoins(string testName, string streamProviderName)
{
_streamId = Guid.NewGuid();
_streamProviderName = streamProviderName;
const int numLoops = 3;
StreamTestUtils.LogStartTest(testName, _streamId, _streamProviderName, logger, HostedCluster);
long consumerGrainId = random.Next();
long producerGrainId = random.Next();
var producerGrain = GetGrain(producerGrainId);
SiloAddress producerLocation = await producerGrain.GetLocation();
var consumerGrain = GetGrain(consumerGrainId);
SiloAddress consumerLocation = await consumerGrain.GetLocation();
output.WriteLine("Grain silo locations: Producer={0} Consumer={1}", producerLocation, consumerLocation);
// Note: This does first SendItem
await Do_BaselineTest(consumerGrainId, producerGrainId);
int expectedReceived = 1;
string when = "SendItem-2";
for (int i = 0; i < numLoops; i++)
{
await producerGrain.SendItem(2);
}
expectedReceived += numLoops;
await CheckConsumerProducerStatus(when, producerGrainId, consumerGrainId, true, true);
await CheckReceivedCounts(when, consumerGrain, expectedReceived, 0);
// Add new silo
//SiloHandle newSilo = StartAdditionalOrleans();
//WaitForLivenessToStabilize();
SiloHandle newSilo = this.HostedCluster.StartAdditionalSilo();
await this.HostedCluster.WaitForLivenessToStabilizeAsync();
when = "After starting additonal silo " + newSilo;
output.WriteLine(when);
CheckSilosRunning(when, numExpectedSilos + 1);
//when = "SendItem-3";
//output.WriteLine(when);
//for (int i = 0; i < numLoops; i++)
//{
// await producerGrain.SendItem(3);
//}
//expectedReceived += numLoops;
//await CheckConsumerProducerStatus(when, producerGrainId, consumerGrainId, true, true);
//await CheckReceivedCounts(when, consumerGrain, expectedReceived, 0);
// Find a Consumer Grain on the new silo
IStreamReliabilityTestGrain newConsumer = CreateGrainOnSilo(newSilo.SiloAddress);
await newConsumer.AddConsumer(_streamId, _streamProviderName);
output.WriteLine("Grain silo locations: Producer={0} OldConsumer={1} NewConsumer={2}", producerLocation, consumerLocation, newSilo.SiloAddress);
////Thread.Sleep(TimeSpan.FromSeconds(2));
when = "SendItem-4";
output.WriteLine(when);
for (int i = 0; i < numLoops; i++)
{
await producerGrain.SendItem(4);
}
expectedReceived += numLoops;
// Old consumer received the newly published messages
await CheckReceivedCounts(when + "-Old", consumerGrain, expectedReceived, 0);
// New consumer received the newly published messages
await CheckReceivedCounts(when + "-New", newConsumer, numLoops, 0);
StreamTestUtils.LogEndTest(testName, logger);
}
private async Task Test_PubSub_Unsubscribe(string testName, string streamProviderName)
{
_streamId = Guid.NewGuid();
_streamProviderName = streamProviderName;
StreamTestUtils.LogStartTest(testName, _streamId, _streamProviderName, logger, HostedCluster);
// Grain Producer -> Grain 2 x Consumer
// Note: PubSub should only count distinct grains, even if a grain has multiple consumer handles
long consumerGrainId = random.Next();
long producerGrainId = random.Next();
string when;
logger.Info("Initializing: ConsumerGrain={0} ProducerGrain={1}", consumerGrainId, producerGrainId);
var consumerGrain = GetGrain(consumerGrainId);
var producerGrain = GetGrain(producerGrainId);
logger.Info("BecomeProducer: StreamId={0} Provider={1}", _streamId, _streamProviderName);
await producerGrain.BecomeProducer(_streamId, _streamProviderName);
await producerGrain.BecomeProducer(_streamId, _streamProviderName);
when = "After BecomeProducer";
// Note: Only semantics guarenteed are that producer will have been registered by time that first msg is sent.
await producerGrain.SendItem(0);
await StreamTestUtils.CheckPubSubCounts(this.InternalClient, output, when, 1, 0, _streamId, _streamProviderName, StreamTestsConstants.StreamReliabilityNamespace);
logger.Info("AddConsumer x 2 : StreamId={0} Provider={1}", _streamId, _streamProviderName);
var c1 = await consumerGrain.AddConsumer(_streamId, _streamProviderName);
when = "After first AddConsumer";
await StreamTestUtils.CheckPubSubCounts(this.InternalClient, output, when, 1, 1, _streamId, _streamProviderName, StreamTestsConstants.StreamReliabilityNamespace);
await CheckConsumerCounts(when, consumerGrain, 1);
var c2 = await consumerGrain.AddConsumer(_streamId, _streamProviderName);
when = "After second AddConsumer";
await StreamTestUtils.CheckPubSubCounts(this.InternalClient, output, when, 1, 2, _streamId, _streamProviderName, StreamTestsConstants.StreamReliabilityNamespace);
await CheckConsumerCounts(when, consumerGrain, 2);
logger.Info("RemoveConsumer: StreamId={0} Provider={1}", _streamId, _streamProviderName);
await consumerGrain.RemoveConsumer(_streamId, _streamProviderName, c1);
when = "After first RemoveConsumer";
await StreamTestUtils.CheckPubSubCounts(this.InternalClient, output, when, 1, 1, _streamId, _streamProviderName, StreamTestsConstants.StreamReliabilityNamespace);
await CheckConsumerCounts(when, consumerGrain, 1);
#if REMOVE_PRODUCER
logger.Info("RemoveProducer: StreamId={0} Provider={1}", _streamId, _streamProviderName);
await producerGrain.RemoveProducer(_streamId, _streamProviderName);
when = "After RemoveProducer";
await CheckPubSubCounts(when, 0, 1);
await CheckConsumerCounts(when, consumerGrain, 1);
#endif
logger.Info("RemoveConsumer: StreamId={0} Provider={1}", _streamId, _streamProviderName);
await consumerGrain.RemoveConsumer(_streamId, _streamProviderName, c2);
when = "After second RemoveConsumer";
#if REMOVE_PRODUCER
await CheckPubSubCounts(when, 0, 0);
#else
await StreamTestUtils.CheckPubSubCounts(this.InternalClient, output, when, 1, 0, _streamId, _streamProviderName, StreamTestsConstants.StreamReliabilityNamespace);
#endif
await CheckConsumerCounts(when, consumerGrain, 0);
StreamTestUtils.LogEndTest(testName, logger);
}
private async Task Test_AddMany_Consumers(string testName, string streamProviderName)
{
const int numLoops = 100;
const int numGrains = 10;
_streamId = Guid.NewGuid();
_streamProviderName = streamProviderName;
StreamTestUtils.LogStartTest(testName, _streamId, _streamProviderName, logger, HostedCluster);
long consumerGrainId = random.Next();
long producerGrainId = random.Next();
var producerGrain = GetGrain(producerGrainId);
var consumerGrain = GetGrain(consumerGrainId);
#if DELETE_AFTER_TEST
_usedGrains.Add(producerGrain);
_usedGrains.Add(consumerGrain);
#endif
// Note: This does first SendItem
await Do_BaselineTest(consumerGrainId, producerGrainId);
int baseId = 10000 * ++baseConsumerId;
var grains1 = await Do_AddConsumerGrains(baseId, numGrains);
for (int i = 0; i < numLoops; i++)
{
await producerGrain.SendItem(2);
}
string when1 = "AddConsumers-Send-2";
// Messages received by original consumer grain
await CheckReceivedCounts(when1, consumerGrain, numLoops + 1, 0);
// Messages received by new consumer grains
// ReSharper disable once AccessToModifiedClosure
await Task.WhenAll(grains1.Select(async g =>
{
await CheckReceivedCounts(when1, g, numLoops, 0);
#if DELETE_AFTER_TEST
_usedGrains.Add(g);
#endif
}));
string when2 = "AddConsumers-Send-3";
baseId = 10000 * ++baseConsumerId;
var grains2 = await Do_AddConsumerGrains(baseId, numGrains);
for (int i = 0; i < numLoops; i++)
{
await producerGrain.SendItem(3);
}
////Thread.Sleep(TimeSpan.FromSeconds(2));
// Messages received by original consumer grain
await CheckReceivedCounts(when2, consumerGrain, numLoops *2 + 1, 0);
// Messages received by new consumer grains
await Task.WhenAll(grains2.Select(g => CheckReceivedCounts(when2, g, numLoops, 0)));
StreamTestUtils.LogEndTest(testName, logger);
}