private async Task <IStreamReliabilityTestGrain[]> Do_AddConsumerGrains(long baseId, int numGrains)
#endif
{
logger.Info("Initializing: BaseId={0} NumGrains={1}", baseId, numGrains);
#if USE_GENERICS
var grains = new IStreamReliabilityTestGrain <int> [numGrains];
#else
var grains = new IStreamReliabilityTestGrain[numGrains];
#endif
List <Task> promises = new List <Task>(numGrains);
for (int i = 0; i < numGrains; i++)
{
grains[i] = GetGrain(i + baseId);
promises.Add(grains[i].Ping());
#if DELETE_AFTER_TEST
_usedGrains.Add(grains[i]);
#endif
}
await Task.WhenAll(promises);
logger.Info("AddConsumer: StreamId={0} Provider={1}", _streamId, _streamProviderName);
await Task.WhenAll(grains.Select(g => g.AddConsumer(_streamId, _streamProviderName)));
return(grains);
}
protected async Task CheckConsumerCounts(string when, IStreamReliabilityTestGrain consumerGrain, int expectedConsumerCount)
#endif
{
int consumerCount = await consumerGrain.GetConsumerCount();
StreamTestUtils.Assert_AreEqual(output, expectedConsumerCount, consumerCount, "ConsumerCount for stream {0} {1}", _streamId, when);
}
protected async Task CheckReceivedCounts(string when, IStreamReliabilityTestGrain consumerGrain, int expectedReceivedCount, int expectedErrorsCount)
#endif
{
long pk = consumerGrain.GetPrimaryKeyLong();
int receivedCount = 0;
for (int i = 0; i < 20; i++)
{
receivedCount = await consumerGrain.GetReceivedCount();
output.WriteLine("After {0}s ReceivedCount={1} for grain {2}", i, receivedCount, pk);
if (receivedCount == expectedReceivedCount)
{
break;
}
Thread.Sleep(TimeSpan.FromSeconds(1));
}
StreamTestUtils.Assert_AreEqual(output, expectedReceivedCount, receivedCount,
"ReceivedCount for stream {0} for grain {1} {2}", _streamId, pk, when);
int errorsCount = await consumerGrain.GetErrorsCount();
StreamTestUtils.Assert_AreEqual(output, expectedErrorsCount, errorsCount, "ErrorsCount for stream {0} for grain {1} {2}", _streamId, pk, when);
}
private async Task Test_AllSilosRestart_PubSubCounts(string testName, string streamProviderName)
{
_streamId = Guid.NewGuid();
_streamProviderName = streamProviderName;
StreamTestUtils.LogStartTest(testName, _streamId, _streamProviderName, logger, HostedCluster);
long consumerGrainId = random.Next();
long producerGrainId = random.Next();
#if USE_GENERICS
IStreamReliabilityTestGrain <int> producerGrain =
#else
IStreamReliabilityTestGrain producerGrain =
#endif
await Do_BaselineTest(consumerGrainId, producerGrainId);
string when = "Before restart all silos";
await StreamTestUtils.CheckPubSubCounts(this.InternalClient, output, when, 1, 1, _streamId, _streamProviderName, StreamTestsConstants.StreamReliabilityNamespace);
// Restart silos
//RestartDefaultSilosButKeepCurrentClient(testName);
RestartAllSilos();
when = "After restart all silos";
CheckSilosRunning(when, numExpectedSilos);
// Note: It is not guaranteed that the list of producers will not get modified / cleaned up during silo shutdown, so can't assume count will be 1 here.
// Expected == -1 means don't care.
await StreamTestUtils.CheckPubSubCounts(this.InternalClient, output, when, -1, 1, _streamId, _streamProviderName, StreamTestsConstants.StreamReliabilityNamespace);
await producerGrain.SendItem(1);
when = "After SendItem";
await StreamTestUtils.CheckPubSubCounts(this.InternalClient, output, when, 1, 1, _streamId, _streamProviderName, StreamTestsConstants.StreamReliabilityNamespace);
var consumerGrain = GetGrain(consumerGrainId);
await CheckReceivedCounts(when, consumerGrain, 1, 0);
StreamTestUtils.LogEndTest(testName, logger);
}
private async Task Test_SiloJoins(string testName, string streamProviderName)
{
_streamId = Guid.NewGuid();
_streamProviderName = streamProviderName;
const int numLoops = 3;
StreamTestUtils.LogStartTest(testName, _streamId, _streamProviderName, logger, HostedCluster);
long consumerGrainId = random.Next();
long producerGrainId = random.Next();
var producerGrain = GetGrain(producerGrainId);
SiloAddress producerLocation = await producerGrain.GetLocation();
var consumerGrain = GetGrain(consumerGrainId);
SiloAddress consumerLocation = await consumerGrain.GetLocation();
output.WriteLine("Grain silo locations: Producer={0} Consumer={1}", producerLocation, consumerLocation);
// Note: This does first SendItem
await Do_BaselineTest(consumerGrainId, producerGrainId);
int expectedReceived = 1;
string when = "SendItem-2";
for (int i = 0; i < numLoops; i++)
{
await producerGrain.SendItem(2);
}
expectedReceived += numLoops;
await CheckConsumerProducerStatus(when, producerGrainId, consumerGrainId, true, true);
await CheckReceivedCounts(when, consumerGrain, expectedReceived, 0);
// Add new silo
//SiloHandle newSilo = StartAdditionalOrleans();
//WaitForLivenessToStabilize();
SiloHandle newSilo = this.HostedCluster.StartAdditionalSilo();
await this.HostedCluster.WaitForLivenessToStabilizeAsync();
when = "After starting additonal silo " + newSilo;
output.WriteLine(when);
CheckSilosRunning(when, numExpectedSilos + 1);
//when = "SendItem-3";
//output.WriteLine(when);
//for (int i = 0; i < numLoops; i++)
//{
// await producerGrain.SendItem(3);
//}
//expectedReceived += numLoops;
//await CheckConsumerProducerStatus(when, producerGrainId, consumerGrainId, true, true);
//await CheckReceivedCounts(when, consumerGrain, expectedReceived, 0);
// Find a Consumer Grain on the new silo
IStreamReliabilityTestGrain newConsumer = CreateGrainOnSilo(newSilo.SiloAddress);
await newConsumer.AddConsumer(_streamId, _streamProviderName);
output.WriteLine("Grain silo locations: Producer={0} OldConsumer={1} NewConsumer={2}", producerLocation, consumerLocation, newSilo.SiloAddress);
////Thread.Sleep(TimeSpan.FromSeconds(2));
when = "SendItem-4";
output.WriteLine(when);
for (int i = 0; i < numLoops; i++)
{
await producerGrain.SendItem(4);
}
expectedReceived += numLoops;
// Old consumer received the newly published messages
await CheckReceivedCounts(when + "-Old", consumerGrain, expectedReceived, 0);
// New consumer received the newly published messages
await CheckReceivedCounts(when + "-New", newConsumer, numLoops, 0);
StreamTestUtils.LogEndTest(testName, logger);
}
protected async Task CheckConsumerCounts <T>(string when, IStreamReliabilityTestGrain <T> consumerGrain, int expectedConsumerCount)
protected async Task CheckReceivedCounts <T>(string when, IStreamReliabilityTestGrain <T> consumerGrain, int expectedReceivedCount, int expectedErrorsCount)