public async Task Rem_Azure_1F_Basic()
{
IReminderTestGrain2 g1 = this.GrainFactory.GetGrain <IReminderTestGrain2>(Guid.NewGuid());
TimeSpan period = await g1.GetReminderPeriod(DR);
Task <bool> test = Task.Run(async() => { await PerGrainFailureTest(g1); return(true); });
Thread.Sleep(period.Multiply(failAfter));
// stop the secondary silo
log.Info("Stopping secondary silo");
await this.HostedCluster.StopSiloAsync(this.HostedCluster.SecondarySilos.First());
await test; // Block until test completes.
}
public async Task Test_Reminders_Basic_ListOps()
{
Guid id = Guid.NewGuid();
log.Info("Start Grain Id = {0}", id);
IReminderTestGrain2 grain = GrainClient.GrainFactory.GetGrain <IReminderTestGrain2>(id);
const int count = 5;
Task <IGrainReminder>[] startReminderTasks = new Task <IGrainReminder> [count];
for (int i = 0; i < count; i++)
{
startReminderTasks[i] = grain.StartReminder(DR + "_" + i);
log.Info("Started {0}_{1}", DR, i);
}
await Task.WhenAll(startReminderTasks);
// do comparison on strings
List <string> registered = (from reminder in startReminderTasks select reminder.Result.ReminderName).ToList();
log.Info("Waited");
List <IGrainReminder> remindersList = await grain.GetRemindersList();
List <string> fetched = (from reminder in remindersList select reminder.ReminderName).ToList();
foreach (var remRegistered in registered)
{
Assert.IsTrue(fetched.Remove(remRegistered),
"Couldn't get reminder {0}. Registered list: {1}, fetched list: {2}",
remRegistered,
Utils.EnumerableToString(registered),
Utils.EnumerableToString(remindersList, r => r.ReminderName));
}
Assert.IsTrue(fetched.Count == 0, "More than registered reminders. Extra: {0}", Utils.EnumerableToString(fetched));
// do some time tests as well
log.Info("Time tests");
TimeSpan period = await grain.GetReminderPeriod(DR);
Thread.Sleep(period.Multiply(2) + LEEWAY); // giving some leeway
for (int i = 0; i < count; i++)
{
long curr = await grain.GetCounter(DR + "_" + i);
Assert.AreEqual(2, curr, string.Format("Incorrect ticks for {0}_{1}", DR, i));
}
}
public async Task Rem_Azure_1F_Basic()
{
log.Info(TestContext.TestName);
IReminderTestGrain2 g1 = GrainClient.GrainFactory.GetGrain <IReminderTestGrain2>(Guid.NewGuid());
TimeSpan period = await g1.GetReminderPeriod(DR);
Task <bool> test = Task.Run(async() => { await PerGrainFailureTest(g1); return(true); });
Thread.Sleep(period.Multiply(failAfter));
// stop the secondary silo
log.Info("Stopping secondary silo");
StopSilo(Secondary);
await test; // Block until test completes.
}
public async Task Test_Reminders_Basic_ListOps()
{
Guid id = Guid.NewGuid();
log.LogInformation("Start Grain Id = {GrainId}", id);
IReminderTestGrain2 grain = this.GrainFactory.GetGrain <IReminderTestGrain2>(id);
const int count = 5;
Task <IGrainReminder>[] startReminderTasks = new Task <IGrainReminder> [count];
for (int i = 0; i < count; i++)
{
startReminderTasks[i] = grain.StartReminder(DR + "_" + i);
log.LogInformation("Started {ReminderName}_{ReminderNumber}", DR, i);
}
await Task.WhenAll(startReminderTasks);
// do comparison on strings
List <string> registered = (from reminder in startReminderTasks select reminder.Result.ReminderName).ToList();
log.LogInformation("Waited");
List <IGrainReminder> remindersList = await grain.GetRemindersList();
List <string> fetched = (from reminder in remindersList select reminder.ReminderName).ToList();
foreach (var remRegistered in registered)
{
Assert.True(fetched.Remove(remRegistered), $"Couldn't get reminder {remRegistered}. " +
$"Registered list: {Utils.EnumerableToString(registered)}, " +
$"fetched list: {Utils.EnumerableToString(remindersList, r => r.ReminderName)}");
}
Assert.True(fetched.Count == 0, $"More than registered reminders. Extra: {Utils.EnumerableToString(fetched)}");
// do some time tests as well
log.LogInformation("Time tests");
TimeSpan period = await grain.GetReminderPeriod(DR);
await Task.Delay(period.Multiply(2) + LEEWAY); // giving some leeway
for (int i = 0; i < count; i++)
{
long curr = await grain.GetCounter(DR + "_" + i);
Assert.Equal(2, curr);
}
}
public async Task Rem_Azure_1F1J_MultiGrain()
{
log.Info(TestContext.TestName);
List <SiloHandle> silos = StartAdditionalSilos(1);
await WaitForLivenessToStabilizeAsync();
IReminderTestGrain2 g1 = GrainClient.GrainFactory.GetGrain <IReminderTestGrain2>(Guid.NewGuid());
IReminderTestGrain2 g2 = GrainClient.GrainFactory.GetGrain <IReminderTestGrain2>(Guid.NewGuid());
IReminderTestGrain2 g3 = GrainClient.GrainFactory.GetGrain <IReminderTestGrain2>(Guid.NewGuid());
IReminderTestGrain2 g4 = GrainClient.GrainFactory.GetGrain <IReminderTestGrain2>(Guid.NewGuid());
IReminderTestGrain2 g5 = GrainClient.GrainFactory.GetGrain <IReminderTestGrain2>(Guid.NewGuid());
TimeSpan period = await g1.GetReminderPeriod(DR);
Task <bool>[] tasks =
{
Task.Run(async() => { await PerGrainFailureTest(g1); return(true); }),
Task.Run(async() => { await PerGrainFailureTest(g2); return(true); }),
Task.Run(async() => { await PerGrainFailureTest(g3); return(true); }),
Task.Run(async() => { await PerGrainFailureTest(g4); return(true); }),
Task.Run(async() => { await PerGrainFailureTest(g5); return(true); })
};
Thread.Sleep(period.Multiply(failAfter));
var siloToKill = silos[random.Next(silos.Count)];
// stop a silo and join a new one in parallel
log.Info("Stopping a silo and joining a silo");
Task <bool> t1 = Task.Factory.StartNew(() =>
{
StopSilo(siloToKill);
return(true);
});
Task <bool> t2 = Task.Factory.StartNew(() =>
{
StartAdditionalSilos(1);
return(true);
});
await Task.WhenAll(new[] { t1, t2 }).WithTimeout(ENDWAIT);
await Task.WhenAll(tasks).WithTimeout(ENDWAIT); // Block until all tasks complete.
log.Info("\n\n\nReminderTest_1F1J_MultiGrain passed OK.\n\n\n");
}
public async Task Rem_Azure_GT_1F1J_MultiGrain()
{
List <SiloHandle> silos = this.HostedCluster.StartAdditionalSilos(1);
await this.HostedCluster.WaitForLivenessToStabilizeAsync();
IReminderTestGrain2 g1 = GrainClient.GrainFactory.GetGrain <IReminderTestGrain2>(Guid.NewGuid());
IReminderTestGrain2 g2 = GrainClient.GrainFactory.GetGrain <IReminderTestGrain2>(Guid.NewGuid());
IReminderTestCopyGrain g3 = GrainClient.GrainFactory.GetGrain <IReminderTestCopyGrain>(Guid.NewGuid());
IReminderTestCopyGrain g4 = GrainClient.GrainFactory.GetGrain <IReminderTestCopyGrain>(Guid.NewGuid());
TimeSpan period = await g1.GetReminderPeriod(DR);
Task[] tasks =
{
Task.Run(() => PerGrainFailureTest(g1)),
Task.Run(() => PerGrainFailureTest(g2)),
Task.Run(() => PerCopyGrainFailureTest(g3)),
Task.Run(() => PerCopyGrainFailureTest(g4)),
};
Thread.Sleep(period.Multiply(failAfter));
var siloToKill = silos[random.Next(silos.Count)];
// stop a silo and join a new one in parallel
log.Info("Stopping a silo and joining a silo");
Task <bool> t1 = Task.Run(() =>
{
this.HostedCluster.StopSilo(siloToKill);
return(true);
});
Task <bool> t2 = Task.Run(() =>
{
this.HostedCluster.StartAdditionalSilos(1);
return(true);
});
await Task.WhenAll(new[] { t1, t2 }).WithTimeout(ENDWAIT);
await Task.WhenAll(tasks).WithTimeout(ENDWAIT); // Block until all tasks complete.
}
public async Task Rem_Azure_1F1J_MultiGrain()
{
List <SiloHandle> silos = await this.HostedCluster.StartAdditionalSilosAsync(1);
await this.HostedCluster.WaitForLivenessToStabilizeAsync();
IReminderTestGrain2 g1 = this.GrainFactory.GetGrain <IReminderTestGrain2>(Guid.NewGuid());
IReminderTestGrain2 g2 = this.GrainFactory.GetGrain <IReminderTestGrain2>(Guid.NewGuid());
IReminderTestGrain2 g3 = this.GrainFactory.GetGrain <IReminderTestGrain2>(Guid.NewGuid());
IReminderTestGrain2 g4 = this.GrainFactory.GetGrain <IReminderTestGrain2>(Guid.NewGuid());
IReminderTestGrain2 g5 = this.GrainFactory.GetGrain <IReminderTestGrain2>(Guid.NewGuid());
TimeSpan period = await g1.GetReminderPeriod(DR);
Task[] tasks =
{
Task.Run(() => PerGrainFailureTest(g1)),
Task.Run(() => PerGrainFailureTest(g2)),
Task.Run(() => PerGrainFailureTest(g3)),
Task.Run(() => PerGrainFailureTest(g4)),
Task.Run(() => PerGrainFailureTest(g5)),
};
Thread.Sleep(period.Multiply(failAfter));
var siloToKill = silos[random.Next(silos.Count)];
// stop a silo and join a new one in parallel
log.Info("Stopping a silo and joining a silo");
Task t1 = Task.Factory.StartNew(async() => await this.HostedCluster.StopSiloAsync(siloToKill));
Task t2 = this.HostedCluster.StartAdditionalSilosAsync(1, true).ContinueWith(t =>
{
t.GetAwaiter().GetResult();
});
await Task.WhenAll(new[] { t1, t2 }).WithTimeout(ENDWAIT);
await Task.WhenAll(tasks).WithTimeout(ENDWAIT); // Block until all tasks complete.
log.Info("\n\n\nReminderTest_1F1J_MultiGrain passed OK.\n\n\n");
}
protected async Task <bool> PerGrainFailureTest(IReminderTestGrain2 grain)
{
TimeSpan period = await grain.GetReminderPeriod(DR);
await grain.StartReminder(DR);
TimeSpan sleepFor = period.Multiply(failCheckAfter) + LEEWAY; // giving some leeway
Thread.Sleep(sleepFor);
long last = await grain.GetCounter(DR);
AssertIsInRange(last, failCheckAfter - 1, failCheckAfter + 1, grain, DR, sleepFor);
await grain.StopReminder(DR);
sleepFor = period.Multiply(2) + LEEWAY; // giving some leeway
Thread.Sleep(sleepFor);
long curr = await grain.GetCounter(DR);
AssertIsInRange(curr, last, last + 1, grain, DR, sleepFor);
return(true);
}
public async Task Rem_Azure_Basic()
{
// start up a test grain and get the period that it's programmed to use.
IReminderTestGrain2 grain = GrainClient.GrainFactory.GetGrain <IReminderTestGrain2>(Guid.NewGuid());
TimeSpan period = await grain.GetReminderPeriod(DR);
// start up the 'DR' reminder and wait for two ticks to pass.
await grain.StartReminder(DR);
Thread.Sleep(period.Multiply(2) + LEEWAY); // giving some leeway
// retrieve the value of the counter-- it should match the sequence number which is the number of periods
// we've waited.
long last = await grain.GetCounter(DR);
Assert.Equal(2, last);
// stop the timer and wait for a whole period.
await grain.StopReminder(DR);
Thread.Sleep(period.Multiply(1) + LEEWAY); // giving some leeway
// the counter should not have changed.
long curr = await grain.GetCounter(DR);
Assert.Equal(last, curr);
}
protected async Task <bool> PerGrainMultiReminderTest(IReminderTestGrain2 g)
{
TimeSpan period = await g.GetReminderPeriod(DR);
this.log.Info("PerGrainMultiReminderTest Period={0} Grain={1}", period, g);
// Each reminder is started 2 periods after the previous reminder
// once all reminders have been started, stop them every 2 periods
// except the default reminder, which we stop after 3 periods instead
// just to test and break the symmetry
// Start Default Reminder
await g.StartReminder(DR);
TimeSpan sleepFor = period.Multiply(2) + LEEWAY; // giving some leeway
Thread.Sleep(sleepFor);
long last = await g.GetCounter(DR);
AssertIsInRange(last, 1, 2, g, DR, sleepFor);
// Start R1
await g.StartReminder(R1);
Thread.Sleep(sleepFor);
last = await g.GetCounter(R1);
AssertIsInRange(last, 1, 2, g, R1, sleepFor);
// Start R2
await g.StartReminder(R2);
Thread.Sleep(sleepFor);
last = await g.GetCounter(R1);
AssertIsInRange(last, 3, 4, g, R1, sleepFor);
last = await g.GetCounter(R2);
AssertIsInRange(last, 1, 2, g, R2, sleepFor);
last = await g.GetCounter(DR);
AssertIsInRange(last, 5, 6, g, DR, sleepFor);
// Stop R1
await g.StopReminder(R1);
Thread.Sleep(sleepFor);
last = await g.GetCounter(R1);
AssertIsInRange(last, 3, 4, g, R1, sleepFor);
last = await g.GetCounter(R2);
AssertIsInRange(last, 3, 4, g, R2, sleepFor);
last = await g.GetCounter(DR);
AssertIsInRange(last, 7, 8, g, DR, sleepFor);
// Stop R2
await g.StopReminder(R2);
sleepFor = period.Multiply(3) + LEEWAY; // giving some leeway
Thread.Sleep(sleepFor);
last = await g.GetCounter(R1);
AssertIsInRange(last, 3, 4, g, R1, sleepFor);
last = await g.GetCounter(R2);
AssertIsInRange(last, 3, 4, g, R2, sleepFor);
last = await g.GetCounter(DR);
AssertIsInRange(last, 10, 12, g, DR, sleepFor);
// Stop Default reminder
await g.StopReminder(DR);
sleepFor = period.Multiply(1) + LEEWAY; // giving some leeway
Thread.Sleep(sleepFor);
last = await g.GetCounter(R1);
AssertIsInRange(last, 3, 4, g, R1, sleepFor);
last = await g.GetCounter(R2);
AssertIsInRange(last, 3, 4, g, R2, sleepFor);
last = await g.GetCounter(DR);
AssertIsInRange(last, 10, 12, g, DR, sleepFor);
return(true);
}
internal async Task <bool> PerGrainMultiReminderTestChurn(IReminderTestGrain2 g)
{
// for churn cases, we do execute start and stop reminders with retries as we don't have the queue-ing
// functionality implemented on the LocalReminderService yet
TimeSpan period = await g.GetReminderPeriod(DR);
this.log.Info("PerGrainMultiReminderTestChurn Period={0} Grain={1}", period, g);
// Start Default Reminder
//g.StartReminder(DR, file + "_" + DR).Wait();
await ExecuteWithRetries(g.StartReminder, DR);
TimeSpan sleepFor = period.Multiply(2);
await Task.Delay(sleepFor);
// Start R1
//g.StartReminder(R1, file + "_" + R1).Wait();
await ExecuteWithRetries(g.StartReminder, R1);
sleepFor = period.Multiply(2);
await Task.Delay(sleepFor);
// Start R2
//g.StartReminder(R2, file + "_" + R2).Wait();
await ExecuteWithRetries(g.StartReminder, R2);
sleepFor = period.Multiply(2);
await Task.Delay(sleepFor);
sleepFor = period.Multiply(1);
await Task.Delay(sleepFor);
// Stop R1
//g.StopReminder(R1).Wait();
await ExecuteWithRetriesStop(g.StopReminder, R1);
sleepFor = period.Multiply(2);
await Task.Delay(sleepFor);
// Stop R2
//g.StopReminder(R2).Wait();
await ExecuteWithRetriesStop(g.StopReminder, R2);
sleepFor = period.Multiply(1);
await Task.Delay(sleepFor);
// Stop Default reminder
//g.StopReminder(DR).Wait();
await ExecuteWithRetriesStop(g.StopReminder, DR);
sleepFor = period.Multiply(1) + LEEWAY; // giving some leeway
await Task.Delay(sleepFor);
long last = await g.GetCounter(R1);
AssertIsInRange(last, 4, 6, g, R1, sleepFor);
last = await g.GetCounter(R2);
AssertIsInRange(last, 4, 6, g, R2, sleepFor);
last = await g.GetCounter(DR);
AssertIsInRange(last, 9, 10, g, DR, sleepFor);
return(true);
}
protected async Task<bool> PerGrainMultiReminderTest(IReminderTestGrain2 g)
{
TimeSpan period = await g.GetReminderPeriod(DR);
logger.Info("PerGrainMultiReminderTest Period={0} Grain={1}", period, g);
// Each reminder is started 2 periods after the previous reminder
// once all reminders have been started, stop them every 2 periods
// except the default reminder, which we stop after 3 periods instead
// just to test and break the symmetry
// Start Default Reminder
await g.StartReminder(DR);
TimeSpan sleepFor = period.Multiply(2) + LEEWAY; // giving some leeway
Thread.Sleep(sleepFor);
long last = await g.GetCounter(DR);
AssertIsInRange(last, 1, 2, g, DR, sleepFor);
// Start R1
await g.StartReminder(R1);
Thread.Sleep(sleepFor);
last = await g.GetCounter(R1);
AssertIsInRange(last, 1, 2, g, R1, sleepFor);
// Start R2
await g.StartReminder(R2);
Thread.Sleep(sleepFor);
last = await g.GetCounter(R1);
AssertIsInRange(last, 3, 4, g, R1, sleepFor);
last = await g.GetCounter(R2);
AssertIsInRange(last, 1, 2, g, R2, sleepFor);
last = await g.GetCounter(DR);
AssertIsInRange(last, 5, 6, g, DR, sleepFor);
// Stop R1
await g.StopReminder(R1);
Thread.Sleep(sleepFor);
last = await g.GetCounter(R1);
AssertIsInRange(last, 3, 4, g, R1, sleepFor);
last = await g.GetCounter(R2);
AssertIsInRange(last, 3, 4, g, R2, sleepFor);
last = await g.GetCounter(DR);
AssertIsInRange(last, 7, 8, g, DR, sleepFor);
// Stop R2
await g.StopReminder(R2);
sleepFor = period.Multiply(3) + LEEWAY; // giving some leeway
Thread.Sleep(sleepFor);
last = await g.GetCounter(R1);
AssertIsInRange(last, 3, 4, g, R1, sleepFor);
last = await g.GetCounter(R2);
AssertIsInRange(last, 3, 4, g, R2, sleepFor);
last = await g.GetCounter(DR);
AssertIsInRange(last, 10, 12, g, DR, sleepFor);
// Stop Default reminder
await g.StopReminder(DR);
sleepFor = period.Multiply(1) + LEEWAY; // giving some leeway
Thread.Sleep(sleepFor);
last = await g.GetCounter(R1);
AssertIsInRange(last, 3, 4, g, R1, sleepFor);
last = await g.GetCounter(R2);
AssertIsInRange(last, 3, 4, g, R2, sleepFor);
last = await g.GetCounter(DR);
AssertIsInRange(last, 10, 12, g, DR, sleepFor);
return true;
}
protected async Task<bool> PerGrainFailureTest(IReminderTestGrain2 grain)
{
TimeSpan period = await grain.GetReminderPeriod(DR);
logger.Info("PerGrainFailureTest Period={0} Grain={1}", period, grain);
await grain.StartReminder(DR);
TimeSpan sleepFor = period.Multiply(failCheckAfter) + LEEWAY; // giving some leeway
Thread.Sleep(sleepFor);
long last = await grain.GetCounter(DR);
AssertIsInRange(last, failCheckAfter - 1, failCheckAfter + 1, grain, DR, sleepFor);
await grain.StopReminder(DR);
sleepFor = period.Multiply(2) + LEEWAY; // giving some leeway
Thread.Sleep(sleepFor);
long curr = await grain.GetCounter(DR);
AssertIsInRange(curr, last, last + 1, grain, DR, sleepFor);
return true;
}
internal async Task<bool> PerGrainMultiReminderTestChurn(IReminderTestGrain2 g)
{
// for churn cases, we do execute start and stop reminders with retries as we don't have the queue-ing
// functionality implemented on the LocalReminderService yet
TimeSpan period = await g.GetReminderPeriod(DR);
logger.Info("PerGrainMultiReminderTestChurn Period={0} Grain={1}", period, g);
// Start Default Reminder
//g.StartReminder(DR, file + "_" + DR).Wait();
await ExecuteWithRetries(g.StartReminder, DR);
TimeSpan sleepFor = period.Multiply(2);
await Task.Delay(sleepFor);
// Start R1
//g.StartReminder(R1, file + "_" + R1).Wait();
await ExecuteWithRetries(g.StartReminder, R1);
sleepFor = period.Multiply(2);
await Task.Delay(sleepFor);
// Start R2
//g.StartReminder(R2, file + "_" + R2).Wait();
await ExecuteWithRetries(g.StartReminder, R2);
sleepFor = period.Multiply(2);
await Task.Delay(sleepFor);
sleepFor = period.Multiply(1);
await Task.Delay(sleepFor);
// Stop R1
//g.StopReminder(R1).Wait();
await ExecuteWithRetriesStop(g.StopReminder, R1);
sleepFor = period.Multiply(2);
await Task.Delay(sleepFor);
// Stop R2
//g.StopReminder(R2).Wait();
await ExecuteWithRetriesStop(g.StopReminder, R2);
sleepFor = period.Multiply(1);
await Task.Delay(sleepFor);
// Stop Default reminder
//g.StopReminder(DR).Wait();
await ExecuteWithRetriesStop(g.StopReminder, DR);
sleepFor = period.Multiply(1) + LEEWAY; // giving some leeway
await Task.Delay(sleepFor);
long last = await g.GetCounter(R1);
AssertIsInRange(last, 4, 6, g, R1, sleepFor);
last = await g.GetCounter(R2);
AssertIsInRange(last, 4, 6, g, R2, sleepFor);
last = await g.GetCounter(DR);
AssertIsInRange(last, 9, 10, g, DR, sleepFor);
return true;
}