public async Task SiloUngracefulShutdown_OutstandingRequestsBreak()
{
var grain = HostedCluster.GrainFactory.GetGrain <ILongRunningTaskGrain <bool> >(Guid.NewGuid());
var instanceId = await grain.GetRuntimeInstanceId();
var target = HostedCluster.GrainFactory.GetGrain <ILongRunningTaskGrain <bool> >(Guid.NewGuid());
var targetInstanceId = await target.GetRuntimeInstanceId();
var isOnSameSilo = instanceId == targetInstanceId;
Assert.False(isOnSameSilo, "Activations must be placed on different silos");
var promise = instanceId.Contains(HostedCluster.Primary.Endpoint.ToString()) ?
grain.CallOtherLongRunningTask(target, true, TimeSpan.FromSeconds(7))
: target.CallOtherLongRunningTask(grain, true, TimeSpan.FromSeconds(7));
await Task.Delay(500);
HostedCluster.KillSilo(HostedCluster.SecondarySilos[0]);
try
{
await promise;
Assert.True(false, "The broken promise exception was not thrown");
}
catch (Exception ex)
{
Assert.Equal(typeof(SiloUnavailableException), ex.GetBaseException().GetType());
}
}
public async Task FindActiveGrains_typed()
{
//IManagementGrain mgmtGrain = GrainClient.GrainFactory.GetGrain<IManagementGrain>(RuntimeInterfaceConstants.SYSTEM_MANAGEMENT_ID);
//Catalog.GetGrainStatistics();
// start a second silo and wait for it
if (HostedCluster.SecondarySilos.Count == 0)
{
HostedCluster.StartAdditionalSilo();
await HostedCluster.WaitForLivenessToStabilizeAsync();
}
// create grains
output.WriteLine("creating and activating grains");
var grain1 = GrainClient.GrainFactory.GetGrain <ISimpleGrain>(1);
var grain2 = GrainClient.GrainFactory.GetGrain <ISimpleGrain>(2);
var grain3 = GrainClient.GrainFactory.GetGrain <ISimpleGrain>(3);
await grain1.GetA();
await grain2.GetA();
await grain3.GetA();
//enumerate active grains
output.WriteLine("\n\nour own grain statistics");
IActiveGrainEnumeratorGrain enumGrain = GrainClient.GrainFactory.GetGrain <IActiveGrainEnumeratorGrain>(0);
IEnumerable <IGrain> activeGrains = enumGrain.GetActiveGrains(typeof(ISimpleGrain)).Result;
foreach (var entry in activeGrains)
{
output.WriteLine("guid = {0}", entry.GetPrimaryKey());
}
Assert.Equal(3, activeGrains.AsQueryable().Count());
}
public async Task PreferLocalPlacementGrain_ShouldNotMigrateWhenOtherSiloKilled(string value)
{
await HostedCluster.WaitForLivenessToStabilizeAsync();
output.WriteLine("******************** Starting test ({0}) ********************", value);
TestSilosStarted(2);
foreach (SiloHandle silo in HostedCluster.GetActiveSilos())
{
NodeConfiguration cfg = silo.Silo.LocalConfig;
Assert.IsNotNull(cfg, "NodeConfiguration should be present for silo " + silo);
output.WriteLine(
"Silo {0} : Address = {1} Proxy gateway: {2}",
cfg.SiloName, cfg.Endpoint, cfg.ProxyGatewayEndpoint);
}
IPEndPoint targetSilo;
if (value == "Primary")
{
targetSilo = HostedCluster.Primary.Endpoint;
}
else
{
targetSilo = HostedCluster.Secondary.Endpoint;
}
Guid proxyKey;
IRandomPlacementTestGrain proxy;
IPEndPoint expected;
do
{
proxyKey = Guid.NewGuid();
proxy = GrainFactory.GetGrain <IRandomPlacementTestGrain>(proxyKey);
expected = await proxy.GetEndpoint();
} while (!targetSilo.Equals(expected));
output.WriteLine("Proxy grain was originally located on silo {0}", expected);
Guid grainKey = proxyKey;
await proxy.StartPreferLocalGrain(grainKey);
IPreferLocalPlacementTestGrain grain = GrainFactory.GetGrain <IPreferLocalPlacementTestGrain>(grainKey);
IPEndPoint actual = await grain.GetEndpoint();
output.WriteLine("PreferLocalPlacement grain was originally located on silo {0}", actual);
Assert.AreEqual(expected, actual,
"PreferLocalPlacement strategy should create activations on the local silo.");
SiloHandle siloToKill = HostedCluster.GetActiveSilos().First(s => !s.Endpoint.Equals(expected));
output.WriteLine("Killing other silo {0} not hosting locally placed grain", siloToKill);
HostedCluster.StopSilo(siloToKill);
IPEndPoint newActual = await grain.GetEndpoint();
output.WriteLine("PreferLocalPlacement grain is now located on silo {0}", newActual);
Assert.AreEqual(expected, newActual,
"PreferLocalPlacement strategy should not move activations when other non-hosting silo fails.");
}
public EchoTaskGrainTests(DefaultClusterFixture fixture)
: base(fixture)
{
if (HostedCluster.SecondarySilos.Count == 0)
{
HostedCluster.StartAdditionalSilo();
HostedCluster.WaitForLivenessToStabilizeAsync().Wait();
}
}
public async Task PreferLocalPlacementGrain_ShouldMigrateWhenHostSiloKilled(string value)
{
await HostedCluster.WaitForLivenessToStabilizeAsync();
output.WriteLine("******************** Starting test ({0}) ********************", value);
TestSilosStarted(2);
foreach (SiloHandle silo in HostedCluster.GetActiveSilos())
{
output.WriteLine(
"Silo {0} : Address = {1} Proxy gateway: {2}",
silo.Name, silo.SiloAddress, silo.GatewayAddress);
}
IPEndPoint targetSilo;
if (value == "Primary")
{
targetSilo = HostedCluster.Primary.SiloAddress.Endpoint;
}
else
{
targetSilo = HostedCluster.SecondarySilos.First().SiloAddress.Endpoint;
}
Guid proxyKey;
IRandomPlacementTestGrain proxy;
IPEndPoint expected;
do
{
proxyKey = Guid.NewGuid();
proxy = GrainFactory.GetGrain <IRandomPlacementTestGrain>(proxyKey);
expected = await proxy.GetEndpoint();
} while (!targetSilo.Equals(expected));
output.WriteLine("Proxy grain was originally located on silo {0}", expected);
Guid grainKey = proxyKey;
await proxy.StartPreferLocalGrain(grainKey);
IPreferLocalPlacementTestGrain grain = GrainFactory.GetGrain <IPreferLocalPlacementTestGrain>(grainKey);
IPEndPoint actual = await grain.GetEndpoint();
output.WriteLine("PreferLocalPlacement grain was originally located on silo {0}", actual);
Assert.Equal(expected, actual); // "PreferLocalPlacement strategy should create activations on the local silo."
SiloHandle siloToKill = HostedCluster.GetActiveSilos().First(s => s.SiloAddress.Endpoint.Equals(expected));
output.WriteLine("Killing silo {0} hosting locally placed grain", siloToKill);
HostedCluster.StopSilo(siloToKill);
IPEndPoint newActual = await grain.GetEndpoint();
output.WriteLine("PreferLocalPlacement grain was recreated on silo {0}", newActual);
Assert.NotEqual(expected, newActual); // "PreferLocalPlacement strategy should recreate activations on other silo if local fails."
}
public async Task SiloUngracefulShutdown_ClientOutstandingRequestsBreak()
{
var grain = HostedCluster.GrainFactory.GetGrain <ILongRunningTaskGrain <bool> >(Guid.NewGuid());
var task = grain.LongRunningTask(true, TimeSpan.FromSeconds(7));
await Task.Delay(500);
HostedCluster.KillSilo(HostedCluster.SecondarySilos[0]);
HostedCluster.KillSilo(HostedCluster.Primary);
await Assert.ThrowsAsync <SiloUnavailableException>(() => task);
}
public async Task SiloGracefulShutdown_PendingRequestTimers()
{
var grain = await GetTimerRequestGrainOnSecondary();
var promise = grain.StartAndWaitTimerTick(TimeSpan.FromSeconds(10));
await Task.Delay(500);
await HostedCluster.StopSiloAsync(HostedCluster.SecondarySilos.First());
await promise;
}
public async Task DelayedQueueRebalancingTests_2()
{
await ValidateAgentsState(2, 2, "1");
HostedCluster.StartAdditionalSilos(2);
await ValidateAgentsState(4, 2, "2");
await Task.Delay(SILO_IMMATURE_PERIOD + LEEWAY);
await ValidateAgentsState(4, 2, "3");
}
public async Task DelayedQueueRebalancingTests_2()
{
await ValidateAgentsState(2, 2, "1");
HostedCluster.RestartStoppedSecondarySilo("Secondary_2");
HostedCluster.RestartStoppedSecondarySilo("Secondary_3");
await ValidateAgentsState(4, 2, "2");
await Task.Delay(SILO_IMMATURE_PERIOD + LEEWAY);
await ValidateAgentsState(4, 2, "3");
}
public BaseTestFixture()
{
var options = new TestClusterOptions(1);
options.ClusterConfiguration.Globals.RegisterBootstrapProvider <OrniscientFilterInterceptor>("OrniscientFilterInterceptor");
//options.ClusterConfiguration.Globals.ResponseTimeout = TimeSpan.FromMinutes(1);
options.ClusterConfiguration.ApplyToAllNodes(nodeConfig => nodeConfig.MaxActiveThreads = 1);
this.HostedCluster = new TestCluster(options);
if (this.HostedCluster.Primary == null)
{
HostedCluster.Deploy();
}
}
public async Task GetDetailedHosts()
{
if (HostedCluster.SecondarySilos.Count == 0)
{
HostedCluster.StartAdditionalSilo();
await HostedCluster.WaitForLivenessToStabilizeAsync();
}
var numberOfActiveSilos = 1 + HostedCluster.SecondarySilos.Count; // Primary + secondaries
var siloStatuses = mgmtGrain.GetDetailedHosts(true).Result;
Assert.NotNull(siloStatuses);
Assert.Equal(numberOfActiveSilos, siloStatuses.Length);
}
public async Task GetHosts()
{
if (HostedCluster.SecondarySilos.Count == 0)
{
HostedCluster.StartAdditionalSilo();
await HostedCluster.WaitForLivenessToStabilizeAsync();
}
var numberOfActiveSilos = 1 + HostedCluster.SecondarySilos.Count; // Primary + secondaries
Dictionary <SiloAddress, SiloStatus> siloStatuses = mgmtGrain.GetHosts(true).Result;
Assert.IsNotNull(siloStatuses, "Got some silo statuses");
Assert.AreEqual(numberOfActiveSilos, siloStatuses.Count, "Number of silo statuses");
}
public async Task SiloGracefulShutdown_StuckTimers()
{
var grain = await GetTimerRequestGrainOnSecondary();
await grain.StartStuckTimer(TimeSpan.Zero);
await Task.Delay(TimeSpan.FromSeconds(1));
var stopwatch = Stopwatch.StartNew();
await HostedCluster.StopSiloAsync(HostedCluster.SecondarySilos.First());
stopwatch.Stop();
Assert.True(stopwatch.Elapsed > DeactivationTimeout);
}
public async Task Test_Indexing_IndexLookup3()
{
//await GrainClient.GrainFactory.DropAllIndexes<IPlayerGrain>();
if (HostedCluster.SecondarySilos.Count == 0)
{
HostedCluster.StartAdditionalSilo();
await HostedCluster.WaitForLivenessToStabilizeAsync();
}
IPlayer2Grain p1 = GrainClient.GrainFactory.GetGrain <IPlayer2Grain>(1);
await p1.SetLocation("Seattle");
//bool isLocIndexCreated = await GrainClient.GrainFactory.CreateAndRegisterIndex<HashIndexSingleBucketInterface<string, IPlayerGrain>, PlayerLocIndexGen>("__GetLocation");
//Assert.IsTrue(isLocIndexCreated);
IPlayer2Grain p2 = GrainClient.GrainFactory.GetGrain <IPlayer2Grain>(2);
IPlayer2Grain p3 = GrainClient.GrainFactory.GetGrain <IPlayer2Grain>(3);
await p2.SetLocation("Seattle");
await p3.SetLocation("San Fransisco");
IndexInterface <string, IPlayer2Grain> locIdx = GrainClient.GrainFactory.GetIndex <string, IPlayer2Grain>("__Location");
while (!await locIdx.IsAvailable())
{
Thread.Sleep(50);
}
output.WriteLine("Before check 1");
Assert.Equal(2, await IndexingTestUtils.CountPlayersStreamingIn <IPlayer2Grain, Player2Properties>("Seattle", output, DELAY_UNTIL_INDEXES_ARE_UPDATED_LAZILY));
await p2.Deactivate();
await Task.Delay(DELAY_UNTIL_INDEXES_ARE_UPDATED_LAZILY);
output.WriteLine("Before check 2");
Assert.Equal(1, await IndexingTestUtils.CountPlayersStreamingIn <IPlayer2Grain, Player2Properties>("Seattle", output, DELAY_UNTIL_INDEXES_ARE_UPDATED_LAZILY));
p2 = GrainClient.GrainFactory.GetGrain <IPlayer2Grain>(2);
output.WriteLine("Before check 3");
Assert.Equal("Seattle", await p2.GetLocation());
output.WriteLine("Before check 4");
Assert.Equal(2, await IndexingTestUtils.CountPlayersStreamingIn <IPlayer2Grain, Player2Properties>("Seattle", output, DELAY_UNTIL_INDEXES_ARE_UPDATED_LAZILY));
output.WriteLine("Done.");
}
public async Task SiloGracefulShutdown_ForwardPendingRequest()
{
var grain = await GetLongRunningTaskGrainOnSecondary <bool>();
// First call should be done on Secondary
var promisesBeforeShutdown = grain.LongRunningTask(true, TimeSpan.FromSeconds(5));
// Second call should be transfered to another silo
var promisesAfterShutdown = grain.LongRunningTask(true, TimeSpan.FromSeconds(5));
// Shutdown the silo where the grain is
await Task.Delay(500);
HostedCluster.StopSilo(HostedCluster.SecondarySilos.First());
await promisesBeforeShutdown;
await promisesAfterShutdown;
}
private MockBootstrapProvider FindBootstrapProvider(string providerName)
{
MockBootstrapProvider providerInUse = null;
List <SiloHandle> silos = HostedCluster.GetActiveSilos().ToList();
foreach (var siloHandle in silos)
{
MockBootstrapProvider provider = (MockBootstrapProvider)siloHandle.Silo.TestHook.GetBootstrapProvider(providerName);
Assert.IsNotNull(provider, "No storage provider found: Name={0} Silo={1}", providerName, siloHandle.Silo.SiloAddress);
providerInUse = provider;
}
if (providerInUse == null)
{
Assert.Fail("Cannot find active storage provider currently in use, Name={0}", providerName);
}
return(providerInUse);
}
public async Task SiloUngracefulShutdown_OutstandingRequestsBreak()
{
var grain = await GetGrainOnTargetSilo(HostedCluster.Primary);
Assert.NotNull(grain);
var target = await GetGrainOnTargetSilo(HostedCluster.SecondarySilos[0]);
Assert.NotNull(target);
var promise = grain.CallOtherLongRunningTask(target, true, TimeSpan.FromSeconds(7));
await Task.Delay(500);
await HostedCluster.KillSiloAsync(HostedCluster.SecondarySilos[0]);
await Assert.ThrowsAsync <SiloUnavailableException>(() => promise);
}
public async Task SiloUngracefulShutdown_ClientOutstandingRequestsBreak()
{
var grain = HostedCluster.GrainFactory.GetGrain <ILongRunningTaskGrain <bool> >(Guid.NewGuid());
var task = grain.LongRunningTask(true, TimeSpan.FromSeconds(7));
await Task.Delay(500);
HostedCluster.KillSilo(HostedCluster.SecondarySilos[0]);
HostedCluster.KillSilo(HostedCluster.Primary);
try
{
await task;
Assert.True(false, "The broken promise exception was not thrown");
}
catch (Exception ex)
{
Assert.Equal(typeof(SiloUnavailableException), ex.GetBaseException().GetType());
}
}
public async Task Test_Indexing_IndexLookup3()
{
//await GrainClient.GrainFactory.DropAllIndexes<IPlayerGrain>();
if (HostedCluster.SecondarySilos.Count == 0)
{
HostedCluster.StartAdditionalSilo();
await HostedCluster.WaitForLivenessToStabilizeAsync();
}
IPlayer2GrainNonFaultTolerant p1 = GrainClient.GrainFactory.GetGrain <IPlayer2GrainNonFaultTolerant>(1);
await p1.SetLocation("Seattle");
//bool isLocIndexCreated = await GrainClient.GrainFactory.CreateAndRegisterIndex<HashIndexSingleBucketInterface<string, IPlayerGrain>, PlayerLocIndexGen>("__GetLocation");
//Assert.IsTrue(isLocIndexCreated);
IPlayer2GrainNonFaultTolerant p2 = GrainClient.GrainFactory.GetGrain <IPlayer2GrainNonFaultTolerant>(2);
IPlayer2GrainNonFaultTolerant p3 = GrainClient.GrainFactory.GetGrain <IPlayer2GrainNonFaultTolerant>(3);
await p2.SetLocation("Seattle");
await p3.SetLocation("San Fransisco");
IndexInterface <string, IPlayer2GrainNonFaultTolerant> locIdx = GrainClient.GrainFactory.GetIndex <string, IPlayer2GrainNonFaultTolerant>("__Location");
while (!await locIdx.IsAvailable())
{
Thread.Sleep(50);
}
Assert.Equal(2, await IndexingTestUtils.CountPlayersStreamingIn <IPlayer2GrainNonFaultTolerant, Player2PropertiesNonFaultTolerant>("Seattle", output));
await p2.Deactivate();
Thread.Sleep(1000);
Assert.Equal(1, await IndexingTestUtils.CountPlayersStreamingIn <IPlayer2GrainNonFaultTolerant, Player2PropertiesNonFaultTolerant>("Seattle", output));
p2 = GrainClient.GrainFactory.GetGrain <IPlayer2GrainNonFaultTolerant>(2);
Assert.Equal("Seattle", await p2.GetLocation());
Assert.Equal(2, await IndexingTestUtils.CountPlayersStreamingIn <IPlayer2GrainNonFaultTolerant, Player2PropertiesNonFaultTolerant>("Seattle", output));
}
public async Task SiloGracefulShutdown_ForwardPendingRequest()
{
var grain = await GetLongRunningTaskGrainOnSecondary <bool>();
var tasks = new List <Task <string> >();
for (int i = 0; i < 100; i++)
{
tasks.Add(grain.GetRuntimeInstanceIdWithDelay(TimeSpan.FromMilliseconds(50)));
}
// Shutdown the silo where the grain is
await Task.Delay(500);
await HostedCluster.StopSiloAsync(HostedCluster.SecondarySilos.First());
var results = await Task.WhenAll(tasks);
Assert.Equal(results[99], HostedCluster.Primary.SiloAddress.ToLongString());
}
private async Task RestartSiloAfterCheckpointTestRunner(string streamNamespace, int streamCount, int eventsInStream)
{
List<Guid> streamGuids = Enumerable.Range(0, streamCount).Select(_ => Guid.NewGuid()).ToList();
try
{
await GenerateEvents(streamNamespace, streamGuids, eventsInStream, 0);
await TestingUtils.WaitUntilAsync(assertIsTrue => CheckCounters(streamNamespace, streamCount, eventsInStream, assertIsTrue), TimeSpan.FromSeconds(60));
HostedCluster.RestartSilo(HostedCluster.SecondarySilos[0]);
await HostedCluster.WaitForLivenessToStabilizeAsync();
await GenerateEvents(streamNamespace, streamGuids, eventsInStream, 0);
await TestingUtils.WaitUntilAsync(assertIsTrue => CheckCounters(streamNamespace, streamCount, eventsInStream * 2, assertIsTrue), TimeSpan.FromSeconds(90));
}
finally
{
var reporter = this.GrainFactory.GetGrain<IGeneratedEventReporterGrain>(GeneratedStreamTestConstants.ReporterId);
reporter.Reset().Ignore();
}
}
