private async Task <ICollectionTestGrain> PickGrainInNonPrimary()
        {
            for (int i = 0; i < 500; i++)
            {
                if (i % 30 == 29)
                {
                    await Task.Delay(1000);               // give some extra time to stabilize if it can't find a suitable grain
                }
                // Create grain such that:
                // Its directory owner is not the Gateway silo. This way Gateway will use its directory cache.
                // Its activation is located on the non Gateway silo as well.
                ICollectionTestGrain grain  = this.testCluster.GrainFactory.GetGrain <ICollectionTestGrain>(i);
                GrainId     grainId         = ((GrainReference)await grain.GetGrainReference()).GrainId;
                SiloAddress primaryForGrain = (await TestUtils.GetDetailedGrainReport(this.testCluster.InternalGrainFactory, grainId, this.testCluster.Primary)).PrimaryForGrain;
                if (primaryForGrain.Equals(this.testCluster.Primary.SiloAddress))
                {
                    continue;
                }
                string siloHostingActivation = await grain.GetRuntimeInstanceId();

                if (this.testCluster.Primary.SiloAddress.ToLongString().Equals(siloHostingActivation))
                {
                    continue;
                }
                this.output.WriteLine("\nCreated grain with key {0} whose primary directory owner is silo {1} and which was activated on silo {2}\n", i, primaryForGrain.ToLongString(), siloHostingActivation);
                return(grain);
            }

            Assert.True(testCluster.GetActiveSilos().Count() > 1, "This logic requires at least 1 non-primary active silo");
            Assert.True(false, "Could not find a grain that activates on a non-primary silo, and has the partition be also managed by a non-primary silo");
            return(null);
        }
        private async Task <ICollectionTestGrain> PickGrain()
        {
            for (int i = 0; i < 100; i++)
            {
                // Create grain such that:
                // Its directory owner is not the Gateway silo. This way Gateway will use its directory cache.
                // Its activation is located on the non Gateway silo as well.
                ICollectionTestGrain grain  = GrainClient.GrainFactory.GetGrain <ICollectionTestGrain>(i);
                GrainId     grainId         = ((GrainReference)await grain.GetGrainReference()).GrainId;
                SiloAddress primaryForGrain = (await TestUtils.GetDetailedGrainReport(grainId, testCluster.Primary)).PrimaryForGrain;
                if (primaryForGrain.Equals(testCluster.Primary.SiloAddress))
                {
                    continue;
                }
                string siloHostingActivation = await grain.GetRuntimeInstanceId();

                if (testCluster.Primary.SiloAddress.ToLongString().Equals(siloHostingActivation))
                {
                    continue;
                }
                logger.Info("\nCreated grain with key {0} whose primary directory owner is silo {1} and which was activated on silo {2}\n", i, primaryForGrain.ToLongString(), siloHostingActivation);
                return(grain);
            }
            return(null);
        }