public void GetHosts() { Dictionary <SiloAddress, SiloStatus> siloStatuses = mgmtGrain.GetHosts(true).Result; Assert.IsNotNull(siloStatuses, "Got some silo statuses"); Assert.AreEqual(2, siloStatuses.Count, "Number of silo statuses"); }
protected async Task Do_Liveness_OracleTest_1() { Console.WriteLine("DeploymentId= {0}", DeploymentId); SiloHandle silo3 = StartAdditionalSilo(); IManagementGrain mgmtGrain = GrainClient.GrainFactory.GetGrain <IManagementGrain>(RuntimeInterfaceConstants.SYSTEM_MANAGEMENT_ID); Dictionary <SiloAddress, SiloStatus> statuses = await mgmtGrain.GetHosts(false); foreach (var pair in statuses) { Console.WriteLine(" ######## Silo {0}, status: {1}", pair.Key, pair.Value); Assert.AreEqual(SiloStatus.Active, pair.Value); } Assert.AreEqual(3, statuses.Count); IPEndPoint address = silo3.Endpoint; Console.WriteLine("About to stop {0}", address); StopSilo(silo3); // TODO: Should we be allowing time for changes to percolate? Console.WriteLine("----------------"); statuses = await mgmtGrain.GetHosts(false); foreach (var pair in statuses) { Console.WriteLine(" ######## Silo {0}, status: {1}", pair.Key, pair.Value); IPEndPoint silo = pair.Key.Endpoint; if (silo.Equals(address)) { Assert.IsTrue(pair.Value.Equals(SiloStatus.ShuttingDown) || pair.Value.Equals(SiloStatus.Stopping) || pair.Value.Equals(SiloStatus.Dead), "SiloStatus for {0} should now be ShuttingDown or Stopping or Dead instead of {1}", silo, pair.Value); } else { Assert.AreEqual(SiloStatus.Active, pair.Value, "SiloStatus for {0}", silo); } } }
protected async Task Do_Liveness_OracleTest_1() { output.WriteLine("ClusterId= {0}", this.HostedCluster.Options.ClusterId); SiloHandle silo3 = this.HostedCluster.StartAdditionalSilo(); IManagementGrain mgmtGrain = this.GrainFactory.GetGrain <IManagementGrain>(0); Dictionary <SiloAddress, SiloStatus> statuses = await mgmtGrain.GetHosts(false); foreach (var pair in statuses) { output.WriteLine(" ######## Silo {0}, status: {1}", pair.Key, pair.Value); Assert.Equal(SiloStatus.Active, pair.Value); } Assert.Equal(3, statuses.Count); IPEndPoint address = silo3.SiloAddress.Endpoint; output.WriteLine("About to stop {0}", address); this.HostedCluster.StopSilo(silo3); // TODO: Should we be allowing time for changes to percolate? output.WriteLine("----------------"); statuses = await mgmtGrain.GetHosts(false); foreach (var pair in statuses) { output.WriteLine(" ######## Silo {0}, status: {1}", pair.Key, pair.Value); IPEndPoint silo = pair.Key.Endpoint; if (silo.Equals(address)) { Assert.True(pair.Value == SiloStatus.ShuttingDown || pair.Value == SiloStatus.Stopping || pair.Value == SiloStatus.Dead, string.Format("SiloStatus for {0} should now be ShuttingDown or Stopping or Dead instead of {1}", silo, pair.Value)); } else { Assert.Equal(SiloStatus.Active, pair.Value); } } }
private static async Task PropigateStatisticsToCluster(IGrainFactory grainFactory) { // force the latched statistics to propigate throughout the cluster. IManagementGrain mgmtGrain = grainFactory.GetGrain <IManagementGrain>(0); var hosts = await mgmtGrain.GetHosts(true); var keys = hosts.Select(kvp => kvp.Key).ToArray(); await mgmtGrain.ForceRuntimeStatisticsCollection(keys); }
public async Task GetHosts() { if (HostedCluster.SecondarySilos.Count == 0) { HostedCluster.StartAdditionalSilo(); await HostedCluster.WaitForLivenessToStabilizeAsync(); } var numberOfActiveSilos = 1 + HostedCluster.SecondarySilos.Count; // Primary + secondaries Dictionary <SiloAddress, SiloStatus> siloStatuses = mgmtGrain.GetHosts(true).Result; Assert.NotNull(siloStatuses); Assert.Equal(numberOfActiveSilos, siloStatuses.Count); }
private void TestSilosStarted(int expected) { IManagementGrain mgmtGrain = GrainFactory.GetGrain <IManagementGrain>(0); Dictionary <SiloAddress, SiloStatus> statuses = mgmtGrain.GetHosts(onlyActive: true).Result; foreach (var pair in statuses) { logger.Info(" ######## Silo {0}, status: {1}", pair.Key, pair.Value); Assert.Equal( SiloStatus.Active, pair.Value); } Assert.Equal(expected, statuses.Count); }
protected void TestSilosStarted(int expected) { IManagementGrain mgmtGrain = GrainClient.GrainFactory.GetGrain <IManagementGrain>(1); // is nornally RuntimeInterfaceConstants.SYSTEM_MANAGEMENT_ID Dictionary <SiloAddress, SiloStatus> statuses = mgmtGrain.GetHosts(onlyActive: true).Result; foreach (var pair in statuses) { logger.Info(" ######## Silo {0}, status: {1}", pair.Key, pair.Value); Assert.Equal( SiloStatus.Active, pair.Value); } Assert.Equal(expected, statuses.Count); }
public async Task PingOtherSiloAsync() { logger.Info("IEchoGrainAsync.PingOtherSilo"); SiloAddress mySilo = Data.Address.Silo; IManagementGrain mgmtGrain = GrainFactory.GetGrain <IManagementGrain>(0); var silos = await mgmtGrain.GetHosts(); SiloAddress siloAddress = silos.Where(pair => !pair.Key.Equals(mySilo)).Select(pair => pair.Key).First(); logger.Info("Sending Ping to remote silo {0}", siloAddress); await GetSiloControlReference(siloAddress).Ping("PingOtherSilo-" + siloAddress); logger.Info("Ping reply received for {0}", siloAddress); }
public async Task PingClusterMemberAsync() { logger.Info("IEchoGrainAsync.PingClusterMemberAsync"); SiloAddress mySilo = Data.Address.Silo; IManagementGrain mgmtGrain = GrainFactory.GetGrain <IManagementGrain>(0); var silos = await mgmtGrain.GetHosts(); SiloAddress siloAddress = silos.Where(pair => !pair.Key.Equals(mySilo)).Select(pair => pair.Key).First(); logger.Info("Sending Ping to remote silo {0}", siloAddress); var oracle = this.internalGrainFactory.GetSystemTarget <IMembershipService>(Constants.MembershipServiceType, siloAddress); await oracle.Ping(1); logger.Info("Ping reply received for {0}", siloAddress); }
public async Task PingClusterMemberAsync() { logger.Info("IEchoGrainAsync.PingClusterMemberAsync"); SiloAddress mySilo = Data.Address.Silo; IManagementGrain mgmtGrain = GrainFactory.GetGrain <IManagementGrain>(RuntimeInterfaceConstants.SYSTEM_MANAGEMENT_ID); var silos = await mgmtGrain.GetHosts(); SiloAddress siloAddress = silos.Where(pair => !pair.Key.Equals(mySilo)).Select(pair => pair.Key).First(); logger.Info("Sending Ping to remote silo {0}", siloAddress); var oracle = InsideRuntimeClient.Current.InternalGrainFactory.GetSystemTarget <IMembershipService>(Constants.MembershipOracleId, siloAddress); await oracle.Ping(1); logger.Info("Ping reply received for {0}", siloAddress); }
protected void TestSilosStarted(int expected) { IManagementGrain mgmtGrain = GrainClient.GrainFactory.GetGrain <IManagementGrain>(RuntimeInterfaceConstants.SYSTEM_MANAGEMENT_ID); Dictionary <SiloAddress, SiloStatus> statuses = mgmtGrain.GetHosts(onlyActive: true).Result; foreach (var pair in statuses) { Console.WriteLine(" ######## Silo {0}, status: {1}", pair.Key, pair.Value); Assert.AreEqual( SiloStatus.Active, pair.Value, "Failed to confirm start of {0} silos ({1} confirmed).", pair.Value, SiloStatus.Active); } Assert.AreEqual(expected, statuses.Count); }
public Dictionary <SiloAddress, SiloStatus> GetHosts() { return(systemManagement.GetHosts().Result); }
public async Task <string[]> GetSilos() { var silos = await _managementGrain.GetHosts(true); return(silos.Keys.Select(p => p.ToString()).ToArray()); }
public async Task RunExperiment(IGrainFactory grainFactory, string expId = "DummyExperimentID", AnalysisRootKind rootKind = AnalysisRootKind.Default) { try { this.ExperimentID = expId; // await systemManagement.ForceActivationCollection(System.TimeSpan.MaxValue); AnalysisClient.ErrorMessage = "OK so far"; var myStatsGrain = StatsHelper.GetStatGrain(grainFactory); await myStatsGrain.ResetStats(); #if COMPUTE_STATS this.series = 0; // Repeat every 30 seconds. IObservable <long> observable = System.Reactive.Linq.Observable.Interval(TimeSpan.FromSeconds(60)); // Token for cancelation CancellationTokenSource source = new CancellationTokenSource(); Action action = (async() => await this.SaveUpdateInfo(myStatsGrain)); // Subscribe the obserable to the task on execution. observable.Subscribe(x => { Task task = new Task(action); task.Start(); }, source.Token); #endif this.stopWatch = Stopwatch.StartNew(); AnalysisClient.ExperimentStatus = ExperimentStatus.Compiling; this.analyzer.RootKind = rootKind; await this.analyzer.InitializeOnDemandOrleansAnalysis(); await this.analyzer.WaitForOnDemandOrleansAnalysisToBeReady(); var compilationElapsedTime = stopWatch.ElapsedMilliseconds; if (AnalysisClient.ExperimentStatus == ExperimentStatus.Cancelled) { AnalysisClient.ErrorMessage = "Cancelled by user"; return; } AnalysisClient.ExperimentStatus = ExperimentStatus.Running; await this.analyzer.ContinueOnDemandOrleansAnalysis(); AnalysisClient.ExperimentStatus = ExperimentStatus.ComputingResults; this.stopWatch.Stop(); #if COMPUTE_STATS source.Cancel(); #endif var totalRecvNetwork = 0L; var totalSentLocal = 0L; var totalSentNetwork = 0L; var totalRecvLocal = 0L; var totalAct = 0L; var totalDeact = 0L; var time = DateTime.Now; var acummulatedPerSiloMemoryUsage = 0L; var maxPerSiloMemoryUsage = 0L; var acummulatedPerSiloCPUUsage = 0D; var maxPerSiloCPUUsage = 0D; var methods = await this.SolutionManager.GetReachableMethodsCountAsync(); this.systemManagement = grainFactory.GetGrain <IManagementGrain>(SYSTEM_MANAGEMENT_ID); await systemManagement.ForceGarbageCollection(null); var hosts = await systemManagement.GetHosts(); var averagePerSiloMemoryUsage = 0L; var orleansStats = await systemManagement.GetRuntimeStatistics(null); #if COMPUTE_STATS //var silos = hosts.Keys.ToArray(); //this.methods = -1; // var messageMetric = new MessageMetrics(); //var myStatsGrain = StatsHelper.GetStatGrain(grainFactory); var silosEnumeration = await myStatsGrain.GetSilos(); var silos = silosEnumeration.ToArray(); var siloComputedStats = new SiloComputedStats[silos.Length]; var silosSize = silos.Length; for (var i = 0; i < silos.Length; i++) { var silo = silos[i]; var addrString = silo; /*/silo.Endpoint.Address.ToString();*/ siloComputedStats[i] = new SiloComputedStats(); siloComputedStats[i].TotalSentLocalSilo += await myStatsGrain.GetSiloLocalMsgs(addrString); siloComputedStats[i].TotalRecvLocalSilo += await myStatsGrain.GetSiloLocalMsgs(addrString); siloComputedStats[i].TotalSentNetworkSilo += await myStatsGrain.GetSiloNetworkSentMsgs(addrString); siloComputedStats[i].TotalRecvNetworkSilo += await myStatsGrain.GetSiloNetworkReceivedMsgs(addrString); siloComputedStats[i].MemoryUsage += await myStatsGrain.GetSiloMemoryUsage(addrString); var activationDic = await myStatsGrain.GetActivationsPerSilo(addrString); var deactivationDic = await myStatsGrain.GetDeactivationsPerSilo(addrString); var activations = activationDic.Sum(items => items.Value); var deactivations = deactivationDic.Sum(items => items.Value); siloComputedStats[i].TotalActivations += activations; siloComputedStats[i].TotalDeactivations += deactivations; siloComputedStats[i].TotalClientMessages += await myStatsGrain.GetTotalClientMsgsPerSilo(addrString); // totalAct += orleansStats[i].ActivationCount; totalAct += activations; totalDeact += deactivations; //AddSiloMetric(silos[i], siloComputedStats[i], time, machines); // Save results in per silo table if (orleansStats.Length <= i) { throw new IndexOutOfRangeException(String.Format("OrlenasStats Lenght is {0} and silos Lenght is {1}", orleansStats.Length, silos.Length)); } AddSiloMetricWithOrleans(silos[i], orleansStats[i], siloComputedStats[i], time, machines); totalSentNetwork += siloComputedStats[i].TotalSentNetworkSilo; totalRecvNetwork += siloComputedStats[i].TotalRecvNetworkSilo; totalSentLocal += siloComputedStats[i].TotalSentLocalSilo; totalRecvLocal += siloComputedStats[i].TotalSentLocalSilo; acummulatedPerSiloMemoryUsage += orleansStats[i].MemoryUsage; acummulatedPerSiloCPUUsage += orleansStats[i].CpuUsage; if (maxPerSiloMemoryUsage < orleansStats[i].MemoryUsage) { maxPerSiloMemoryUsage = orleansStats[i].MemoryUsage; } if (maxPerSiloCPUUsage < orleansStats[i].CpuUsage) { maxPerSiloCPUUsage = orleansStats[i].CpuUsage; } } var avgLatency = await myStatsGrain.GetAverageLatency(); var maxLatency = await myStatsGrain.GetMaxLatency(); var maxLatencyMsg = await myStatsGrain.GetMaxLatencyMsg(); var totalMessages = await myStatsGrain.GetTotalMessages(); var clientMessages = await myStatsGrain.GetTotalClientMessages(); #else var avgLatency = 0; var maxLatency = 0; var maxLatencyMsg = ""; var totalMessages = 0; var clientMessages = 0; for (int i = 0; i < this.machines; i++) { if (maxPerSiloMemoryUsage < orleansStats[i].MemoryUsage) { maxPerSiloMemoryUsage = orleansStats[i].MemoryUsage; } acummulatedPerSiloMemoryUsage += orleansStats[i].MemoryUsage; } var silosSize = this.machines; // hosts.Keys.Count; #endif averagePerSiloMemoryUsage = acummulatedPerSiloMemoryUsage / silosSize; var testFullName = this.subject; var results = new SubjectExperimentResults() { ExpID = expId, Time = time, Subject = testFullName, Machines = machines, Methods = methods, Messages = totalMessages, ClientMessages = clientMessages, // SolutionAnalyzer.MessageCounter, CompilationTime = compilationElapsedTime, ElapsedTime = stopWatch.ElapsedMilliseconds, Activations = totalAct, Deactivations = totalDeact, Observations = "From web", PartitionKey = expId + " " + testFullName, // + " " + time.ToFileTime().ToString(), RowKey = testFullName + " " + time.ToFileTime().ToString(), TotalRecvNetwork = totalRecvNetwork, TotalSentLocal = totalSentLocal, TotalSentNetwork = totalSentNetwork, TotalRecvLocal = totalRecvLocal, AverageLatency = avgLatency, MaxLatency = maxLatency, MaxLatencyMsg = maxLatencyMsg, AveragePerSiloMemoryUsage = averagePerSiloMemoryUsage, // acummulatedPerSiloMemoryUsage / silosSize, AveragePerSiloCPUUsage = acummulatedPerSiloCPUUsage / silosSize, MaxPerSiloMemoryUsage = maxPerSiloMemoryUsage, MaxPerSiloCPUUsage = maxPerSiloCPUUsage }; // Save results in main table this.AddSubjetResults(results); //SaveResults(@"Y:\"); AnalysisClient.ExperimentStatus = ExperimentStatus.Ready; AnalysisClient.ErrorMessage = "OK"; Logger.LogWarning(GrainClient.Logger, "AnalysisClient", "RunExperiment", "Finished OK"); } catch (Exception ex) { var innerEx = ex; while (innerEx is AggregateException) { innerEx = innerEx.InnerException; } AnalysisClient.ErrorMessage = "Error connecting to Orleans: " + innerEx + " at " + DateTime.Now; AnalysisClient.ExperimentStatus = ExperimentStatus.Failed; Logger.LogError(GrainClient.Logger, "AnalysisClient", "RunExperiment", "Finished with ERRORS {0}", ex); throw ex; } }