public void GetHosts()
{
Dictionary <SiloAddress, SiloStatus> siloStatuses = mgmtGrain.GetHosts(true).Result;
Assert.IsNotNull(siloStatuses, "Got some silo statuses");
Assert.AreEqual(2, siloStatuses.Count, "Number of silo statuses");
}
protected async Task Do_Liveness_OracleTest_1()
{
Console.WriteLine("DeploymentId= {0}", DeploymentId);
SiloHandle silo3 = StartAdditionalSilo();
IManagementGrain mgmtGrain = GrainClient.GrainFactory.GetGrain <IManagementGrain>(RuntimeInterfaceConstants.SYSTEM_MANAGEMENT_ID);
Dictionary <SiloAddress, SiloStatus> statuses = await mgmtGrain.GetHosts(false);
foreach (var pair in statuses)
{
Console.WriteLine(" ######## Silo {0}, status: {1}", pair.Key, pair.Value);
Assert.AreEqual(SiloStatus.Active, pair.Value);
}
Assert.AreEqual(3, statuses.Count);
IPEndPoint address = silo3.Endpoint;
Console.WriteLine("About to stop {0}", address);
StopSilo(silo3);
// TODO: Should we be allowing time for changes to percolate?
Console.WriteLine("----------------");
statuses = await mgmtGrain.GetHosts(false);
foreach (var pair in statuses)
{
Console.WriteLine(" ######## Silo {0}, status: {1}", pair.Key, pair.Value);
IPEndPoint silo = pair.Key.Endpoint;
if (silo.Equals(address))
{
Assert.IsTrue(pair.Value.Equals(SiloStatus.ShuttingDown) ||
pair.Value.Equals(SiloStatus.Stopping) ||
pair.Value.Equals(SiloStatus.Dead),
"SiloStatus for {0} should now be ShuttingDown or Stopping or Dead instead of {1}",
silo, pair.Value);
}
else
{
Assert.AreEqual(SiloStatus.Active, pair.Value, "SiloStatus for {0}", silo);
}
}
}
protected async Task Do_Liveness_OracleTest_1()
{
output.WriteLine("ClusterId= {0}", this.HostedCluster.Options.ClusterId);
SiloHandle silo3 = this.HostedCluster.StartAdditionalSilo();
IManagementGrain mgmtGrain = this.GrainFactory.GetGrain <IManagementGrain>(0);
Dictionary <SiloAddress, SiloStatus> statuses = await mgmtGrain.GetHosts(false);
foreach (var pair in statuses)
{
output.WriteLine(" ######## Silo {0}, status: {1}", pair.Key, pair.Value);
Assert.Equal(SiloStatus.Active, pair.Value);
}
Assert.Equal(3, statuses.Count);
IPEndPoint address = silo3.SiloAddress.Endpoint;
output.WriteLine("About to stop {0}", address);
this.HostedCluster.StopSilo(silo3);
// TODO: Should we be allowing time for changes to percolate?
output.WriteLine("----------------");
statuses = await mgmtGrain.GetHosts(false);
foreach (var pair in statuses)
{
output.WriteLine(" ######## Silo {0}, status: {1}", pair.Key, pair.Value);
IPEndPoint silo = pair.Key.Endpoint;
if (silo.Equals(address))
{
Assert.True(pair.Value == SiloStatus.ShuttingDown ||
pair.Value == SiloStatus.Stopping ||
pair.Value == SiloStatus.Dead,
string.Format("SiloStatus for {0} should now be ShuttingDown or Stopping or Dead instead of {1}", silo, pair.Value));
}
else
{
Assert.Equal(SiloStatus.Active, pair.Value);
}
}
}
private static async Task PropigateStatisticsToCluster(IGrainFactory grainFactory)
{
// force the latched statistics to propigate throughout the cluster.
IManagementGrain mgmtGrain =
grainFactory.GetGrain <IManagementGrain>(0);
var hosts = await mgmtGrain.GetHosts(true);
var keys = hosts.Select(kvp => kvp.Key).ToArray();
await mgmtGrain.ForceRuntimeStatisticsCollection(keys);
}
public async Task GetHosts()
{
if (HostedCluster.SecondarySilos.Count == 0)
{
HostedCluster.StartAdditionalSilo();
await HostedCluster.WaitForLivenessToStabilizeAsync();
}
var numberOfActiveSilos = 1 + HostedCluster.SecondarySilos.Count; // Primary + secondaries
Dictionary <SiloAddress, SiloStatus> siloStatuses = mgmtGrain.GetHosts(true).Result;
Assert.NotNull(siloStatuses);
Assert.Equal(numberOfActiveSilos, siloStatuses.Count);
}
private void TestSilosStarted(int expected)
{
IManagementGrain mgmtGrain = GrainFactory.GetGrain <IManagementGrain>(0);
Dictionary <SiloAddress, SiloStatus> statuses = mgmtGrain.GetHosts(onlyActive: true).Result;
foreach (var pair in statuses)
{
logger.Info(" ######## Silo {0}, status: {1}", pair.Key, pair.Value);
Assert.Equal(
SiloStatus.Active,
pair.Value);
}
Assert.Equal(expected, statuses.Count);
}
protected void TestSilosStarted(int expected)
{
IManagementGrain mgmtGrain = GrainClient.GrainFactory.GetGrain <IManagementGrain>(1); // is nornally RuntimeInterfaceConstants.SYSTEM_MANAGEMENT_ID
Dictionary <SiloAddress, SiloStatus> statuses = mgmtGrain.GetHosts(onlyActive: true).Result;
foreach (var pair in statuses)
{
logger.Info(" ######## Silo {0}, status: {1}", pair.Key, pair.Value);
Assert.Equal(
SiloStatus.Active,
pair.Value);
}
Assert.Equal(expected, statuses.Count);
}
public async Task PingOtherSiloAsync()
{
logger.Info("IEchoGrainAsync.PingOtherSilo");
SiloAddress mySilo = Data.Address.Silo;
IManagementGrain mgmtGrain = GrainFactory.GetGrain <IManagementGrain>(0);
var silos = await mgmtGrain.GetHosts();
SiloAddress siloAddress = silos.Where(pair => !pair.Key.Equals(mySilo)).Select(pair => pair.Key).First();
logger.Info("Sending Ping to remote silo {0}", siloAddress);
await GetSiloControlReference(siloAddress).Ping("PingOtherSilo-" + siloAddress);
logger.Info("Ping reply received for {0}", siloAddress);
}
public async Task PingClusterMemberAsync()
{
logger.Info("IEchoGrainAsync.PingClusterMemberAsync");
SiloAddress mySilo = Data.Address.Silo;
IManagementGrain mgmtGrain = GrainFactory.GetGrain <IManagementGrain>(0);
var silos = await mgmtGrain.GetHosts();
SiloAddress siloAddress = silos.Where(pair => !pair.Key.Equals(mySilo)).Select(pair => pair.Key).First();
logger.Info("Sending Ping to remote silo {0}", siloAddress);
var oracle = this.internalGrainFactory.GetSystemTarget <IMembershipService>(Constants.MembershipServiceType, siloAddress);
await oracle.Ping(1);
logger.Info("Ping reply received for {0}", siloAddress);
}
public async Task PingClusterMemberAsync()
{
logger.Info("IEchoGrainAsync.PingClusterMemberAsync");
SiloAddress mySilo = Data.Address.Silo;
IManagementGrain mgmtGrain = GrainFactory.GetGrain <IManagementGrain>(RuntimeInterfaceConstants.SYSTEM_MANAGEMENT_ID);
var silos = await mgmtGrain.GetHosts();
SiloAddress siloAddress = silos.Where(pair => !pair.Key.Equals(mySilo)).Select(pair => pair.Key).First();
logger.Info("Sending Ping to remote silo {0}", siloAddress);
var oracle = InsideRuntimeClient.Current.InternalGrainFactory.GetSystemTarget <IMembershipService>(Constants.MembershipOracleId, siloAddress);
await oracle.Ping(1);
logger.Info("Ping reply received for {0}", siloAddress);
}
protected void TestSilosStarted(int expected)
{
IManagementGrain mgmtGrain = GrainClient.GrainFactory.GetGrain <IManagementGrain>(RuntimeInterfaceConstants.SYSTEM_MANAGEMENT_ID);
Dictionary <SiloAddress, SiloStatus> statuses = mgmtGrain.GetHosts(onlyActive: true).Result;
foreach (var pair in statuses)
{
Console.WriteLine(" ######## Silo {0}, status: {1}", pair.Key, pair.Value);
Assert.AreEqual(
SiloStatus.Active,
pair.Value,
"Failed to confirm start of {0} silos ({1} confirmed).",
pair.Value,
SiloStatus.Active);
}
Assert.AreEqual(expected, statuses.Count);
}
public Dictionary <SiloAddress, SiloStatus> GetHosts()
{
return(systemManagement.GetHosts().Result);
}
public async Task <string[]> GetSilos()
{
var silos = await _managementGrain.GetHosts(true);
return(silos.Keys.Select(p => p.ToString()).ToArray());
}
public async Task RunExperiment(IGrainFactory grainFactory, string expId = "DummyExperimentID", AnalysisRootKind rootKind = AnalysisRootKind.Default)
{
try
{
this.ExperimentID = expId;
// await systemManagement.ForceActivationCollection(System.TimeSpan.MaxValue);
AnalysisClient.ErrorMessage = "OK so far";
var myStatsGrain = StatsHelper.GetStatGrain(grainFactory);
await myStatsGrain.ResetStats();
#if COMPUTE_STATS
this.series = 0;
// Repeat every 30 seconds.
IObservable <long> observable = System.Reactive.Linq.Observable.Interval(TimeSpan.FromSeconds(60));
// Token for cancelation
CancellationTokenSource source = new CancellationTokenSource();
Action action = (async() => await this.SaveUpdateInfo(myStatsGrain));
// Subscribe the obserable to the task on execution.
observable.Subscribe(x => { Task task = new Task(action); task.Start(); }, source.Token);
#endif
this.stopWatch = Stopwatch.StartNew();
AnalysisClient.ExperimentStatus = ExperimentStatus.Compiling;
this.analyzer.RootKind = rootKind;
await this.analyzer.InitializeOnDemandOrleansAnalysis();
await this.analyzer.WaitForOnDemandOrleansAnalysisToBeReady();
var compilationElapsedTime = stopWatch.ElapsedMilliseconds;
if (AnalysisClient.ExperimentStatus == ExperimentStatus.Cancelled)
{
AnalysisClient.ErrorMessage = "Cancelled by user";
return;
}
AnalysisClient.ExperimentStatus = ExperimentStatus.Running;
await this.analyzer.ContinueOnDemandOrleansAnalysis();
AnalysisClient.ExperimentStatus = ExperimentStatus.ComputingResults;
this.stopWatch.Stop();
#if COMPUTE_STATS
source.Cancel();
#endif
var totalRecvNetwork = 0L;
var totalSentLocal = 0L;
var totalSentNetwork = 0L;
var totalRecvLocal = 0L;
var totalAct = 0L;
var totalDeact = 0L;
var time = DateTime.Now;
var acummulatedPerSiloMemoryUsage = 0L;
var maxPerSiloMemoryUsage = 0L;
var acummulatedPerSiloCPUUsage = 0D;
var maxPerSiloCPUUsage = 0D;
var methods = await this.SolutionManager.GetReachableMethodsCountAsync();
this.systemManagement = grainFactory.GetGrain <IManagementGrain>(SYSTEM_MANAGEMENT_ID);
await systemManagement.ForceGarbageCollection(null);
var hosts = await systemManagement.GetHosts();
var averagePerSiloMemoryUsage = 0L;
var orleansStats = await systemManagement.GetRuntimeStatistics(null);
#if COMPUTE_STATS
//var silos = hosts.Keys.ToArray();
//this.methods = -1;
// var messageMetric = new MessageMetrics();
//var myStatsGrain = StatsHelper.GetStatGrain(grainFactory);
var silosEnumeration = await myStatsGrain.GetSilos();
var silos = silosEnumeration.ToArray();
var siloComputedStats = new SiloComputedStats[silos.Length];
var silosSize = silos.Length;
for (var i = 0; i < silos.Length; i++)
{
var silo = silos[i];
var addrString = silo; /*/silo.Endpoint.Address.ToString();*/
siloComputedStats[i] = new SiloComputedStats();
siloComputedStats[i].TotalSentLocalSilo += await myStatsGrain.GetSiloLocalMsgs(addrString);
siloComputedStats[i].TotalRecvLocalSilo += await myStatsGrain.GetSiloLocalMsgs(addrString);
siloComputedStats[i].TotalSentNetworkSilo += await myStatsGrain.GetSiloNetworkSentMsgs(addrString);
siloComputedStats[i].TotalRecvNetworkSilo += await myStatsGrain.GetSiloNetworkReceivedMsgs(addrString);
siloComputedStats[i].MemoryUsage += await myStatsGrain.GetSiloMemoryUsage(addrString);
var activationDic = await myStatsGrain.GetActivationsPerSilo(addrString);
var deactivationDic = await myStatsGrain.GetDeactivationsPerSilo(addrString);
var activations = activationDic.Sum(items => items.Value);
var deactivations = deactivationDic.Sum(items => items.Value);
siloComputedStats[i].TotalActivations += activations;
siloComputedStats[i].TotalDeactivations += deactivations;
siloComputedStats[i].TotalClientMessages += await myStatsGrain.GetTotalClientMsgsPerSilo(addrString);
// totalAct += orleansStats[i].ActivationCount;
totalAct += activations;
totalDeact += deactivations;
//AddSiloMetric(silos[i], siloComputedStats[i], time, machines);
// Save results in per silo table
if (orleansStats.Length <= i)
{
throw new IndexOutOfRangeException(String.Format("OrlenasStats Lenght is {0} and silos Lenght is {1}", orleansStats.Length, silos.Length));
}
AddSiloMetricWithOrleans(silos[i], orleansStats[i], siloComputedStats[i], time, machines);
totalSentNetwork += siloComputedStats[i].TotalSentNetworkSilo;
totalRecvNetwork += siloComputedStats[i].TotalRecvNetworkSilo;
totalSentLocal += siloComputedStats[i].TotalSentLocalSilo;
totalRecvLocal += siloComputedStats[i].TotalSentLocalSilo;
acummulatedPerSiloMemoryUsage += orleansStats[i].MemoryUsage;
acummulatedPerSiloCPUUsage += orleansStats[i].CpuUsage;
if (maxPerSiloMemoryUsage < orleansStats[i].MemoryUsage)
{
maxPerSiloMemoryUsage = orleansStats[i].MemoryUsage;
}
if (maxPerSiloCPUUsage < orleansStats[i].CpuUsage)
{
maxPerSiloCPUUsage = orleansStats[i].CpuUsage;
}
}
var avgLatency = await myStatsGrain.GetAverageLatency();
var maxLatency = await myStatsGrain.GetMaxLatency();
var maxLatencyMsg = await myStatsGrain.GetMaxLatencyMsg();
var totalMessages = await myStatsGrain.GetTotalMessages();
var clientMessages = await myStatsGrain.GetTotalClientMessages();
#else
var avgLatency = 0;
var maxLatency = 0;
var maxLatencyMsg = "";
var totalMessages = 0;
var clientMessages = 0;
for (int i = 0; i < this.machines; i++)
{
if (maxPerSiloMemoryUsage < orleansStats[i].MemoryUsage)
{
maxPerSiloMemoryUsage = orleansStats[i].MemoryUsage;
}
acummulatedPerSiloMemoryUsage += orleansStats[i].MemoryUsage;
}
var silosSize = this.machines; // hosts.Keys.Count;
#endif
averagePerSiloMemoryUsage = acummulatedPerSiloMemoryUsage / silosSize;
var testFullName = this.subject;
var results = new SubjectExperimentResults()
{
ExpID = expId,
Time = time,
Subject = testFullName,
Machines = machines,
Methods = methods,
Messages = totalMessages,
ClientMessages = clientMessages, // SolutionAnalyzer.MessageCounter,
CompilationTime = compilationElapsedTime,
ElapsedTime = stopWatch.ElapsedMilliseconds,
Activations = totalAct,
Deactivations = totalDeact,
Observations = "From web",
PartitionKey = expId + " " + testFullName, // + " " + time.ToFileTime().ToString(),
RowKey = testFullName + " " + time.ToFileTime().ToString(),
TotalRecvNetwork = totalRecvNetwork,
TotalSentLocal = totalSentLocal,
TotalSentNetwork = totalSentNetwork,
TotalRecvLocal = totalRecvLocal,
AverageLatency = avgLatency,
MaxLatency = maxLatency,
MaxLatencyMsg = maxLatencyMsg,
AveragePerSiloMemoryUsage = averagePerSiloMemoryUsage, // acummulatedPerSiloMemoryUsage / silosSize,
AveragePerSiloCPUUsage = acummulatedPerSiloCPUUsage / silosSize,
MaxPerSiloMemoryUsage = maxPerSiloMemoryUsage,
MaxPerSiloCPUUsage = maxPerSiloCPUUsage
};
// Save results in main table
this.AddSubjetResults(results);
//SaveResults(@"Y:\");
AnalysisClient.ExperimentStatus = ExperimentStatus.Ready;
AnalysisClient.ErrorMessage = "OK";
Logger.LogWarning(GrainClient.Logger, "AnalysisClient", "RunExperiment", "Finished OK");
}
catch (Exception ex)
{
var innerEx = ex;
while (innerEx is AggregateException)
{
innerEx = innerEx.InnerException;
}
AnalysisClient.ErrorMessage = "Error connecting to Orleans: " + innerEx + " at " + DateTime.Now;
AnalysisClient.ExperimentStatus = ExperimentStatus.Failed;
Logger.LogError(GrainClient.Logger, "AnalysisClient", "RunExperiment", "Finished with ERRORS {0}", ex);
throw ex;
}
}