internal void AddSiloMetric(string siloAddr, SiloComputedStats siloComputedStat, DateTime time, int machines)
{
var siloStat = new SiloRuntimeStats()
{
ExpID = this.ExperimentID,
Time = time,
Machines = machines,
Address = siloAddr.ToString(),
CPU = -1,
MemoryUsage = siloComputedStat.MemoryUsage,
//Activations = siloMetric.ActivationCount,
Activations = siloComputedStat.TotalActivations,
RecentlyUsedActivations = -1, // siloMetric.RecentlyUsedActivationCount,
ReceivedMessages = siloComputedStat.TotalRecvLocalSilo + siloComputedStat.TotalRecvNetworkSilo,
SentMessages = siloComputedStat.TotalSentLocalSilo + siloComputedStat.TotalSentNetworkSilo,
PartitionKey = this.ExperimentID,
RowKey = siloAddr.ToString() + ":" + time.ToFileTime().ToString(),
TotalRecvNetworkSilo = siloComputedStat.TotalRecvNetworkSilo,
TotalSentLocalSilo = siloComputedStat.TotalSentLocalSilo,
TotalSentNetworkSilo = siloComputedStat.TotalSentNetworkSilo,
TotalRecvLocalSilo = siloComputedStat.TotalRecvLocalSilo
};
if (this.siloMetrics == null)
{
this.siloMetrics = CreateTable("SiloMetrics");
}
var insertOperation = TableOperation.Insert(siloStat);
// Execute the insert operation.
this.siloMetrics.Execute(insertOperation);
}
public async Task RunExperiment(IGrainFactory grainFactory, string expId = "DummyExperimentID", AnalysisRootKind rootKind = AnalysisRootKind.Default)
{
try
{
this.ExperimentID = expId;
// await systemManagement.ForceActivationCollection(System.TimeSpan.MaxValue);
AnalysisClient.ErrorMessage = "OK so far";
var myStatsGrain = StatsHelper.GetStatGrain(grainFactory);
await myStatsGrain.ResetStats();
#if COMPUTE_STATS
this.series = 0;
// Repeat every 30 seconds.
IObservable <long> observable = System.Reactive.Linq.Observable.Interval(TimeSpan.FromSeconds(60));
// Token for cancelation
CancellationTokenSource source = new CancellationTokenSource();
Action action = (async() => await this.SaveUpdateInfo(myStatsGrain));
// Subscribe the obserable to the task on execution.
observable.Subscribe(x => { Task task = new Task(action); task.Start(); }, source.Token);
#endif
this.stopWatch = Stopwatch.StartNew();
AnalysisClient.ExperimentStatus = ExperimentStatus.Compiling;
this.analyzer.RootKind = rootKind;
await this.analyzer.InitializeOnDemandOrleansAnalysis();
await this.analyzer.WaitForOnDemandOrleansAnalysisToBeReady();
var compilationElapsedTime = stopWatch.ElapsedMilliseconds;
if (AnalysisClient.ExperimentStatus == ExperimentStatus.Cancelled)
{
AnalysisClient.ErrorMessage = "Cancelled by user";
return;
}
AnalysisClient.ExperimentStatus = ExperimentStatus.Running;
await this.analyzer.ContinueOnDemandOrleansAnalysis();
AnalysisClient.ExperimentStatus = ExperimentStatus.ComputingResults;
this.stopWatch.Stop();
#if COMPUTE_STATS
source.Cancel();
#endif
var totalRecvNetwork = 0L;
var totalSentLocal = 0L;
var totalSentNetwork = 0L;
var totalRecvLocal = 0L;
var totalAct = 0L;
var totalDeact = 0L;
var time = DateTime.Now;
var acummulatedPerSiloMemoryUsage = 0L;
var maxPerSiloMemoryUsage = 0L;
var acummulatedPerSiloCPUUsage = 0D;
var maxPerSiloCPUUsage = 0D;
var methods = await this.SolutionManager.GetReachableMethodsCountAsync();
this.systemManagement = grainFactory.GetGrain <IManagementGrain>(SYSTEM_MANAGEMENT_ID);
await systemManagement.ForceGarbageCollection(null);
var hosts = await systemManagement.GetHosts();
var averagePerSiloMemoryUsage = 0L;
var orleansStats = await systemManagement.GetRuntimeStatistics(null);
#if COMPUTE_STATS
//var silos = hosts.Keys.ToArray();
//this.methods = -1;
// var messageMetric = new MessageMetrics();
//var myStatsGrain = StatsHelper.GetStatGrain(grainFactory);
var silosEnumeration = await myStatsGrain.GetSilos();
var silos = silosEnumeration.ToArray();
var siloComputedStats = new SiloComputedStats[silos.Length];
var silosSize = silos.Length;
for (var i = 0; i < silos.Length; i++)
{
var silo = silos[i];
var addrString = silo; /*/silo.Endpoint.Address.ToString();*/
siloComputedStats[i] = new SiloComputedStats();
siloComputedStats[i].TotalSentLocalSilo += await myStatsGrain.GetSiloLocalMsgs(addrString);
siloComputedStats[i].TotalRecvLocalSilo += await myStatsGrain.GetSiloLocalMsgs(addrString);
siloComputedStats[i].TotalSentNetworkSilo += await myStatsGrain.GetSiloNetworkSentMsgs(addrString);
siloComputedStats[i].TotalRecvNetworkSilo += await myStatsGrain.GetSiloNetworkReceivedMsgs(addrString);
siloComputedStats[i].MemoryUsage += await myStatsGrain.GetSiloMemoryUsage(addrString);
var activationDic = await myStatsGrain.GetActivationsPerSilo(addrString);
var deactivationDic = await myStatsGrain.GetDeactivationsPerSilo(addrString);
var activations = activationDic.Sum(items => items.Value);
var deactivations = deactivationDic.Sum(items => items.Value);
siloComputedStats[i].TotalActivations += activations;
siloComputedStats[i].TotalDeactivations += deactivations;
siloComputedStats[i].TotalClientMessages += await myStatsGrain.GetTotalClientMsgsPerSilo(addrString);
// totalAct += orleansStats[i].ActivationCount;
totalAct += activations;
totalDeact += deactivations;
//AddSiloMetric(silos[i], siloComputedStats[i], time, machines);
// Save results in per silo table
if (orleansStats.Length <= i)
{
throw new IndexOutOfRangeException(String.Format("OrlenasStats Lenght is {0} and silos Lenght is {1}", orleansStats.Length, silos.Length));
}
AddSiloMetricWithOrleans(silos[i], orleansStats[i], siloComputedStats[i], time, machines);
totalSentNetwork += siloComputedStats[i].TotalSentNetworkSilo;
totalRecvNetwork += siloComputedStats[i].TotalRecvNetworkSilo;
totalSentLocal += siloComputedStats[i].TotalSentLocalSilo;
totalRecvLocal += siloComputedStats[i].TotalSentLocalSilo;
acummulatedPerSiloMemoryUsage += orleansStats[i].MemoryUsage;
acummulatedPerSiloCPUUsage += orleansStats[i].CpuUsage;
if (maxPerSiloMemoryUsage < orleansStats[i].MemoryUsage)
{
maxPerSiloMemoryUsage = orleansStats[i].MemoryUsage;
}
if (maxPerSiloCPUUsage < orleansStats[i].CpuUsage)
{
maxPerSiloCPUUsage = orleansStats[i].CpuUsage;
}
}
var avgLatency = await myStatsGrain.GetAverageLatency();
var maxLatency = await myStatsGrain.GetMaxLatency();
var maxLatencyMsg = await myStatsGrain.GetMaxLatencyMsg();
var totalMessages = await myStatsGrain.GetTotalMessages();
var clientMessages = await myStatsGrain.GetTotalClientMessages();
#else
var avgLatency = 0;
var maxLatency = 0;
var maxLatencyMsg = "";
var totalMessages = 0;
var clientMessages = 0;
for (int i = 0; i < this.machines; i++)
{
if (maxPerSiloMemoryUsage < orleansStats[i].MemoryUsage)
{
maxPerSiloMemoryUsage = orleansStats[i].MemoryUsage;
}
acummulatedPerSiloMemoryUsage += orleansStats[i].MemoryUsage;
}
var silosSize = this.machines; // hosts.Keys.Count;
#endif
averagePerSiloMemoryUsage = acummulatedPerSiloMemoryUsage / silosSize;
var testFullName = this.subject;
var results = new SubjectExperimentResults()
{
ExpID = expId,
Time = time,
Subject = testFullName,
Machines = machines,
Methods = methods,
Messages = totalMessages,
ClientMessages = clientMessages, // SolutionAnalyzer.MessageCounter,
CompilationTime = compilationElapsedTime,
ElapsedTime = stopWatch.ElapsedMilliseconds,
Activations = totalAct,
Deactivations = totalDeact,
Observations = "From web",
PartitionKey = expId + " " + testFullName, // + " " + time.ToFileTime().ToString(),
RowKey = testFullName + " " + time.ToFileTime().ToString(),
TotalRecvNetwork = totalRecvNetwork,
TotalSentLocal = totalSentLocal,
TotalSentNetwork = totalSentNetwork,
TotalRecvLocal = totalRecvLocal,
AverageLatency = avgLatency,
MaxLatency = maxLatency,
MaxLatencyMsg = maxLatencyMsg,
AveragePerSiloMemoryUsage = averagePerSiloMemoryUsage, // acummulatedPerSiloMemoryUsage / silosSize,
AveragePerSiloCPUUsage = acummulatedPerSiloCPUUsage / silosSize,
MaxPerSiloMemoryUsage = maxPerSiloMemoryUsage,
MaxPerSiloCPUUsage = maxPerSiloCPUUsage
};
// Save results in main table
this.AddSubjetResults(results);
//SaveResults(@"Y:\");
AnalysisClient.ExperimentStatus = ExperimentStatus.Ready;
AnalysisClient.ErrorMessage = "OK";
Logger.LogWarning(GrainClient.Logger, "AnalysisClient", "RunExperiment", "Finished OK");
}
catch (Exception ex)
{
var innerEx = ex;
while (innerEx is AggregateException)
{
innerEx = innerEx.InnerException;
}
AnalysisClient.ErrorMessage = "Error connecting to Orleans: " + innerEx + " at " + DateTime.Now;
AnalysisClient.ExperimentStatus = ExperimentStatus.Failed;
Logger.LogError(GrainClient.Logger, "AnalysisClient", "RunExperiment", "Finished with ERRORS {0}", ex);
throw ex;
}
}