// publish stats to a grain void ProcessStats(object state) { var providerRuntime = state as IProviderRuntime; var dashboardGrain = providerRuntime.GrainFactory.GetGrain <IDashboardGrain>(0); // flush the dictionary GrainTraceEntry[] data; lock (sync) { data = this.grainTrace.Values.ToArray(); this.grainTrace.Clear(); } foreach (var item in data) { item.Grain = TypeFormatter.Parse(item.Grain); } try { Dispatch(async() => { await dashboardGrain.SubmitTracing(siloAddress, data).ConfigureAwait(false); return(null); }).Wait(30000); } catch (Exception ex) { this.Logger.Log(100001, Severity.Warning, "Exception thrown sending tracing to dashboard grain", new object[0], ex); } }
private void ProcessStats(object state) { if (dispatcher.CanDispatch()) { var currentTrace = Interlocked.Exchange(ref grainTrace, new ConcurrentDictionary <string, SiloGrainTraceEntry>()); var items = currentTrace.Values.ToArray(); foreach (var item in items) { item.Grain = TypeFormatter.Parse(item.Grain); } try { dispatcher.DispatchAsync(() => { this.dashboardGrain = this.dashboardGrain ?? grainFactory.GetGrain <IDashboardGrain>(0); return(dashboardGrain.SubmitTracing(siloAddress, items.AsImmutable())); }).Ignore(); } catch (Exception ex) { logger.LogWarning(100001, ex, "Exception thrown sending tracing to dashboard grain"); } } }
private void ProcessStats(object state) { if (dispatcher.CanDispatch()) { var currentTrace = Interlocked.Exchange(ref grainTrace, new ConcurrentDictionary <string, GrainTraceEntry>()); var items = currentTrace.Values.ToArray(); foreach (var item in items) { item.Grain = TypeFormatter.Parse(item.Grain); } try { dispatcher.DispatchAsync(async() => { var dashboardGrain = grainFactory.GetGrain <IDashboardGrain>(0); await dashboardGrain.SubmitTracing(siloAddress, items).ConfigureAwait(false); }).Wait(30000); } catch (Exception ex) { logger.LogWarning(100001, "Exception thrown sending tracing to dashboard grain", ex); } } }
async Task Callback(object _) { var metricsGrain = this.GrainFactory.GetGrain <IManagementGrain>(0); var activationCountTask = metricsGrain.GetTotalActivationCount(); var hostsTask = metricsGrain.GetDetailedHosts(true); var simpleGrainStatsTask = metricsGrain.GetSimpleGrainStatistics(); await Task.WhenAll(activationCountTask, hostsTask, simpleGrainStatsTask); this.Counters.TotalActivationCount = activationCountTask.Result; this.Counters.TotalActiveHostCount = hostsTask.Result.Count(x => x.Status == SiloStatus.Active); this.Counters.TotalActivationCountHistory.Enqueue(activationCountTask.Result); this.Counters.TotalActiveHostCountHistory.Enqueue(this.Counters.TotalActiveHostCount); while (this.Counters.TotalActivationCountHistory.Count > Dashboard.HistoryLength) { this.Counters.TotalActivationCountHistory.Dequeue(); } while (this.Counters.TotalActiveHostCountHistory.Count > Dashboard.HistoryLength) { this.Counters.TotalActiveHostCountHistory.Dequeue(); } // TODO - whatever max elapsed time var elapsedTime = Math.Min((DateTime.UtcNow - this.StartTime).TotalSeconds, 100); this.Counters.Hosts = hostsTask.Result.Select(x => new SiloDetails { FaultZone = x.FaultZone, HostName = x.HostName, IAmAliveTime = x.IAmAliveTime.ToString("o"), ProxyPort = x.ProxyPort, RoleName = x.RoleName, SiloAddress = x.SiloAddress.ToParsableString(), SiloName = x.SiloName, StartTime = x.StartTime.ToString("o"), Status = x.Status.ToString(), UpdateZone = x.UpdateZone }).ToArray(); this.Counters.SimpleGrainStats = simpleGrainStatsTask.Result.Select(x => { var grainName = TypeFormatter.Parse(x.GrainType); return(new SimpleGrainStatisticCounter { ActivationCount = x.ActivationCount, GrainType = grainName, SiloAddress = x.SiloAddress.ToParsableString(), TotalAwaitTime = this.history.Where(n => n.Grain == grainName && n.SiloAddress == x.SiloAddress.ToParsableString()).SumZero(n => n.ElapsedTime), TotalCalls = this.history.Where(n => n.Grain == grainName && n.SiloAddress == x.SiloAddress.ToParsableString()).SumZero(n => n.Count), TotalExceptions = this.history.Where(n => n.Grain == grainName && n.SiloAddress == x.SiloAddress.ToParsableString()).SumZero(n => n.ExceptionCount), TotalSeconds = elapsedTime }); }).ToArray(); }
internal void RecalculateCounters(int activationCount, SiloDetails[] hosts, IList <SimpleGrainStatistic> simpleGrainStatistics) { this.Counters.TotalActivationCount = activationCount; this.Counters.TotalActiveHostCount = hosts.Count(x => x.SiloStatus == SiloStatus.Active); this.Counters.TotalActivationCountHistory.Enqueue(activationCount); this.Counters.TotalActiveHostCountHistory.Enqueue(this.Counters.TotalActiveHostCount); while (this.Counters.TotalActivationCountHistory.Count > Dashboard.HistoryLength) { this.Counters.TotalActivationCountHistory.Dequeue(); } while (this.Counters.TotalActiveHostCountHistory.Count > Dashboard.HistoryLength) { this.Counters.TotalActiveHostCountHistory.Dequeue(); } // TODO - whatever max elapsed time var elapsedTime = Math.Min((DateTime.UtcNow - this.StartTime).TotalSeconds, 100); this.Counters.Hosts = hosts; var aggregatedTotals = this.history .GroupBy(x => new GrainSiloKey(x.Grain, x.SiloAddress)) .ToDictionary(g => g.Key, g => new AggregatedGrainTotals { TotalAwaitTime = g.Sum(x => x.ElapsedTime), TotalCalls = g.Sum(x => x.Count), TotalExceptions = g.Sum(x => x.ExceptionCount) }); this.Counters.SimpleGrainStats = simpleGrainStatistics.Select(x => { var grainName = TypeFormatter.Parse(x.GrainType); var siloAddress = x.SiloAddress.ToParsableString(); AggregatedGrainTotals totals; if (!aggregatedTotals.TryGetValue(new GrainSiloKey(grainName, siloAddress), out totals)) { totals = new AggregatedGrainTotals(); } return(new SimpleGrainStatisticCounter { ActivationCount = x.ActivationCount, GrainType = grainName, SiloAddress = x.SiloAddress.ToParsableString(), TotalAwaitTime = totals.TotalAwaitTime, TotalCalls = totals.TotalCalls, TotalExceptions = totals.TotalExceptions, TotalSeconds = elapsedTime }); }).ToArray(); }
internal void RecalculateCounters(int activationCount, SiloDetails[] hosts, IList <SimpleGrainStatistic> simpleGrainStatistics) { counters.TotalActivationCount = activationCount; counters.TotalActiveHostCount = hosts.Count(x => x.SiloStatus == SiloStatus.Active); counters.TotalActivationCountHistory = counters.TotalActivationCountHistory.Enqueue(activationCount).Dequeue(); counters.TotalActiveHostCountHistory = counters.TotalActiveHostCountHistory.Enqueue(counters.TotalActiveHostCount).Dequeue(); // TODO - whatever max elapsed time var elapsedTime = Math.Min((DateTime.UtcNow - startTime).TotalSeconds, 100); counters.Hosts = hosts; var aggregatedTotals = history.GroupByGrainAndSilo().ToLookup(x => (x.Grain, x.SiloAddress)); //var aggregatedTotals = history.ToLookup(x => (x.Grain, x.SiloAddress)); counters.SimpleGrainStats = simpleGrainStatistics.Select(x => { var grainName = TypeFormatter.Parse(x.GrainType); var siloAddress = x.SiloAddress.ToParsableString(); var result = new SimpleGrainStatisticCounter { ActivationCount = x.ActivationCount, GrainType = grainName, SiloAddress = siloAddress, TotalSeconds = elapsedTime }; foreach (var item in aggregatedTotals[(grainName, siloAddress)]) { result.TotalAwaitTime += item.ElapsedTime; result.TotalCalls += item.Count; result.TotalExceptions += item.ExceptionCount; } return(result); }).ToArray();
public Task SubmitTracing(string siloIdentity, GrainTraceEntry[] grainTrace) { var now = DateTime.UtcNow; foreach (var entry in grainTrace) { // sync clocks entry.Period = now; } // fill in any previously captured methods which aren't in this reporting window var allGrainTrace = new List <GrainTraceEntry>(grainTrace); var values = this.history.Where(x => x.SiloAddress == siloIdentity).GroupBy(x => x.GrainAndMethod).Select(x => x.First()); foreach (var value in values) { if (!grainTrace.Any(x => x.GrainAndMethod == value.GrainAndMethod)) { allGrainTrace.Add(new GrainTraceEntry { Count = 0, ElapsedTime = 0, Grain = TypeFormatter.Parse(value.Grain), Method = value.Method, Period = now, SiloAddress = siloIdentity }); } } var retirementWindow = DateTime.UtcNow.AddSeconds(-100); history.AddRange(allGrainTrace); history.RemoveAll(x => x.Period < retirementWindow); return(TaskDone.Done); }
internal void RecalculateCounters(int activationCount, SiloDetails[] hosts, IList <DetailedGrainStatistic> simpleGrainStatistics) { counters.TotalActivationCount = activationCount; counters.TotalActiveHostCount = hosts.Count(x => x.SiloStatus == SiloStatus.Active); counters.TotalActivationCountHistory = counters.TotalActivationCountHistory.Enqueue(activationCount).Dequeue(); counters.TotalActiveHostCountHistory = counters.TotalActiveHostCountHistory.Enqueue(counters.TotalActiveHostCount).Dequeue(); // TODO - whatever max elapsed time var elapsedTime = Math.Min((DateTime.UtcNow - startTime).TotalSeconds, 100); foreach (var item in counters.Hosts) { foreach (var item2 in hosts) { if (item2.SiloAddress == item.SiloAddress) { item2.ExtraData = item.ExtraData; } } } counters.Hosts = hosts; var aggregatedTotals = history.GroupByGrainAndSilo().ToLookup(x => (x.Grain, x.SiloAddress)); var grainstatts = simpleGrainStatistics.Select(x => { var grainName = TypeFormatter.Parse(x.GrainType); if (MultiGrainAttribute.IsRecordClass(x.GrainType)) { if (x.GrainIdentity.PrimaryKeyString != null) { grainName += $":{x.GrainIdentity.PrimaryKeyString}"; } else { grainName += $":{x.GrainIdentity.PrimaryKeyLong}"; } } var siloAddress = x.SiloAddress.ToParsableString(); var result = new SimpleGrainStatisticCounter { //ActivationCount = x.ActivationCount, ActivationCount = 1, GrainType = grainName, SiloAddress = siloAddress, TotalSeconds = elapsedTime }; foreach (var item in aggregatedTotals[(grainName, siloAddress)]) { result.TotalAwaitTime += item.ElapsedTime; result.TotalCalls += item.Count; result.TotalExceptions += item.ExceptionCount; } return(result); }).ToArray();