// publish stats to a grain
void ProcessStats(object state)
{
var providerRuntime = state as IProviderRuntime;
var dashboardGrain = providerRuntime.GrainFactory.GetGrain <IDashboardGrain>(0);
// flush the dictionary
GrainTraceEntry[] data;
lock (sync)
{
data = this.grainTrace.Values.ToArray();
this.grainTrace.Clear();
}
foreach (var item in data)
{
item.Grain = TypeFormatter.Parse(item.Grain);
}
try
{
Dispatch(async() =>
{
await dashboardGrain.SubmitTracing(siloAddress, data).ConfigureAwait(false);
return(null);
}).Wait(30000);
}
catch (Exception ex)
{
this.Logger.Log(100001, Severity.Warning, "Exception thrown sending tracing to dashboard grain", new object[0], ex);
}
}
private void ProcessStats(object state)
{
if (dispatcher.CanDispatch())
{
var currentTrace = Interlocked.Exchange(ref grainTrace, new ConcurrentDictionary <string, SiloGrainTraceEntry>());
var items = currentTrace.Values.ToArray();
foreach (var item in items)
{
item.Grain = TypeFormatter.Parse(item.Grain);
}
try
{
dispatcher.DispatchAsync(() =>
{
this.dashboardGrain = this.dashboardGrain ?? grainFactory.GetGrain <IDashboardGrain>(0);
return(dashboardGrain.SubmitTracing(siloAddress, items.AsImmutable()));
}).Ignore();
}
catch (Exception ex)
{
logger.LogWarning(100001, ex, "Exception thrown sending tracing to dashboard grain");
}
}
}
private void ProcessStats(object state)
{
if (dispatcher.CanDispatch())
{
var currentTrace = Interlocked.Exchange(ref grainTrace, new ConcurrentDictionary <string, GrainTraceEntry>());
var items = currentTrace.Values.ToArray();
foreach (var item in items)
{
item.Grain = TypeFormatter.Parse(item.Grain);
}
try
{
dispatcher.DispatchAsync(async() =>
{
var dashboardGrain = grainFactory.GetGrain <IDashboardGrain>(0);
await dashboardGrain.SubmitTracing(siloAddress, items).ConfigureAwait(false);
}).Wait(30000);
}
catch (Exception ex)
{
logger.LogWarning(100001, "Exception thrown sending tracing to dashboard grain", ex);
}
}
}
async Task Callback(object _)
{
var metricsGrain = this.GrainFactory.GetGrain <IManagementGrain>(0);
var activationCountTask = metricsGrain.GetTotalActivationCount();
var hostsTask = metricsGrain.GetDetailedHosts(true);
var simpleGrainStatsTask = metricsGrain.GetSimpleGrainStatistics();
await Task.WhenAll(activationCountTask, hostsTask, simpleGrainStatsTask);
this.Counters.TotalActivationCount = activationCountTask.Result;
this.Counters.TotalActiveHostCount = hostsTask.Result.Count(x => x.Status == SiloStatus.Active);
this.Counters.TotalActivationCountHistory.Enqueue(activationCountTask.Result);
this.Counters.TotalActiveHostCountHistory.Enqueue(this.Counters.TotalActiveHostCount);
while (this.Counters.TotalActivationCountHistory.Count > Dashboard.HistoryLength)
{
this.Counters.TotalActivationCountHistory.Dequeue();
}
while (this.Counters.TotalActiveHostCountHistory.Count > Dashboard.HistoryLength)
{
this.Counters.TotalActiveHostCountHistory.Dequeue();
}
// TODO - whatever max elapsed time
var elapsedTime = Math.Min((DateTime.UtcNow - this.StartTime).TotalSeconds, 100);
this.Counters.Hosts = hostsTask.Result.Select(x => new SiloDetails
{
FaultZone = x.FaultZone,
HostName = x.HostName,
IAmAliveTime = x.IAmAliveTime.ToString("o"),
ProxyPort = x.ProxyPort,
RoleName = x.RoleName,
SiloAddress = x.SiloAddress.ToParsableString(),
SiloName = x.SiloName,
StartTime = x.StartTime.ToString("o"),
Status = x.Status.ToString(),
UpdateZone = x.UpdateZone
}).ToArray();
this.Counters.SimpleGrainStats = simpleGrainStatsTask.Result.Select(x => {
var grainName = TypeFormatter.Parse(x.GrainType);
return(new SimpleGrainStatisticCounter
{
ActivationCount = x.ActivationCount,
GrainType = grainName,
SiloAddress = x.SiloAddress.ToParsableString(),
TotalAwaitTime = this.history.Where(n => n.Grain == grainName && n.SiloAddress == x.SiloAddress.ToParsableString()).SumZero(n => n.ElapsedTime),
TotalCalls = this.history.Where(n => n.Grain == grainName && n.SiloAddress == x.SiloAddress.ToParsableString()).SumZero(n => n.Count),
TotalExceptions = this.history.Where(n => n.Grain == grainName && n.SiloAddress == x.SiloAddress.ToParsableString()).SumZero(n => n.ExceptionCount),
TotalSeconds = elapsedTime
});
}).ToArray();
}
internal void RecalculateCounters(int activationCount, SiloDetails[] hosts,
IList <SimpleGrainStatistic> simpleGrainStatistics)
{
this.Counters.TotalActivationCount = activationCount;
this.Counters.TotalActiveHostCount = hosts.Count(x => x.SiloStatus == SiloStatus.Active);
this.Counters.TotalActivationCountHistory.Enqueue(activationCount);
this.Counters.TotalActiveHostCountHistory.Enqueue(this.Counters.TotalActiveHostCount);
while (this.Counters.TotalActivationCountHistory.Count > Dashboard.HistoryLength)
{
this.Counters.TotalActivationCountHistory.Dequeue();
}
while (this.Counters.TotalActiveHostCountHistory.Count > Dashboard.HistoryLength)
{
this.Counters.TotalActiveHostCountHistory.Dequeue();
}
// TODO - whatever max elapsed time
var elapsedTime = Math.Min((DateTime.UtcNow - this.StartTime).TotalSeconds, 100);
this.Counters.Hosts = hosts;
var aggregatedTotals = this.history
.GroupBy(x => new GrainSiloKey(x.Grain, x.SiloAddress))
.ToDictionary(g => g.Key, g => new AggregatedGrainTotals
{
TotalAwaitTime = g.Sum(x => x.ElapsedTime),
TotalCalls = g.Sum(x => x.Count),
TotalExceptions = g.Sum(x => x.ExceptionCount)
});
this.Counters.SimpleGrainStats = simpleGrainStatistics.Select(x =>
{
var grainName = TypeFormatter.Parse(x.GrainType);
var siloAddress = x.SiloAddress.ToParsableString();
AggregatedGrainTotals totals;
if (!aggregatedTotals.TryGetValue(new GrainSiloKey(grainName, siloAddress), out totals))
{
totals = new AggregatedGrainTotals();
}
return(new SimpleGrainStatisticCounter
{
ActivationCount = x.ActivationCount,
GrainType = grainName,
SiloAddress = x.SiloAddress.ToParsableString(),
TotalAwaitTime = totals.TotalAwaitTime,
TotalCalls = totals.TotalCalls,
TotalExceptions = totals.TotalExceptions,
TotalSeconds = elapsedTime
});
}).ToArray();
}
internal void RecalculateCounters(int activationCount, SiloDetails[] hosts,
IList <SimpleGrainStatistic> simpleGrainStatistics)
{
counters.TotalActivationCount = activationCount;
counters.TotalActiveHostCount = hosts.Count(x => x.SiloStatus == SiloStatus.Active);
counters.TotalActivationCountHistory = counters.TotalActivationCountHistory.Enqueue(activationCount).Dequeue();
counters.TotalActiveHostCountHistory = counters.TotalActiveHostCountHistory.Enqueue(counters.TotalActiveHostCount).Dequeue();
// TODO - whatever max elapsed time
var elapsedTime = Math.Min((DateTime.UtcNow - startTime).TotalSeconds, 100);
counters.Hosts = hosts;
var aggregatedTotals = history.GroupByGrainAndSilo().ToLookup(x => (x.Grain, x.SiloAddress));
//var aggregatedTotals = history.ToLookup(x => (x.Grain, x.SiloAddress));
counters.SimpleGrainStats = simpleGrainStatistics.Select(x =>
{
var grainName = TypeFormatter.Parse(x.GrainType);
var siloAddress = x.SiloAddress.ToParsableString();
var result = new SimpleGrainStatisticCounter
{
ActivationCount = x.ActivationCount,
GrainType = grainName,
SiloAddress = siloAddress,
TotalSeconds = elapsedTime
};
foreach (var item in aggregatedTotals[(grainName, siloAddress)])
{
result.TotalAwaitTime += item.ElapsedTime;
result.TotalCalls += item.Count;
result.TotalExceptions += item.ExceptionCount;
}
return(result);
}).ToArray();
public Task SubmitTracing(string siloIdentity, GrainTraceEntry[] grainTrace)
{
var now = DateTime.UtcNow;
foreach (var entry in grainTrace)
{
// sync clocks
entry.Period = now;
}
// fill in any previously captured methods which aren't in this reporting window
var allGrainTrace = new List <GrainTraceEntry>(grainTrace);
var values = this.history.Where(x => x.SiloAddress == siloIdentity).GroupBy(x => x.GrainAndMethod).Select(x => x.First());
foreach (var value in values)
{
if (!grainTrace.Any(x => x.GrainAndMethod == value.GrainAndMethod))
{
allGrainTrace.Add(new GrainTraceEntry
{
Count = 0,
ElapsedTime = 0,
Grain = TypeFormatter.Parse(value.Grain),
Method = value.Method,
Period = now,
SiloAddress = siloIdentity
});
}
}
var retirementWindow = DateTime.UtcNow.AddSeconds(-100);
history.AddRange(allGrainTrace);
history.RemoveAll(x => x.Period < retirementWindow);
return(TaskDone.Done);
}
internal void RecalculateCounters(int activationCount, SiloDetails[] hosts,
IList <DetailedGrainStatistic> simpleGrainStatistics)
{
counters.TotalActivationCount = activationCount;
counters.TotalActiveHostCount = hosts.Count(x => x.SiloStatus == SiloStatus.Active);
counters.TotalActivationCountHistory = counters.TotalActivationCountHistory.Enqueue(activationCount).Dequeue();
counters.TotalActiveHostCountHistory = counters.TotalActiveHostCountHistory.Enqueue(counters.TotalActiveHostCount).Dequeue();
// TODO - whatever max elapsed time
var elapsedTime = Math.Min((DateTime.UtcNow - startTime).TotalSeconds, 100);
foreach (var item in counters.Hosts)
{
foreach (var item2 in hosts)
{
if (item2.SiloAddress == item.SiloAddress)
{
item2.ExtraData = item.ExtraData;
}
}
}
counters.Hosts = hosts;
var aggregatedTotals = history.GroupByGrainAndSilo().ToLookup(x => (x.Grain, x.SiloAddress));
var grainstatts = simpleGrainStatistics.Select(x =>
{
var grainName = TypeFormatter.Parse(x.GrainType);
if (MultiGrainAttribute.IsRecordClass(x.GrainType))
{
if (x.GrainIdentity.PrimaryKeyString != null)
{
grainName += $":{x.GrainIdentity.PrimaryKeyString}";
}
else
{
grainName += $":{x.GrainIdentity.PrimaryKeyLong}";
}
}
var siloAddress = x.SiloAddress.ToParsableString();
var result = new SimpleGrainStatisticCounter
{
//ActivationCount = x.ActivationCount,
ActivationCount = 1,
GrainType = grainName,
SiloAddress = siloAddress,
TotalSeconds = elapsedTime
};
foreach (var item in aggregatedTotals[(grainName, siloAddress)])
{
result.TotalAwaitTime += item.ElapsedTime;
result.TotalCalls += item.Count;
result.TotalExceptions += item.ExceptionCount;
}
return(result);
}).ToArray();
