public void MetricsAvailableHandler(IntervalMetrics metrics)
{
OpenFileIfNeeded(metrics);
string csvLine = string.Format("{0}, {1}, {2,22}, {3,8}, {4,5:f0}, {5,5:f1}, {6,5}, {7,5}, {8,5}, {9,5}, {10,5}, {11,5}, {12,5}",
metrics.ResourceDescription,
metrics.TestDescription,
metrics.EndTime.ToString(),
metrics.Count,
metrics.RPS,
metrics.CpuPercentage,
metrics.AverageLatencyMS,
metrics.Percentile50,
metrics.Percentile90,
metrics.Percentile95,
metrics.Percentile99,
metrics.Percentile995,
metrics.Percentile999);
lock (_lock)
{
_fileWriter.WriteLine(csvLine);
_fileWriter.Close();
_fileWriter = File.AppendText(_filePath);
}
}
private async Task ReportObservedTPSAsync()
{
_lastReportTime = DateTime.Now;
// Write TPS periodically
while (_enabled)
{
try
{
// Wait for start of next interval
await Task.Delay(GetMillisecondsToStartOfNextInterval());
// Note info for current interval, resetting interval aggretation data members
DateTime currentTimeSnapshot = DateTime.Now;
var recentLatencyTimes = Interlocked.Exchange(ref _intervalLatencyTimes, new ConcurrentDictionary <int, int>());
var recentTotalRequestCharge = Interlocked.Exchange(ref _totalRequestCharge, 0.0);
// Calculate one second interval metrics
IntervalMetrics m = InitPerSecondIntervalMetric(currentTimeSnapshot, currentTimeSnapshot - _lastReportTime, recentLatencyTimes, recentTotalRequestCharge);
PopulateExtendedMetrics?.Invoke(m);
PerSecondMetricsAvailable?.Invoke(m);
// Update one minute interval metrics & post if needed
UpdatePerMinuteIntervalMetrics(m);
// Remember this interval
_lastReportTime = currentTimeSnapshot;
await Task.Delay(10); // Don't run too fast & report more than once per interval
}
catch (Exception x)
{
Tracer.TraceWarning("Ignoring exception on background metrics reporting thread. Exception = {0}", x.ToString());
}
}
}
public void MetricsAvailableHandler(IntervalMetrics metrics)
{
if (null == myClient)
{
myClient = new HttpClient();
}
PerfMetrics p = new PerfMetrics()
{
RequestUnits = 0.0,
RPS = metrics.RPS,
SourceId = mySourceId,
TimeStamp = DateTime.Now
};
string jp = JsonConvert.SerializeObject(p);
StringContent content = new StringContent(jp, System.Text.Encoding.UTF8, "application/json");
try
{
// Fire and forget ... best effort ... async ...
myClient.PostAsync("http://10.0.0.8:5000/api/PerfMetrics", content);
}
catch (Exception x)
{
Tracer.TraceVerbose($"Exception: {x.Message}");
}
}
private void OpenFileIfNeeded(IntervalMetrics metrics)
{
if (null == _fileWriter)
{
lock (_lock)
{
if (null == _fileWriter)
{
DateTime startTime = DateTime.Now;
_filePath = string.Format("{0}-{1}-{2}-{3}-{4}-{5}-{6}-{7}.csv",
Environment.MachineName,
metrics.ProcessId,
startTime.Year,
startTime.Month,
startTime.Day,
startTime.Hour,
startTime.Minute,
startTime.Second);
_fileWriter = File.AppendText(_filePath);
_fileWriter.WriteLine("\"ResourceDescription\",\"TestDescription\",\"IntervalTime\",\"Count\",\"RPS\",\"AverageLatency\",\"P50\",\"P90\",\"P95\",\"P99\",\"P99.5\",\"P99.9\"");
}
}
}
}
private IntervalMetrics InitPerMinuteIntervalMetric(IntervalMetrics currentSecondMetrics)
{
return(new IntervalMetrics
(
currentSecondMetrics,
currentSecondMetrics.EndTime.Truncate(TimeSpan.FromMinutes(1)),
TimeSpan.FromMinutes(1)
));
}
public void Aggregate(IntervalMetrics other)
{
// Aggregate latency times
foreach (var latencyTime in other.LatencyDistributionMS.Keys)
{
int otherValue = other.LatencyDistributionMS[latencyTime];
LatencyDistributionMS.AddOrUpdate(latencyTime, otherValue, (key, currentValue) => currentValue + otherValue);
}
// Recompute other values
ComputeCalculatedAttributes();
}
public void MetricsAvailableHandlerSingleLine(IntervalMetrics metrics)
{
Console.WriteLine(
"Resource: {0,-30}, Test: {1,-20}, Time: {2,-22}, RPS: {3,5:f0}, Average:{4,5}, P50:{5,5}, P90:{6,5}, P95:{7,5}, P99:{8,5}",
metrics.ResourceDescription, // 0
metrics.TestDescription, // 1
metrics.EndTime.ToString(), // 2
metrics.RPS, // 3
metrics.AverageLatencyMS, // 4
metrics.Percentile50, // 5
metrics.Percentile90, // 6
metrics.Percentile95, // 7
metrics.Percentile99); // 8
}
private async Task ReportObservedTpsAsync(CancellationToken cancellationToken)
{
_lastReportTime = DateTime.Now;
// Write TPS periodically
while (true)
{
// Bail if cancelled
cancellationToken.ThrowIfCancellationRequested();
try
{
// Wait for start of next interval
await Task.Delay(GetMillisecondsToStartOfNextInterval(), cancellationToken);
// Bail if cancelled
cancellationToken.ThrowIfCancellationRequested();
// Note info for current interval, resetting interval aggretation data members
DateTime currentTimeSnapshot = DateTime.Now;
var recentLatencyTimes = Interlocked.Exchange(ref _intervalLatencyTimes,
new ConcurrentDictionary <int, int>());
var recentCpuPercentageReported = Interlocked.Exchange(ref _totalCpuPercentageReported, 0.0);
// Calculate one second interval metrics
IntervalMetrics m = InitPerSecondIntervalMetric(currentTimeSnapshot,
currentTimeSnapshot - _lastReportTime, recentLatencyTimes, recentCpuPercentageReported);
PopulateExtendedMetrics?.Invoke(m);
PerSecondMetricsAvailable?.Invoke(m);
// Update one minute interval metrics & post if needed
UpdatePerMinuteIntervalMetrics(m);
// Remember this interval
_lastReportTime = currentTimeSnapshot;
// Don't run too fast & report more than once per interval
await Task.Delay(10, cancellationToken);
}
catch (OperationCanceledException)
{
// Don't throw exception when cancelled, just exit silently
break;
}
catch (Exception x)
{
Tracer.TraceWarning($"Ignoring exception on background metrics reporting thread. Exception = {x.ToString()}");
}
}
}
private void UpdatePerMinuteIntervalMetrics(IntervalMetrics currentSecondMetrics)
{
// Create per minute metric at program start
if (null == _perMinuteMetrics)
{
_perMinuteMetrics = InitPerMinuteIntervalMetric(currentSecondMetrics);
}
// If transitioned to a new minute, notify & start new minute
else if (currentSecondMetrics.EndTime.Minute != _perMinuteMetrics.EndTime.Minute)
{
PerMinuteMetricsAvailable?.Invoke(_perMinuteMetrics);
_perMinuteMetrics = InitPerMinuteIntervalMetric(currentSecondMetrics);
}
// If in same minute as last time aggregate this second with current minute
else
{
_perMinuteMetrics.Aggregate(currentSecondMetrics);
}
}
public void MetricsAvailableHandler(IntervalMetrics metrics)
{
_lookbackData.Add(metrics);
if (_lookbackData.Count > _lookbackSeconds)
{
_lookbackData.RemoveAt(0);
}
var endTime = metrics.EndTime;
var duration = metrics.EndTime - _lookbackData[0].EndTime + TimeSpan.FromSeconds(1);
var currentAggregation = new IntervalMetrics(_lookbackData[0], endTime, duration);
for (int i = 1; i < _lookbackData.Count; i++)
{
currentAggregation.Aggregate(_lookbackData[i]);
}
//Tracer.TraceInfo($"endTime: {currentAggregation.EndTime} duration: {currentAggregation.Duration} count: {currentAggregation.Count} rps: {currentAggregation.RPS}");
MetricsAvailableHandlerSingleLine(currentAggregation);
}
public void MetricsAvailableHandlerSingleLine(IntervalMetrics metrics)
{
List <string> messagesToFlush = null;
var line = string.Format(
"Resource: {0, -23} , Test: {1,-25} , Time: {2,-22}, RPS: {3,7:f2}, CPU: {9,5:f1}, Average:{4,5}, P50:{5,5}, P90:{6,5}, P95:{7,5}, P99:{8,5}",
metrics.ResourceDescription, // 0
metrics.TestDescription, // 1
metrics.EndTime.ToString(), // 2
metrics.RPS, // 3
metrics.AverageLatencyMS, // 4
metrics.Percentile50, // 5
metrics.Percentile90, // 6
metrics.Percentile95, // 7
metrics.Percentile99, // 8
metrics.CpuPercentage); // 9
lock (_bufferedOutput)
{
_bufferedOutput.Add(line);
if (_bufferedOutput.Count >= _simultaneousCount)
{
messagesToFlush = _bufferedOutput;
_bufferedOutput = new List <string>();
}
}
if (messagesToFlush != null)
{
messagesToFlush.Sort();
foreach (var m in messagesToFlush)
{
Console.WriteLine(m);
}
Console.WriteLine();
}
}
public void MetricsAvailableHandler(IntervalMetrics metrics)
{
MetricsAvailableHandlerSingleLine(metrics);
}
public IntervalMetrics(IntervalMetrics other, DateTime endTime, TimeSpan duration)
: this(endTime, duration, other.TestDescription, other.MachineName, other.ProcessId, other.ResourceDescription, other.LatencyDistributionMS, other.TotalCpuPercentageReported)
{
}
