public void MetricsAvailableHandler(IntervalMetrics metrics) { OpenFileIfNeeded(metrics); string csvLine = string.Format("{0}, {1}, {2,22}, {3,8}, {4,5:f0}, {5,5:f1}, {6,5}, {7,5}, {8,5}, {9,5}, {10,5}, {11,5}, {12,5}", metrics.ResourceDescription, metrics.TestDescription, metrics.EndTime.ToString(), metrics.Count, metrics.RPS, metrics.CpuPercentage, metrics.AverageLatencyMS, metrics.Percentile50, metrics.Percentile90, metrics.Percentile95, metrics.Percentile99, metrics.Percentile995, metrics.Percentile999); lock (_lock) { _fileWriter.WriteLine(csvLine); _fileWriter.Close(); _fileWriter = File.AppendText(_filePath); } }
private async Task ReportObservedTPSAsync() { _lastReportTime = DateTime.Now; // Write TPS periodically while (_enabled) { try { // Wait for start of next interval await Task.Delay(GetMillisecondsToStartOfNextInterval()); // Note info for current interval, resetting interval aggretation data members DateTime currentTimeSnapshot = DateTime.Now; var recentLatencyTimes = Interlocked.Exchange(ref _intervalLatencyTimes, new ConcurrentDictionary <int, int>()); var recentTotalRequestCharge = Interlocked.Exchange(ref _totalRequestCharge, 0.0); // Calculate one second interval metrics IntervalMetrics m = InitPerSecondIntervalMetric(currentTimeSnapshot, currentTimeSnapshot - _lastReportTime, recentLatencyTimes, recentTotalRequestCharge); PopulateExtendedMetrics?.Invoke(m); PerSecondMetricsAvailable?.Invoke(m); // Update one minute interval metrics & post if needed UpdatePerMinuteIntervalMetrics(m); // Remember this interval _lastReportTime = currentTimeSnapshot; await Task.Delay(10); // Don't run too fast & report more than once per interval } catch (Exception x) { Tracer.TraceWarning("Ignoring exception on background metrics reporting thread. Exception = {0}", x.ToString()); } } }
public void MetricsAvailableHandler(IntervalMetrics metrics) { if (null == myClient) { myClient = new HttpClient(); } PerfMetrics p = new PerfMetrics() { RequestUnits = 0.0, RPS = metrics.RPS, SourceId = mySourceId, TimeStamp = DateTime.Now }; string jp = JsonConvert.SerializeObject(p); StringContent content = new StringContent(jp, System.Text.Encoding.UTF8, "application/json"); try { // Fire and forget ... best effort ... async ... myClient.PostAsync("http://10.0.0.8:5000/api/PerfMetrics", content); } catch (Exception x) { Tracer.TraceVerbose($"Exception: {x.Message}"); } }
private void OpenFileIfNeeded(IntervalMetrics metrics) { if (null == _fileWriter) { lock (_lock) { if (null == _fileWriter) { DateTime startTime = DateTime.Now; _filePath = string.Format("{0}-{1}-{2}-{3}-{4}-{5}-{6}-{7}.csv", Environment.MachineName, metrics.ProcessId, startTime.Year, startTime.Month, startTime.Day, startTime.Hour, startTime.Minute, startTime.Second); _fileWriter = File.AppendText(_filePath); _fileWriter.WriteLine("\"ResourceDescription\",\"TestDescription\",\"IntervalTime\",\"Count\",\"RPS\",\"AverageLatency\",\"P50\",\"P90\",\"P95\",\"P99\",\"P99.5\",\"P99.9\""); } } } }
private IntervalMetrics InitPerMinuteIntervalMetric(IntervalMetrics currentSecondMetrics) { return(new IntervalMetrics ( currentSecondMetrics, currentSecondMetrics.EndTime.Truncate(TimeSpan.FromMinutes(1)), TimeSpan.FromMinutes(1) )); }
public void Aggregate(IntervalMetrics other) { // Aggregate latency times foreach (var latencyTime in other.LatencyDistributionMS.Keys) { int otherValue = other.LatencyDistributionMS[latencyTime]; LatencyDistributionMS.AddOrUpdate(latencyTime, otherValue, (key, currentValue) => currentValue + otherValue); } // Recompute other values ComputeCalculatedAttributes(); }
public void MetricsAvailableHandlerSingleLine(IntervalMetrics metrics) { Console.WriteLine( "Resource: {0,-30}, Test: {1,-20}, Time: {2,-22}, RPS: {3,5:f0}, Average:{4,5}, P50:{5,5}, P90:{6,5}, P95:{7,5}, P99:{8,5}", metrics.ResourceDescription, // 0 metrics.TestDescription, // 1 metrics.EndTime.ToString(), // 2 metrics.RPS, // 3 metrics.AverageLatencyMS, // 4 metrics.Percentile50, // 5 metrics.Percentile90, // 6 metrics.Percentile95, // 7 metrics.Percentile99); // 8 }
private async Task ReportObservedTpsAsync(CancellationToken cancellationToken) { _lastReportTime = DateTime.Now; // Write TPS periodically while (true) { // Bail if cancelled cancellationToken.ThrowIfCancellationRequested(); try { // Wait for start of next interval await Task.Delay(GetMillisecondsToStartOfNextInterval(), cancellationToken); // Bail if cancelled cancellationToken.ThrowIfCancellationRequested(); // Note info for current interval, resetting interval aggretation data members DateTime currentTimeSnapshot = DateTime.Now; var recentLatencyTimes = Interlocked.Exchange(ref _intervalLatencyTimes, new ConcurrentDictionary <int, int>()); var recentCpuPercentageReported = Interlocked.Exchange(ref _totalCpuPercentageReported, 0.0); // Calculate one second interval metrics IntervalMetrics m = InitPerSecondIntervalMetric(currentTimeSnapshot, currentTimeSnapshot - _lastReportTime, recentLatencyTimes, recentCpuPercentageReported); PopulateExtendedMetrics?.Invoke(m); PerSecondMetricsAvailable?.Invoke(m); // Update one minute interval metrics & post if needed UpdatePerMinuteIntervalMetrics(m); // Remember this interval _lastReportTime = currentTimeSnapshot; // Don't run too fast & report more than once per interval await Task.Delay(10, cancellationToken); } catch (OperationCanceledException) { // Don't throw exception when cancelled, just exit silently break; } catch (Exception x) { Tracer.TraceWarning($"Ignoring exception on background metrics reporting thread. Exception = {x.ToString()}"); } } }
private void UpdatePerMinuteIntervalMetrics(IntervalMetrics currentSecondMetrics) { // Create per minute metric at program start if (null == _perMinuteMetrics) { _perMinuteMetrics = InitPerMinuteIntervalMetric(currentSecondMetrics); } // If transitioned to a new minute, notify & start new minute else if (currentSecondMetrics.EndTime.Minute != _perMinuteMetrics.EndTime.Minute) { PerMinuteMetricsAvailable?.Invoke(_perMinuteMetrics); _perMinuteMetrics = InitPerMinuteIntervalMetric(currentSecondMetrics); } // If in same minute as last time aggregate this second with current minute else { _perMinuteMetrics.Aggregate(currentSecondMetrics); } }
public void MetricsAvailableHandler(IntervalMetrics metrics) { _lookbackData.Add(metrics); if (_lookbackData.Count > _lookbackSeconds) { _lookbackData.RemoveAt(0); } var endTime = metrics.EndTime; var duration = metrics.EndTime - _lookbackData[0].EndTime + TimeSpan.FromSeconds(1); var currentAggregation = new IntervalMetrics(_lookbackData[0], endTime, duration); for (int i = 1; i < _lookbackData.Count; i++) { currentAggregation.Aggregate(_lookbackData[i]); } //Tracer.TraceInfo($"endTime: {currentAggregation.EndTime} duration: {currentAggregation.Duration} count: {currentAggregation.Count} rps: {currentAggregation.RPS}"); MetricsAvailableHandlerSingleLine(currentAggregation); }
public void MetricsAvailableHandlerSingleLine(IntervalMetrics metrics) { List <string> messagesToFlush = null; var line = string.Format( "Resource: {0, -23} , Test: {1,-25} , Time: {2,-22}, RPS: {3,7:f2}, CPU: {9,5:f1}, Average:{4,5}, P50:{5,5}, P90:{6,5}, P95:{7,5}, P99:{8,5}", metrics.ResourceDescription, // 0 metrics.TestDescription, // 1 metrics.EndTime.ToString(), // 2 metrics.RPS, // 3 metrics.AverageLatencyMS, // 4 metrics.Percentile50, // 5 metrics.Percentile90, // 6 metrics.Percentile95, // 7 metrics.Percentile99, // 8 metrics.CpuPercentage); // 9 lock (_bufferedOutput) { _bufferedOutput.Add(line); if (_bufferedOutput.Count >= _simultaneousCount) { messagesToFlush = _bufferedOutput; _bufferedOutput = new List <string>(); } } if (messagesToFlush != null) { messagesToFlush.Sort(); foreach (var m in messagesToFlush) { Console.WriteLine(m); } Console.WriteLine(); } }
public void MetricsAvailableHandler(IntervalMetrics metrics) { MetricsAvailableHandlerSingleLine(metrics); }
public IntervalMetrics(IntervalMetrics other, DateTime endTime, TimeSpan duration) : this(endTime, duration, other.TestDescription, other.MachineName, other.ProcessId, other.ResourceDescription, other.LatencyDistributionMS, other.TotalCpuPercentageReported) { }