/// <summary> /// Get the current scale status (vote) by querying all active monitors for their /// scale status. /// </summary> /// <param name="context">The context to use for the scale decision.</param> /// <returns>The scale vote.</returns> public virtual async Task <ScaleStatusResult> GetScaleStatusAsync(ScaleStatusContext context) { var monitors = _monitorManager.GetMonitors(); List <ScaleVote> votes = new List <ScaleVote>(); if (monitors.Any()) { // get the collection of current metrics for each monitor var monitorMetrics = await _metricsRepository.ReadMetricsAsync(monitors); _logger.LogDebug($"Computing scale status (WorkerCount={context.WorkerCount})"); _logger.LogDebug($"{monitorMetrics.Count} scale monitors to sample"); // for each monitor, ask it to return its scale status (vote) based on // the metrics and context info (e.g. worker count) foreach (var pair in monitorMetrics) { var monitor = pair.Key; var metrics = pair.Value; try { // create a new context instance to avoid modifying the // incoming context var scaleStatusContext = new ScaleStatusContext { WorkerCount = context.WorkerCount, Metrics = metrics }; var result = monitor.GetScaleStatus(scaleStatusContext); _logger.LogDebug($"Monitor '{monitor.Descriptor.Id}' voted '{result.Vote.ToString()}'"); votes.Add(result.Vote); } catch (Exception exc) when(!exc.IsFatal()) { // if a particular monitor fails, log and continue _logger.LogError(exc, $"Failed to query scale status for monitor '{monitor.Descriptor.Id}'."); } } } else { // no monitors registered // this can happen if the host is offline } var vote = GetAggregateScaleVote(votes, context, _logger); return(new ScaleStatusResult { Vote = vote }); }
private async Task TakeMetricsSamplesAsync() { try { // get the monitors // if the host is offline, no monitors will be returned var monitors = _monitorManager.GetMonitors(); if (monitors.Any()) { _logger.LogDebug($"Taking metrics samples for {monitors.Count()} monitor(s)."); var metricsMap = new Dictionary <IScaleMonitor, ScaleMetrics>(); foreach (var monitor in monitors) { ScaleMetrics metrics = null; try { // take a metrics sample for each monitor metrics = await monitor.GetMetricsAsync(); metricsMap[monitor] = metrics; // log the metrics json to provide visibility into monitor activity var json = JsonConvert.SerializeObject(metrics); _logger.LogDebug($"Scale metrics sample for monitor '{monitor.Descriptor.Id}': {json}"); } catch (Exception exc) when(!exc.IsFatal()) { // if a particular monitor fails, log and continue _logger.LogError(exc, $"Failed to collect scale metrics sample for monitor '{monitor.Descriptor.Id}'."); } } if (metricsMap.Count > 0) { // persist the metrics samples await _metricsRepository.WriteMetricsAsync(metricsMap); } } } catch (Exception exc) when(!exc.IsFatal()) { _logger.LogError(exc, "Failed to collect/persist scale metrics."); } }