/// <inheritdoc />
public async Task <HealthCheckResult> CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default)
{
using var stream = new MemoryStream();
await _registry.CollectAndExportAsTextAsync(stream, cancellationToken);
stream.Position = 0;
// convert the stream to a string
using var reader = new StreamReader(stream);
var text = reader.ReadToEnd();
var metricLine = text?.Split('\n').FirstOrDefault(s => s.StartsWith(MetricWatch));
if (metricLine != null)
{
var memory = ParseLong(metricLine);
if (memory > MaxMemoryUsage)
{
return(HealthCheckResult.Unhealthy("Memory usage is too high"));
}
}
return(HealthCheckResult.Healthy("Memory is good"));
}
private string GetPrometheusContent()
{
string content;
using (MemoryStream myStream = new MemoryStream())
{
_collectorRegistry.CollectAndExportAsTextAsync(myStream).Wait();
myStream.Seek(0, SeekOrigin.Begin);
using (StreamReader sr = new StreamReader(myStream))
{
content = sr.ReadToEnd();
}
}
return(content);
}
private void onPrometheusHttpRequest(TheRequestData tRequest)
{
try
{
tRequest.AllowStatePush = false; //No need to track state
if (HoldOffOnSenderLoop && (DateTimeOffset.Now - _lastPollTime) > maxMetricSampleRate)
{
// Using polling: generate all metrics now
GenerateMetrics();
_lastPollTime = DateTimeOffset.Now;
}
using (var outputStream = new MemoryStream())
{
try
{
_registry.CollectAndExportAsTextAsync(outputStream, TheBaseAssets.MasterSwitchCancelationToken).Wait();
}
catch (ScrapeFailedException ex)
{
tRequest.StatusCode = 500;
EventsSentErrorCountSinceStart++;
if (!string.IsNullOrWhiteSpace(ex.Message))
{
tRequest.ResponseBuffer = Encoding.UTF8.GetBytes(ex.Message);
}
return;
}
if (tRequest.Header.TryGetValue("Accept", out var acceptHeader))
{
var acceptHeaders = acceptHeader?.Split(',');
var contentType = "text/plain";
tRequest.ResponseMimeType = contentType;
tRequest.StatusCode = 200;
tRequest.ResponseBuffer = outputStream.ToArray();
EventsSentSinceStart++;
TheThing.SetSafePropertyNumber(MyBaseThing, "QValue", EventsSentSinceStart);
LastSendTime = DateTimeOffset.Now;
TheThing.SetSafePropertyString(MyBaseThing, "StateSensorUnit", TheCommonUtils.GetDateTimeString(LastSendTime, -1));
}
else
{
tRequest.StatusCode = 406;
tRequest.ResponseBuffer = Encoding.UTF8.GetBytes("No accept header");
EventsSentErrorCountSinceStart++;
return;
}
}
}
catch (Exception ex) when(!(ex is OperationCanceledException))
{
EventsSentErrorCountSinceStart++;
TheBaseAssets.MySYSLOG.WriteToLog(95307, TSM.L(eDEBUG_LEVELS.OFF) ? null : new TSM(strPrometheusExporter, $"Error in metric server: {this.MyBaseThing?.Address}", eMsgLevel.l3_ImportantMessage, ex.ToString()));
try
{
tRequest.StatusCode = 500;
}
catch
{
// Might be too late in request processing to set response code, so just ignore.
}
}
}
protected Task StartServer(CancellationToken cancel)
{
// This will ensure that any failures to start are nicely thrown from StartServerAsync.
_httpListener.Start();
// Kick off the actual processing to a new thread and return a Task for the processing thread.
return(Task.Factory.StartNew(delegate
{
try
{
Thread.CurrentThread.Name = "Metric Server"; //Max length 16 chars (Linux limitation)
while (!cancel.IsCancellationRequested)
{
// There is no way to give a CancellationToken to GCA() so, we need to hack around it a bit.
var getContext = _httpListener.GetContextAsync();
getContext.Wait(cancel);
var context = getContext.Result;
// Kick the request off to a background thread for processing.
_ = Task.Factory.StartNew(async delegate
{
var request = context.Request;
var response = context.Response;
try
{
Thread.CurrentThread.Name = "Metric Process";
CollectorRegistry registry = Metrics.NewCustomRegistry();
MetricFactory metricFactory = Metrics.WithCustomRegistry(registry);
try
{
foreach (var callback in _scrapeCallbacks)
{
callback(metricFactory, request.QueryString);
}
await Task.WhenAll(_scrapeAsyncCallbacks.Select(callback => callback(cancel, metricFactory, request.QueryString)));
var extraTexts = await Task.WhenAll(_scrapeAsyncCallbacksWithExtraText.Select(callback => callback(cancel, metricFactory, request.QueryString)));
using (MemoryStream ms = new MemoryStream())
{
if (extraTexts != null)
{
using StreamWriter writer = new StreamWriter(ms, new UTF8Encoding(false), 1024, true);
foreach (var extraText in extraTexts)
{
if (string.IsNullOrEmpty(extraText))
{
continue;
}
await writer.WriteLineAsync(extraText);
await writer.WriteLineAsync();
}
}
await registry.CollectAndExportAsTextAsync(ms, cancel);
ms.Position = 0;
response.ContentType = PrometheusConstants.ExporterContentType;
response.StatusCode = 200;
await ms.CopyToAsync(response.OutputStream, 81920, cancel);
}
response.OutputStream.Dispose();
}
catch (ScrapeFailedException ex)
{
// This can only happen before anything is written to the stream, so it
// should still be safe to update the status code and report an error.
response.StatusCode = 503;
if (!string.IsNullOrWhiteSpace(ex.Message))
{
using var writer = new StreamWriter(response.OutputStream);
writer.Write(ex.Message);
}
}
}
catch (Exception ex) when(!(ex is OperationCanceledException))
{
if (!_httpListener.IsListening)
{
return; // We were shut down.
}
Trace.WriteLine(string.Format("Error in {0}: {1}", nameof(MetricServer), ex));
try
{
response.StatusCode = 500;
}
catch
{
// Might be too late in request processing to set response code, so just ignore.
}
}
finally
{
response.Close();
}
}, TaskCreationOptions.LongRunning);
}
}