private void SendMetrics()
{
if (Monitor.TryEnter(_syncObj))
{
try
{
using (var stream = new MemoryStream())
{
ScrapeHandler.ProcessScrapeRequest(_registry.CollectAll(), ContentType, stream);
using (var client = new WebClient())
{
client.UploadData(_endpoint, "POST", stream.ToArray());
}
}
}
catch (Exception e)
{
Trace.WriteLine(string.Format("Exception in send metrics: {0}", e));
}
finally
{
Monitor.Exit(_syncObj);
}
}
}
private void StartLoop(IScheduler scheduler)
{
//delegate allocations below - but that's fine as it's not really on the "critical path" (polled relatively infrequently) - and it's much more readable this way
scheduler.Schedule(repeatAction => _httpListener.BeginGetContext(ar =>
{
try
{
var httpListenerContext = _httpListener.EndGetContext(ar);
var request = httpListenerContext.Request;
var response = httpListenerContext.Response;
response.StatusCode = 200;
var acceptHeader = request.Headers.Get("Accept");
var acceptHeaders = acceptHeader == null ? null : acceptHeader.Split(',');
var contentType = ScrapeHandler.GetContentType(acceptHeaders);
response.ContentType = contentType;
using (var outputStream = response.OutputStream)
{
var collected = _registry.CollectAll();
ScrapeHandler.ProcessScrapeRequest(collected, contentType, outputStream);
}
response.Close();
}
catch (Exception e)
{
Trace.WriteLine(string.Format("Error in MetricsServer: {0}", e));
}
repeatAction.Invoke();
}, null));
}
protected override Task StartServer(CancellationToken cancel)
{
// Kick off the actual processing to a new thread and return a Task for the processing thread.
return(Task.Run(async delegate
{
while (true)
{
// We schedule approximately at the configured interval. There may be some small accumulation for the
// part of the loop we do not measure but it is close enough to be acceptable for all practical scenarios.
var duration = Stopwatch.StartNew();
try
{
var metrics = _registry.CollectAll();
var stream = new MemoryStream();
ScrapeHandler.ProcessScrapeRequest(metrics, ContentType, stream);
stream.Position = 0;
// StreamContent takes ownership of the stream.
var response = await _httpClient.PostAsync(_targetUrl, new StreamContent(stream));
// If anything goes wrong, we want to get at least an entry in the trace log.
response.EnsureSuccessStatusCode();
}
catch (ScrapeFailedException ex)
{
Trace.WriteLine($"Skipping metrics push due to failed scrape: {ex.Message}");
}
catch (Exception ex) when(!(ex is OperationCanceledException))
{
Trace.WriteLine(string.Format("Error in MetricPusher: {0}", ex));
}
// We always stop after pushing metrics, to ensure that the latest state is flushed when told to stop.
if (cancel.IsCancellationRequested)
{
break;
}
var sleepTime = _pushInterval - duration.Elapsed;
// Sleep until the interval elapses or the pusher is asked to shut down.
if (sleepTime > TimeSpan.Zero)
{
try
{
await Task.Delay(sleepTime, cancel);
}
catch (OperationCanceledException)
{
// The task was cancelled.
// We continue the loop here to ensure final state gets pushed.
continue;
}
}
}
}));
}
public void Configure(IApplicationBuilder app)
{
app.Run(context => {
var response = context.Response;
var request = context.Request;
response.StatusCode = 200;
var acceptHeader = request.Headers["Accept"];
var contentType = ScrapeHandler.GetContentType(acceptHeader);
response.ContentType = contentType;
using (var outputStream = response.Body) {
var collected = _registry.CollectAll();
ScrapeHandler.ProcessScrapeRequest(collected, contentType, outputStream);
};
return(Task.FromResult(true));
});
}
public async Task Invoke(HttpContext context)
{
// We just handle the root URL (/metrics or whatnot).
if (!string.IsNullOrWhiteSpace(context.Request.Path.Value))
{
await _next(context);
return;
}
var request = context.Request;
var response = context.Response;
var acceptHeaders = request.Headers["Accept"];
var contentType = ScrapeHandler.GetContentType(acceptHeaders);
response.ContentType = contentType;
IEnumerable <MetricFamily> metrics;
try
{
metrics = _registry.CollectAll();
}
catch (ScrapeFailedException ex)
{
response.StatusCode = 503;
if (!string.IsNullOrWhiteSpace(ex.Message))
{
using (var writer = new StreamWriter(response.Body))
await writer.WriteAsync(ex.Message);
}
return;
}
response.StatusCode = 200;
using (var outputStream = response.Body)
ScrapeHandler.ProcessScrapeRequest(metrics, contentType, outputStream);
}
protected override Task StartServer(CancellationToken cancel)
{
// This will ensure that any failures to start are nicely thrown from StartServerAsync.
_httpListener.Start();
// Kick off the actual processing to a new thread and return a Task for the processing thread.
return(Task.Factory.StartNew(delegate
{
try
{
while (!cancel.IsCancellationRequested)
{
// There is no way to give a CancellationToken to GCA() so, we need to hack around it a bit.
var getContext = _httpListener.GetContextAsync();
getContext.Wait(cancel);
var context = getContext.Result;
var request = context.Request;
var response = context.Response;
try
{
IEnumerable <MetricFamily> metrics;
try
{
metrics = _registry.CollectAll();
}
catch (ScrapeFailedException ex)
{
response.StatusCode = 503;
if (!string.IsNullOrWhiteSpace(ex.Message))
{
using (var writer = new StreamWriter(response.OutputStream))
writer.Write(ex.Message);
}
continue;
}
var acceptHeader = request.Headers.Get("Accept");
var acceptHeaders = acceptHeader?.Split(',');
var contentType = ScrapeHandler.GetContentType(acceptHeaders);
response.ContentType = contentType;
response.StatusCode = 200;
using (var outputStream = response.OutputStream)
ScrapeHandler.ProcessScrapeRequest(metrics, contentType, outputStream);
}
catch (Exception ex) when(!(ex is OperationCanceledException))
{
Trace.WriteLine(string.Format("Error in MetricsServer: {0}", ex));
try
{
response.StatusCode = 500;
}
catch
{
// Might be too late in request processing to set response code, so just ignore.
}
}
finally
{
response.Close();
}
}
}
finally
{
_httpListener.Stop();
_httpListener.Close();
}
}, TaskCreationOptions.LongRunning));
}
/// <summary>
/// Push metrics to single pushgateway endpoint
/// </summary>
/// <param name="metrics">Collection of metrics</param>
/// <param name="endpoints">PushGateway endpoints - fault-tolerance</param>
/// <param name="job">job name</param>
/// <param name="instance">instance name</param>
/// <param name="contentType">content-type</param>
/// <param name="additionalLabels">additional labels</param>
/// <returns></returns>
public async Task PushAsync(IEnumerable <MetricFamily> metrics, string[] endpoints, string job, string instance, string contentType = ContentType, IEnumerable <Tuple <string, string> > additionalLabels = null)
{
if (endpoints == null || !endpoints.Any())
{
throw new ArgumentNullException(nameof(endpoints));
}
if (string.IsNullOrEmpty(job))
{
throw new ArgumentNullException(nameof(job));
}
var tasks = new List <Task <HttpResponseMessage> >(endpoints.Length);
var streamsToDispose = new List <Stream>();
foreach (var endpoint in endpoints)
{
if (string.IsNullOrEmpty(endpoint))
{
throw new ArgumentNullException(nameof(endpoint));
}
var url = $"{endpoint.TrimEnd('/')}/metrics/job/{job}";
if (!string.IsNullOrEmpty(instance))
{
url = $"{url}/instance/{instance}";
}
var sb = new StringBuilder();
sb.Append(url);
if (additionalLabels != null)
{
foreach (var pair in additionalLabels)
{
if (pair == null || string.IsNullOrEmpty(pair.Item1) || string.IsNullOrEmpty(pair.Item2))
{
// TODO: Surely this should throw an exception?
Trace.WriteLine("Ignoring invalid label set");
continue;
}
sb.AppendFormat("/{0}/{1}", pair.Item1, pair.Item2);
}
}
if (!Uri.TryCreate(sb.ToString(), UriKind.Absolute, out var targetUrl))
{
throw new ArgumentException("Endpoint must be a valid url", nameof(endpoint));
}
var memoryStream = new MemoryStream();
streamsToDispose.Add(memoryStream);
ScrapeHandler.ProcessScrapeRequest(metrics, contentType, memoryStream);
memoryStream.Position = 0;
var streamContent = new StreamContent(memoryStream);
tasks.Add(_httpClient.PostAsync(targetUrl, streamContent));
}
await Task.WhenAll(tasks).ConfigureAwait(false);
Exception exception = null;
foreach (var task in tasks)
{
var response = await task.ConfigureAwait(false);
try
{
response.EnsureSuccessStatusCode();
}
catch (Exception ex)
{
exception = ex;
}
}
streamsToDispose.ForEach(s => s.Dispose());
if (exception != null)
{
throw exception;
}
}