private bool IsSystemProcessCgroupMemoryStatsInactiveFileBytesEnabled(IReadOnlyList <WildcardMatcher> disabledMetrics) =>
!WildcardMatcher.IsAnyMatch(
disabledMetrics, SystemProcessCgroupMemoryStatsInactiveFileBytes);
public MetricsCollector(IApmLogger logger, IPayloadSender payloadSender, IConfigurationReader configurationReader)
{
_logger = logger.Scoped(nameof(MetricsCollector));
_payloadSender = payloadSender;
var interval = configurationReader.MetricsIntervalInMilliseconds;
// ReSharper disable once CompareOfFloatsByEqualityOperator
if (interval == 0)
{
_logger.Info()?.Log("Collecting metrics is disabled - the agent won't collect metrics");
return;
}
MetricsProviders = new List <IMetricsProvider>();
if (!WildcardMatcher.IsAnyMatch(configurationReader.DisableMetrics, ProcessTotalCpuTimeProvider.ProcessCpuTotalPct))
{
MetricsProviders.Add(new ProcessTotalCpuTimeProvider(_logger));
}
if (!WildcardMatcher.IsAnyMatch(configurationReader.DisableMetrics, SystemTotalCpuProvider.SystemCpuTotalPct))
{
MetricsProviders.Add(new SystemTotalCpuProvider(_logger));
}
var collectProcessWorkingSet = !WildcardMatcher.IsAnyMatch(configurationReader.DisableMetrics,
ProcessWorkingSetAndVirtualMemoryProvider.ProcessWorkingSetMemory);
var collectProcessVirtualMemory = !WildcardMatcher.IsAnyMatch(configurationReader.DisableMetrics,
ProcessWorkingSetAndVirtualMemoryProvider.ProcessVirtualMemory);
if (collectProcessVirtualMemory || collectProcessWorkingSet)
{
MetricsProviders.Add(new ProcessWorkingSetAndVirtualMemoryProvider(collectProcessVirtualMemory, collectProcessWorkingSet));
}
var collectTotalMemory = !WildcardMatcher.IsAnyMatch(configurationReader.DisableMetrics,
FreeAndTotalMemoryProvider.TotalMemory);
var collectFreeMemory = !WildcardMatcher.IsAnyMatch(configurationReader.DisableMetrics,
FreeAndTotalMemoryProvider.FreeMemory);
if (collectFreeMemory || collectTotalMemory)
{
MetricsProviders.Add(new FreeAndTotalMemoryProvider(collectFreeMemory, collectTotalMemory));
}
var collectGcCount = !WildcardMatcher.IsAnyMatch(configurationReader.DisableMetrics,
GcMetricsProvider.GcCountName);
var collectGen0Size = !WildcardMatcher.IsAnyMatch(configurationReader.DisableMetrics,
GcMetricsProvider.GcGen0SizeName);
var collectGen1Size = !WildcardMatcher.IsAnyMatch(configurationReader.DisableMetrics,
GcMetricsProvider.GcGen1SizeName);
var collectGen2Size = !WildcardMatcher.IsAnyMatch(configurationReader.DisableMetrics,
GcMetricsProvider.GcGen2SizeName);
var collectGen3Size = !WildcardMatcher.IsAnyMatch(configurationReader.DisableMetrics,
GcMetricsProvider.GcGen3SizeName);
if (collectGcCount || collectGen0Size || collectGen1Size || collectGen2Size || collectGen3Size)
{
MetricsProviders.Add(new GcMetricsProvider(_logger, collectGcCount, collectGen0Size, collectGen1Size, collectGen2Size,
collectGen3Size));
}
_logger.Info()?.Log("Collecting metrics in {interval} milliseconds interval", interval);
_timer = new Timer(interval);
_timer.Elapsed += (sender, args) => { CollectAllMetrics(); };
}
/// <summary>
/// Extracts the request body using measure to prevent the 'read once' problem (cannot read after the body ha been already
/// read).
/// </summary>
/// <param name="request"></param>
/// <param name="logger"></param>
/// <returns></returns>
public static string ExtractRequestBody(this HttpRequest request, IApmLogger logger, IConfigSnapshot configSnapshot)
{
string body = null;
try
{
if (request.HasFormContentType)
{
var form = request.Form;
var itemProcessed = 0;
if (form != null && form.Count > 0)
{
var sb = new StringBuilder();
foreach (var item in form)
{
sb.Append(item.Key);
sb.Append("=");
if (WildcardMatcher.IsAnyMatch(configSnapshot.SanitizeFieldNames, item.Key))
{
sb.Append(Elastic.Apm.Consts.Redacted);
}
else
{
sb.Append(item.Value);
}
itemProcessed++;
if (itemProcessed != form.Count)
{
sb.Append("&");
}
}
body = sb.ToString();
}
}
else
{
request.EnableBuffering();
request.Body.Position = 0;
using (var reader = new StreamReader(request.Body,
Encoding.UTF8,
false,
1024 * 2,
true))
body = reader.ReadToEnd();
// Truncate the body to the first 2kb if it's longer
if (body.Length > Consts.RequestBodyMaxLength)
{
body = body.Substring(0, Consts.RequestBodyMaxLength);
}
request.Body.Position = 0;
}
}
catch (IOException ioException)
{
logger.Error()?.LogException(ioException, "IO Error reading request body");
}
catch (Exception e)
{
logger.Error()?.LogException(e, "Error reading request body");
}
return(body);
}
private bool IsSystemProcessCgroupMemoryMemLimitBytesEnabled(IReadOnlyList <WildcardMatcher> disabledMetrics) => !WildcardMatcher.IsAnyMatch(
disabledMetrics, SystemProcessCgroupMemoryMemLimitBytes);
private bool MatchesIgnoreMessageQueues(string routingKey)
{
return(WildcardMatcher.IsAnyMatch(_ApmAgent.ConfigurationReader?.IgnoreMessageQueues, routingKey));
}
/// <summary>
/// Extracts the request body, up to a specified maximum length.
/// The request body that is read is buffered.
/// </summary>
/// <param name="request">The request</param>
/// <param name="logger">The logger</param>
/// <param name="configuration">The configuration snapshot</param>
/// <returns></returns>
public static string ExtractRequestBody(this HttpRequest request, IApmLogger logger, IConfiguration configuration)
{
string body = null;
var longerThanMaxLength = false;
try
{
//Ensure the request Microsoft.AspNetCore.Http.HttpRequest.Body can be read multiple times
request.EnableBuffering();
if (request.HasFormContentType)
{
var form = request.Form;
var itemProcessed = 0;
if (form != null && form.Count > 0)
{
var sb = new StringBuilder();
foreach (var item in form)
{
sb.Append(item.Key);
sb.Append("=");
if (WildcardMatcher.IsAnyMatch(configuration.SanitizeFieldNames, item.Key))
{
sb.Append(Elastic.Apm.Consts.Redacted);
}
else
{
sb.Append(item.Value);
}
itemProcessed++;
if (itemProcessed != form.Count)
{
sb.Append("&");
}
// perf: check length once per iteration and truncate at the end, rather than each append
if (sb.Length > RequestBodyMaxLength)
{
longerThanMaxLength = true;
break;
}
}
body = sb.ToString(0, Math.Min(sb.Length, RequestBodyMaxLength));
}
}
else
{
// allow synchronous reading of the request stream, which is false by default from 3.0 onwards.
// Reading must be synchronous as it can happen within a synchronous diagnostic listener method
var bodyControlFeature = request.HttpContext.Features.Get <IHttpBodyControlFeature>();
if (bodyControlFeature != null)
{
bodyControlFeature.AllowSynchronousIO = true;
}
var requestBody = request.Body;
requestBody.Position = 0;
var arrayPool = ArrayPool <char> .Shared;
var capacity = 512;
var buffer = arrayPool.Rent(capacity);
var totalRead = 0;
int read;
// requestBody.Length is 0 on initial buffering - length relates to how much has been read and buffered.
// Read to just beyond request body max length so that we can determine if truncation will occur
try
{
// TODO: can we assume utf-8 encoding?
using var reader = new StreamReader(requestBody, Encoding.UTF8, false, buffer.Length, true);
while ((read = reader.Read(buffer, 0, capacity)) != 0)
{
totalRead += read;
if (totalRead > RequestBodyMaxLength)
{
longerThanMaxLength = true;
break;
}
}
}
finally
{
arrayPool.Return(buffer);
}
requestBody.Position = 0;
capacity = Math.Min(totalRead, RequestBodyMaxLength);
buffer = arrayPool.Rent(capacity);
try
{
using var reader = new StreamReader(requestBody, Encoding.UTF8, false, RequestBodyMaxLength, true);
read = reader.ReadBlock(buffer, 0, capacity);
body = new string(buffer, 0, read);
}
finally
{
arrayPool.Return(buffer);
}
requestBody.Position = 0;
}
}
catch (IOException ioException)
{
logger.Error()?.LogException(ioException, "IO Error reading request body");
}
catch (Exception e)
{
logger.Error()?.LogException(e, "Error reading request body");
}
if (longerThanMaxLength)
{
logger.Debug()?.Log("truncated body to max length {MaxLength}", RequestBodyMaxLength);
}
return(body);
}
internal static ITransaction StartTransactionAsync(HttpContext context, IApmLogger logger, ITracer tracer, IConfiguration configuration)
{
try
{
if (WildcardMatcher.IsAnyMatch(configuration?.TransactionIgnoreUrls, context.Request.Path))
{
logger.Debug()?.Log("Request ignored based on TransactionIgnoreUrls, url: {urlPath}", context.Request.Path);
return(null);
}
ITransaction transaction;
var transactionName = $"{context.Request.Method} {context.Request.Path}";
var containsTraceParentHeader =
context.Request.Headers.TryGetValue(TraceContext.TraceParentHeaderName, out var traceParentHeader);
var containsPrefixedTraceParentHeader = false;
if (!containsTraceParentHeader)
{
containsPrefixedTraceParentHeader = context.Request.Headers.TryGetValue(TraceContext.TraceParentHeaderNamePrefixed, out traceParentHeader);
}
if (containsPrefixedTraceParentHeader || containsTraceParentHeader)
{
var tracingData = context.Request.Headers.TryGetValue(TraceContext.TraceStateHeaderName, out var traceStateHeader)
? TraceContext.TryExtractTracingData(traceParentHeader, traceStateHeader)
: TraceContext.TryExtractTracingData(traceParentHeader);
if (tracingData != null)
{
logger.Debug()
?.Log(
"Incoming request with {TraceParentHeaderName} header. DistributedTracingData: {DistributedTracingData}. Continuing trace.",
containsPrefixedTraceParentHeader ? TraceContext.TraceParentHeaderNamePrefixed : TraceContext.TraceParentHeaderName,
tracingData);
transaction = tracer.StartTransaction(transactionName, ApiConstants.TypeRequest, tracingData);
}
else
{
logger.Debug()
?.Log(
"Incoming request with invalid {TraceParentHeaderName} header (received value: {TraceParentHeaderValue}). Starting trace with new trace id.",
containsPrefixedTraceParentHeader ? TraceContext.TraceParentHeaderNamePrefixed : TraceContext.TraceParentHeaderName,
traceParentHeader);
transaction = tracer.StartTransaction(transactionName, ApiConstants.TypeRequest);
}
}
else
{
logger.Debug()?.Log("Incoming request. Starting Trace.");
transaction = tracer.StartTransaction(transactionName, ApiConstants.TypeRequest);
}
return(transaction);
}
catch (Exception ex)
{
logger?.Error()?.LogException(ex, "Exception thrown while trying to start transaction");
return(null);
}
}
public GcMetricsProvider(IApmLogger logger, IReadOnlyList <WildcardMatcher> disabledMetrics)
{
_collectGcCount = !WildcardMatcher.IsAnyMatch(disabledMetrics, GcCountName);
_collectGcTime = !WildcardMatcher.IsAnyMatch(disabledMetrics, GcTimeName);
_collectGcGen0Size = !WildcardMatcher.IsAnyMatch(disabledMetrics, GcGen0SizeName);
_collectGcGen1Size = !WildcardMatcher.IsAnyMatch(disabledMetrics, GcGen1SizeName);
_collectGcGen2Size = !WildcardMatcher.IsAnyMatch(disabledMetrics, GcGen2SizeName);
_collectGcGen3Size = !WildcardMatcher.IsAnyMatch(disabledMetrics, GcGen3SizeName);
if (!IsEnabled(disabledMetrics))
{
return;
}
_logger = logger.Scoped(DbgName);
if (PlatformDetection.IsDotNetFullFramework)
{
// if the OS doesn't support filtering, the processes and runtimes tracked by the trace event session source can grow
// unbounded, leading to the application consuming ever more memory.
if (!TraceEventProviderOptions.FilteringSupported)
{
_logger.Info()?.Log("TraceEventSession does not support filtering on this operating system. GC metrics won't be collected");
return;
}
try
{
TraceEventSessionName = SessionNamePrefix + Guid.NewGuid();
_traceEventSession = new TraceEventSession(TraceEventSessionName);
_traceEventSession.Source.NeedLoadedDotNetRuntimes();
_traceEventSession.EnableProvider(
ClrTraceEventParser.ProviderGuid,
TraceEventLevel.Informational,
(ulong)ClrTraceEventParser.Keywords.GC, // garbage collector details
new TraceEventProviderOptions {
ProcessIDFilter = new List <int>(1)
{
Process.GetCurrentProcess().Id
}
}
);
_traceEventSession.Source.Clr.GCHeapStats += ClrOnGCHeapStats;
_traceEventSession.Source.AddCallbackOnProcessStart(process =>
{
process.AddCallbackOnDotNetRuntimeLoad(runtime =>
{
runtime.GCEnd += RuntimeGCEnd;
});
});
_traceEventSessionTask = Task.Run(() => _traceEventSession.Source.Process());
}
catch (Exception e)
{
_logger.Warning()?.LogException(e, "TraceEventSession initialization failed - GC metrics won't be collected");
return;
}
}
if (PlatformDetection.IsDotNetCore || PlatformDetection.IsDotNet)
{
_eventListener = new GcEventListener(this, logger);
}
}
private bool IsGcGen3SizeName(IReadOnlyList <WildcardMatcher> disabledMetrics) => !WildcardMatcher.IsAnyMatch(disabledMetrics, GcGen3SizeName);
public bool IsEnabled(IReadOnlyList <WildcardMatcher> disabledMetrics) => !WildcardMatcher.IsAnyMatch(disabledMetrics, ProcessCpuTotalPct);
private bool IsGcCountNameEnabled(IReadOnlyList <WildcardMatcher> disabledMetrics) => !WildcardMatcher.IsAnyMatch(disabledMetrics, GcCountName);
public MetricsCollector(IApmLogger logger, IPayloadSender payloadSender, IConfigSnapshotProvider configSnapshotProvider)
{
_logger = logger.Scoped(nameof(MetricsCollector));
_payloadSender = payloadSender;
_configSnapshotProvider = configSnapshotProvider;
var currentConfigSnapshot = configSnapshotProvider.CurrentSnapshot;
var interval = currentConfigSnapshot.MetricsIntervalInMilliseconds;
// ReSharper disable once CompareOfFloatsByEqualityOperator
if (interval == 0)
{
_logger.Info()?.Log("Collecting metrics is disabled - the agent won't collect metrics");
return;
}
MetricsProviders = new List <IMetricsProvider>();
if (!WildcardMatcher.IsAnyMatch(currentConfigSnapshot.DisableMetrics, ProcessTotalCpuTimeProvider.ProcessCpuTotalPct))
{
MetricsProviders.Add(new ProcessTotalCpuTimeProvider(_logger));
}
if (!WildcardMatcher.IsAnyMatch(currentConfigSnapshot.DisableMetrics, SystemTotalCpuProvider.SystemCpuTotalPct))
{
MetricsProviders.Add(new SystemTotalCpuProvider(_logger));
}
var collectProcessWorkingSet = !WildcardMatcher.IsAnyMatch(currentConfigSnapshot.DisableMetrics,
ProcessWorkingSetAndVirtualMemoryProvider.ProcessWorkingSetMemory);
var collectProcessVirtualMemory = !WildcardMatcher.IsAnyMatch(currentConfigSnapshot.DisableMetrics,
ProcessWorkingSetAndVirtualMemoryProvider.ProcessVirtualMemory);
if (collectProcessVirtualMemory || collectProcessWorkingSet)
{
MetricsProviders.Add(new ProcessWorkingSetAndVirtualMemoryProvider(collectProcessVirtualMemory, collectProcessWorkingSet));
}
var collectTotalMemory = !WildcardMatcher.IsAnyMatch(currentConfigSnapshot.DisableMetrics,
FreeAndTotalMemoryProvider.TotalMemory);
var collectFreeMemory = !WildcardMatcher.IsAnyMatch(currentConfigSnapshot.DisableMetrics,
FreeAndTotalMemoryProvider.FreeMemory);
if (collectFreeMemory || collectTotalMemory)
{
MetricsProviders.Add(new FreeAndTotalMemoryProvider(collectFreeMemory, collectTotalMemory));
}
var collectGcCount = !WildcardMatcher.IsAnyMatch(currentConfigSnapshot.DisableMetrics,
GcMetricsProvider.GcCountName);
var collectGen0Size = !WildcardMatcher.IsAnyMatch(currentConfigSnapshot.DisableMetrics,
GcMetricsProvider.GcGen0SizeName);
var collectGen1Size = !WildcardMatcher.IsAnyMatch(currentConfigSnapshot.DisableMetrics,
GcMetricsProvider.GcGen1SizeName);
var collectGen2Size = !WildcardMatcher.IsAnyMatch(currentConfigSnapshot.DisableMetrics,
GcMetricsProvider.GcGen2SizeName);
var collectGen3Size = !WildcardMatcher.IsAnyMatch(currentConfigSnapshot.DisableMetrics,
GcMetricsProvider.GcGen3SizeName);
var collectGcTime = !WildcardMatcher.IsAnyMatch(currentConfigSnapshot.DisableMetrics,
GcMetricsProvider.GcTimeName);
if (collectGcCount || collectGen0Size || collectGen1Size || collectGen2Size || collectGen3Size)
{
try
{
MetricsProviders.Add(new GcMetricsProvider(_logger, collectGcCount, collectGen0Size, collectGen1Size, collectGen2Size,
collectGen3Size, collectGcTime));
}
catch (Exception e)
{
_logger.Warning()?.LogException(e, "Failed loading {ProviderName}", nameof(GcMetricsProvider));
}
}
var collectCgroupMemLimitBytes = !WildcardMatcher.IsAnyMatch(currentConfigSnapshot.DisableMetrics,
CgroupMetricsProvider.SystemProcessCgroupMemoryMemLimitBytes);
var collectCgroupMemUsageBytes = !WildcardMatcher.IsAnyMatch(currentConfigSnapshot.DisableMetrics,
CgroupMetricsProvider.SystemProcessCgroupMemoryMemUsageBytes);
var collectCgroupStatsInactiveFileBytes = !WildcardMatcher.IsAnyMatch(currentConfigSnapshot.DisableMetrics,
CgroupMetricsProvider.SystemProcessCgroupMemoryStatsInactiveFileBytes);
if (collectCgroupMemLimitBytes || collectCgroupMemUsageBytes || collectCgroupStatsInactiveFileBytes)
{
MetricsProviders.Add(
new CgroupMetricsProvider(_logger, collectCgroupMemLimitBytes, collectCgroupMemUsageBytes,
collectCgroupStatsInactiveFileBytes));
}
_logger.Info()?.Log("Collecting metrics in {interval} milliseconds interval", interval);
_timer = new Timer(interval);
_timer.Elapsed += (sender, args) => { CollectAllMetrics(); };
}
private bool IsProcessWorkingSetMemoryEnabled(IReadOnlyList <WildcardMatcher> disabledMetrics) => !WildcardMatcher.IsAnyMatch(disabledMetrics, ProcessWorkingSetMemory);
private bool IsTotalMemoryEnabled(IReadOnlyList <WildcardMatcher> disabledMetrics) => !WildcardMatcher.IsAnyMatch(disabledMetrics, TotalMemory);
public bool IsEnabled(IReadOnlyList <WildcardMatcher> disabledMetrics) => WildcardMatcher.IsAnyMatch(disabledMetrics, BreakdownMetricsProvider.SpanSelfTime);