public HerculesSinkMetricsCounters(IMetricContext context, HerculesSink herculesSink)
{
this.herculesSink = herculesSink;
tags = context.Tags;
registration = context.Register(this, null);
}
public static IMetricContext Get([NotNull] HerculesSinkSettings settings, [CanBeNull] ILog log)
{
if (defaultMetricContext != null)
{
return(defaultMetricContext);
}
lock (Sync)
{
if (defaultMetricContext != null)
{
return(defaultMetricContext);
}
var memoryLimit = 8 * 1024 * 1024;
var sinkSettings = new HerculesSinkSettings(settings.Cluster, settings.ApiKeyProvider)
{
AdditionalSetup = settings.AdditionalSetup,
MaximumMemoryConsumption = memoryLimit,
MaximumPerStreamMemoryConsumption = memoryLimit
};
log = (log ?? LogProvider.Get()).WithMinimumLevel(LogLevel.Warn);
var sink = new HerculesSink(sinkSettings, log);
var metricsSender = new HerculesMetricSender(new HerculesMetricSenderSettings(sink));
return(defaultMetricContext = new MetricContext(new MetricContextConfig(metricsSender)));
}
}
public HostingShutdown(
ApplicationShutdown appShutdown,
IServiceBeacon serviceBeacon,
IServiceLocator serviceLocator,
IVostokApplicationIdentity identity,
IMetricContext instanceMetrics,
ILog log,
CancellationToken token,
TimeSpan totalTimeout,
TimeSpan beaconTimeout,
bool beaconWaitEnabled,
bool sendAnnotation)
{
this.appShutdown = appShutdown;
this.serviceBeacon = serviceBeacon;
this.serviceLocator = serviceLocator;
this.identity = identity;
this.instanceMetrics = instanceMetrics;
this.log = log.ForContext <HostingShutdown>();
this.totalTimeout = totalTimeout;
this.beaconTimeout = beaconTimeout;
this.beaconWaitEnabled = beaconWaitEnabled;
this.sendAnnotation = sendAnnotation;
hostShutdownBudget = TimeBudget.CreateNew(totalTimeout);
tokenRegistration = token.Register(OnHostShutdownTriggered);
}
private HerculesSinkMetrics(HerculesSink herculesSink, IMetricContext context, ILog log)
{
this.log = log.ForContext <HerculesSink>();
this.herculesSink = herculesSink;
tags = context.Tags;
context.Register(this, ScrapePeriod);
}
/// <summary>
/// <para>Enables reporting of system metrics of the host.</para>
/// <para>Note that provided <see cref="IMetricContext"/> should contain tags sufficient to decouple these metrics from others.</para>
/// </summary>
public static IDisposable ReportMetrics(
[NotNull] this HostMetricsCollector collector,
[NotNull] IMetricContext metricContext,
TimeSpan?period = null)
=> metricContext.CreateMultiFuncGauge(
() => ProvideMetrics(collector),
new FuncGaugeConfig {
ScrapePeriod = period
}) as IDisposable;
public VostokApplicationMetrics(IMetricContext root, IVostokApplicationIdentity identity)
{
Root = root;
Project = Root.WithTag(WellKnownApplicationIdentityProperties.Project, identity.Project);
Subproject = identity.Subproject == null
? Project
: Project.WithTag(WellKnownApplicationIdentityProperties.Subproject, identity.Subproject);
Environment = Subproject.WithTag(WellKnownApplicationIdentityProperties.Environment, identity.Environment);
Application = Environment.WithTag(WellKnownApplicationIdentityProperties.Application, identity.Application);
Instance = Application.WithTag(WellKnownApplicationIdentityProperties.Instance, identity.Instance);
}
public ReportingObserver(IMetricContext metricContext, TimeSpan?period)
{
var config = new FloatingGaugeConfig
{
ResetOnScrape = true,
Unit = WellKnownUnits.Milliseconds,
ScrapePeriod = period
};
gcTotalDuration = metricContext.CreateFloatingGauge("GcTotalDurationMs", "GcType", config);
gcLongestDuration = metricContext.CreateFloatingGauge("GcLongestDurationMs", "GcType", config);
}
public ThrottlingMetrics(
[NotNull] IThrottlingProvider provider,
[NotNull] IMetricContext metricContext,
[NotNull] ThrottlingMetricsOptions options)
{
eventSubscription = provider.Subscribe(this as IObserver <IThrottlingEvent>);
resultSubscription = provider.Subscribe(this as IObserver <IThrottlingResult>);
var integerGaugeConfig = new IntegerGaugeConfig {
InitialValue = 0, ResetOnScrape = true
};
var propertiesGaugeConfig = new IntegerGaugeConfig {
InitialValue = 0, ResetOnScrape = true, SendZeroValues = false
};
var floatingGaugeConfig = new FloatingGaugeConfig {
InitialValue = 0, ResetOnScrape = true
};
var summaryConfig = new SummaryConfig();
var counterConfig = new CounterConfig();
if (options.ScrapePeriod != null)
{
integerGaugeConfig.ScrapePeriod = options.ScrapePeriod;
floatingGaugeConfig.ScrapePeriod = options.ScrapePeriod;
summaryConfig.ScrapePeriod = options.ScrapePeriod;
counterConfig.ScrapePeriod = options.ScrapePeriod;
counterConfig.AggregationParameters = new Dictionary <string, string>().SetAggregationPeriod(options.ScrapePeriod.Value);
}
waitTimeSummary = metricContext.CreateSummary("queueWaitTime", summaryConfig);
rejectionCounter = metricContext.CreateCounter("rejectionsCount", "status", counterConfig);
maxCapacityLimit = metricContext.CreateIntegerGauge("maxCapacityLimit", integerGaugeConfig);
maxCapacityConsumed = metricContext.CreateIntegerGauge("maxCapacityConsumed", integerGaugeConfig);
maxCapacityUtilization = metricContext.CreateFloatingGauge("maxCapacityUtilization", floatingGaugeConfig);
maxQueueLimit = metricContext.CreateIntegerGauge("maxQueueLimit", integerGaugeConfig);
maxQueueSize = metricContext.CreateIntegerGauge("maxQueueSize", integerGaugeConfig);
maxQueueUtilization = metricContext.CreateFloatingGauge("maxQueueUtilization", floatingGaugeConfig);
maxConsumptionPerProperty = options.PropertiesWithConsumptionTracking
.ToDictionary(
property => property,
property => metricContext
.WithTag("scope", "property")
.WithTag("propertyName", property)
.CreateIntegerGauge("maxConsumption", "propertyValue", propertiesGaugeConfig));
propertyConsumptionTrackingThreshold = options.PropertyConsumptionTrackingThreshold;
}
public void SetUp()
{
cancellationTokenSource = new CancellationTokenSource();
leftCoordinatesStorage = new InMemoryCoordinatesStorage();
rightCoordinatesStorage = new InMemoryCoordinatesStorage();
senderSettings = new HerculesMetricSenderSettings(Hercules.Instance.Sink);
var management = Hercules.Instance.Management;
var streamNames = new[] { senderSettings.FinalStream, senderSettings.TimersStream, senderSettings.CountersStream, senderSettings.HistogramsStream };
var streamCreateTasks = streamNames.Select(
name => Task.Run(
async() =>
{
var createResult = await management.CreateStreamAsync(
new CreateStreamQuery(name)
{
Partitions = streamPartitions,
ShardingKey = new[] { "tagsHash" }
},
20.Seconds());
createResult.EnsureSuccess();
}))
.ToArray();
Task.WaitAll(streamCreateTasks);
var herculesSender = new HerculesMetricSender(new HerculesMetricSenderSettings(Hercules.Instance.Sink));
testMetricSender = new TestsHelpers.TestMetricSender();
metricContext = new MetricContext(
new MetricContextConfig(
new CompositeMetricEventSender(
new IMetricEventSender[]
{
herculesSender,
testMetricSender
}))
{
ErrorCallback = error => log.Error(error)
});
aggregators = new List <AggregatorApplication>();
}
public VostokApplicationMetrics(IMetricContext root, IVostokApplicationIdentity identity)
{
Root = root;
Project = Root.WithTag(WellKnownApplicationIdentityProperties.Project, identity.Project);
Subproject = identity.Subproject == null
? Project
: Project.WithTag(WellKnownApplicationIdentityProperties.Subproject, identity.Subproject);
Environment = Subproject.WithTag(WellKnownApplicationIdentityProperties.Environment, identity.Environment);
Application = Environment.WithTag(WellKnownApplicationIdentityProperties.Application, identity.Application);
var instance = identity.Instance;
if (string.Equals(instance, EnvironmentInfo.Host, StringComparison.InvariantCultureIgnoreCase))
{
instance = instance.ToLowerInvariant();
}
Instance = Application.WithTag(WellKnownApplicationIdentityProperties.Instance, instance);
}
public static (HostingShutdown hosting, ApplicationShutdown application) Create(
IServiceBeacon serviceBeacon,
IServiceLocator serviceLocator,
IVostokApplicationIdentity identity,
IMetricContext instanceMetrics,
ILog log,
int?port,
IReadOnlyList <CancellationToken> tokens,
TimeSpan totalTimeout,
TimeSpan beaconTimeout,
bool beaconWaitEnabled,
bool sendAnnotation)
{
var hasRealBeacon = serviceBeacon is ServiceBeacon;
var hasRealLocator = serviceLocator is ServiceLocator;
// (iloktionov): No point in waiting for beacon deregistration for apps without external port or when SD is disabled.
beaconWaitEnabled &= port.HasValue && hasRealBeacon && hasRealLocator;
// (iloktionov): No point in reducing app shutdown timeout right from the start when SD is disabled.
beaconTimeout = hasRealBeacon ? TimeSpanArithmetics.Min(beaconTimeout, totalTimeout.Divide(3)) : TimeSpan.Zero;
// (iloktionov): Artificially reduce initial app shutdown timeout by beacon shutdown timeout so that it's value doesn't drop abruptly on shutdown.
var applicationShutdown = new ApplicationShutdown(log, totalTimeout - beaconTimeout);
var hostingToken = tokens.Any() ? CancellationTokenSource.CreateLinkedTokenSource(tokens.ToArray()).Token : default;
var hostingShutdown = new HostingShutdown(
applicationShutdown,
serviceBeacon,
serviceLocator,
identity,
instanceMetrics,
log,
hostingToken,
totalTimeout,
beaconTimeout,
beaconWaitEnabled,
sendAnnotation);
return(hostingShutdown, applicationShutdown);
}
public MetricsProvider(IMetricContext metricContext, string environment, string clientName)
{
var sendZeroValuesCounterConfig = new CounterConfig
{
ScrapePeriod = TimeSpan.FromMinutes(1),
SendZeroValues = false
};
var metricsTags = new MetricTags(
new MetricTag("project", SingularConstants.ProjectName),
new MetricTag("environment", environment),
new MetricTag("application", SingularConstants.ServiceName),
new MetricTag("cluster", SingularConstants.DefaultCluster),
new MetricTag("version", Version),
new MetricTag("clientName", clientName)
);
var metricsContext = metricContext.WithTags(metricsTags);
requestReasonsCounter = metricsContext.CreateCounter("client", "reason", sendZeroValuesCounterConfig);
}
private HealthCheckMetrics(IHealthTracker healthTracker, IMetricContext context)
{
this.context = context;
healthTracker.ObserveReports().Subscribe(this);
}
private HerculesSinkMetrics(HerculesSink herculesSink, IMetricContext context, ILog log)
{
this.log = log.ForContext <HerculesSink>();
this.herculesSink = herculesSink;
tags = context.Tags;
}
public MetricsModule(IMetricContext metricContext)
{
codesCounter = metricContext.CreateCounter("code", "value");
timer = metricContext.CreateTimer("time");
}
public static HerculesSinkMetricsCounters CreateHerculesSinkMetricsCounters([NotNull] this IMetricContext metricContext, [NotNull] HerculesSink herculesSink) => new HerculesSinkMetricsCounters(metricContext, herculesSink);
public static IDisposable ReportMetrics([NotNull] this GarbageCollectionMonitor monitor, [NotNull] IMetricContext metricContext) => monitor.Subscribe(new ReportingObserver(metricContext));
private void RegisterDnsMonitor(SystemMetricsSettings settings, BuildContext context, IMetricContext metricContext)
{
var dnsMonitor = new DnsMonitor();
context.HostExtensions.AsMutable().Add(dnsMonitor);
if (settings.EnableDnsEventsMetrics)
{
context.RegisterDisposable(dnsMonitor.ReportMetrics(metricContext, settings.ProcessMetricsReportingPeriod));
}
context.RegisterDisposable(dnsMonitor);
}
private void RegisterProcessMonitor(SystemMetricsSettings settings, BuildContext context, IMetricContext metricContext)
{
var processMonitor = new CurrentProcessMonitor();
context.HostExtensions.AsMutable().Add(processMonitor);
if (settings.EnableProcessMetricsLogging)
{
context.DisposableHostExtensions.Add(processMonitor.LogPeriodically(context.Log, settings.ProcessMetricsLoggingPeriod));
}
if (settings.EnableProcessMetricsReporting)
{
var collectorSettings = new CurrentProcessMetricsSettings
{
CpuCoresLimitProvider = () => context.ApplicationLimits.CpuUnits,
MemoryBytesLimitProvider = () => context.ApplicationLimits.MemoryBytes
};
var collector = new CurrentProcessMetricsCollector(collectorSettings);
collector.ReportMetrics(metricContext);
}
}
public static void ReportLaunching(IVostokApplicationIdentity identity, IMetricContext context)
=> context.SendAnnotation($"Instance {identity.Instance} is launching (just started) on host {EnvironmentInfo.Host}.",
(HostTag, EnvironmentInfo.Host), (EventTypeTag, "Launching"));
public static IDisposable ReportMetrics([NotNull] this GarbageCollectionMonitor monitor, [NotNull] IMetricContext metricContext, [CanBeNull] TimeSpan?period) => monitor.Subscribe(new ReportingObserver(metricContext, period));
// ReSharper disable once IntroduceOptionalParameters.Local
public ReportingObserver(IMetricContext metricContext) : this(metricContext, null)
{
}
public static MetricsProvider Get(IMetricContext metricContext, string environment, string clientName)
{
return(Cache.GetOrAdd(Tuple.Create(metricContext, environment, clientName), t => new MetricsProvider(t.Item1, t.Item2, t.Item3)));
}
public static IDisposable CreateThrottlingMetrics(
[NotNull] this IMetricContext context,
[NotNull] IThrottlingProvider provider,
[CanBeNull] ThrottlingMetricsOptions options = null)
=> new ThrottlingMetrics(provider, context.WithTag(WellKnownTagKeys.Component, "throttling"), options ?? new ThrottlingMetricsOptions());
public LogLevelMetrics(LogEventLevelCounter counter, IMetricContext context)
{
this.counter = counter;
context.CreateMultiFuncGauge(ProvideMetrics);
}
public static HerculesSinkSettings SetupMetrics([NotNull] this HerculesSinkSettings settings, [NotNull] IMetricContext metricContext)
{
var oldSetup = settings.AdditionalSetup;
settings.AdditionalSetup = setup =>
{
oldSetup?.Invoke(setup);
var newModule = new MetricsModule(metricContext);
var oldModules = setup.Modules.Values.SelectMany(v => v.Before);
var oldModule = oldModules.OfType <MetricsModule>().FirstOrDefault();
if (oldModule != null)
{
oldModule.AddNestedModule(newModule);
}
else
{
setup.AddRequestModule(newModule);
}
};
return(settings);
}
private void RegisterGcMonitor(SystemMetricsSettings settings, BuildContext context, IMetricContext metricContext)
{
var gcMonitor = new GarbageCollectionMonitor();
context.HostExtensions.AsMutable().Add(gcMonitor);
context.DisposableHostExtensions.Add(gcMonitor);
if (settings.EnableGcEventsLogging)
{
context.DisposableHostExtensions.Add(gcMonitor.LogCollections(context.Log, gc => gc.Duration >= settings.GcMinimumDurationForLogging));
}
if (settings.EnableGcEventsMetrics)
{
context.DisposableHostExtensions.Add(gcMonitor.ReportMetrics(metricContext));
}
}
public static IDisposable ReportMetrics([NotNull] this DnsMonitor monitor, [NotNull] IMetricContext metricContext, TimeSpan?period = null) => monitor.Subscribe(new ReportingObserver(metricContext, period));
private void RegisterHostMonitor(SystemMetricsSettings settings, BuildContext context, IMetricContext metricContext)
{
var hostMonitor = new HostMonitor();
context.HostExtensions.AsMutable().Add(hostMonitor);
if (settings.EnableHostMetricsLogging)
{
context.DisposableHostExtensions.Add(hostMonitor.LogPeriodically(context.Log, settings.HostMetricsLoggingPeriod));
}
if (settings.EnableHostMetricsReporting)
{
new HostMetricsCollector().ReportMetrics(metricContext);
}
}
public ReportingObserver(IMetricContext metricContext, TimeSpan?period)
{
lookupLatency = metricContext.CreateSummary("DnsLookupLatency", new SummaryConfig {
Unit = WellKnownUnits.Milliseconds, ScrapePeriod = period
});
}
