public SamplingPriority GetSamplingPriority(Span span)
{
var traceId = span.TraceId;
if (_rules.Count > 0)
{
foreach (var rule in _rules)
{
if (rule.IsMatch(span))
{
var sampleRate = rule.GetSamplingRate(span);
Log.Debug(
"Matched on rule {0}. Applying rate of {1} to trace id {2}",
rule.RuleName,
sampleRate,
traceId);
return(GetSamplingPriority(span, sampleRate));
}
}
}
Log.Debug("No rules matched for trace {0}", traceId);
return(SamplingPriority.AutoKeep);
}
public void WriteTrace(Span[] trace)
{
if (_synchronousSend)
{
_api.SendTracesAsync(new[] { trace }).Wait();
return;
}
var success = _tracesBuffer.Push(trace);
if (!success)
{
Log.Debug("Trace buffer is full. Dropping a trace from the buffer.");
}
if (_statsd != null)
{
_statsd.AppendIncrementCount(TracerMetricNames.Queue.EnqueuedTraces);
_statsd.AppendIncrementCount(TracerMetricNames.Queue.EnqueuedSpans, trace.Length);
if (!success)
{
_statsd.AppendIncrementCount(TracerMetricNames.Queue.DroppedTraces);
_statsd.AppendIncrementCount(TracerMetricNames.Queue.DroppedSpans, trace.Length);
}
_statsd.Send();
}
}
private void OnHostingHttpRequestInStart(object arg)
{
var httpContext = (HttpContext)HttpRequestInStartHttpContextFetcher.Fetch(arg);
if (ShouldIgnore(httpContext))
{
if (_isLogLevelDebugEnabled)
{
Log.Debug("Ignoring request");
}
}
else
{
HttpRequest request = httpContext.Request;
string host = request.Host.Value;
string httpMethod = request.Method?.ToUpperInvariant() ?? "UNKNOWN";
string url = GetUrl(request);
string resourceUrl = UriHelpers.GetRelativeUrl(new Uri(url), tryRemoveIds: true)
.ToLowerInvariant();
var propagator = _tracer.Propagator;
SpanContext propagatedContext = ExtractPropagatedContext(propagator, request);
Span span = _tracer.StartSpan(HttpRequestInOperationName, propagatedContext)
.SetTag(Tags.InstrumentationName, ComponentName);
IPAddress remoteIp = null;
if (Tracing.Tracer.Instance.Settings.AddClientIpToServerSpans)
{
remoteIp = httpContext?.Connection?.RemoteIpAddress;
}
span.DecorateWebServerSpan(resourceUrl, httpMethod, host, url, remoteIp);
span.SetTag(Tags.InstrumentationName, IntegrationName);
// set analytics sample rate if enabled
var analyticsSampleRate = _tracer.Settings.GetIntegrationAnalyticsSampleRate(IntegrationName, enabledWithGlobalSetting: true);
span.SetMetric(Tags.Analytics, analyticsSampleRate);
Scope scope = _tracer.ActivateSpan(span);
_options.OnRequest?.Invoke(scope.Span, httpContext);
}
}
public float GetSamplingRate(Span span)
{
Log.Debug("Using the default sampling logic");
var env = span.GetTag(Tags.Environment);
var service = span.ServiceName;
var key = $"service:{service},env:{env}";
if (_sampleRates.TryGetValue(key, out var sampleRate))
{
span.SetMetric(Metrics.SamplingAgentDecision, sampleRate);
return(sampleRate);
}
Log.Debug("Could not establish sample rate for trace {0}", span.TraceId);
return(1);
}
internal Span(SpanContext context, DateTimeOffset?start)
{
Context = context;
Context.Span = this;
ServiceName = context.ServiceName;
StartTime = start ?? Context.TraceContext.UtcNow;
Logger.Debug(
"Span started: [s_id: {0}, p_id: {1}, t_id: {2}]",
SpanId,
Context.ParentId,
TraceId);
}
public virtual IDisposable SubscribeIfMatch(DiagnosticListener diagnosticListener)
{
if (ListenerNames.Any(diagnosticListener.Name.Contains))
{
return(diagnosticListener.Subscribe(this, IsEventEnabled));
}
else
{
Log.Debug($"{diagnosticListener.Name} not subscribing to {this.GetType().Name}.");
}
return(null);
}
public ISpanBuilder AddReference(string referenceType, global::OpenTracing.ISpanContext referencedContext)
{
lock (_lock)
{
if (referenceType == References.ChildOf)
{
_parent = referencedContext;
return(this);
}
}
Log.Debug("ISpanBuilder.AddReference is not implemented for other references than ChildOf by SignalFx.Tracing");
return(this);
}
public async Task SendTracesAsync(Span[][] traces)
{
// retry up to 5 times with exponential back-off
var retryLimit = 5;
var retryCount = 1;
var sleepDuration = 100; // in milliseconds
while (true)
{
HttpResponseMessage responseMessage;
try
{
// re-create content on every retry because some versions of HttpClient always dispose of it, so we can't reuse.
using (var content = new ZipkinContent(traces, _settings))
{
responseMessage = await _client.PostAsync(_settings.EndpointUrl, content).ConfigureAwait(false);
responseMessage.EnsureSuccessStatusCode();
return;
}
}
catch (Exception ex)
{
if (ex.InnerException is InvalidOperationException ioe)
{
Log.Error("A fatal error occurred while sending traces to {Endpoint}\n{Exception}", _settings.EndpointUrl, ex.Message);
return;
}
if (retryCount >= retryLimit)
{
// stop retrying
Log.Error("No more retries to send traces to {Endpoint}\n{Exception}", _settings.EndpointUrl, ex.Message);
return;
}
Log.Debug("Error sending traces to {Endpoint}\n{Exception}", _settings.EndpointUrl, ex.Message);
// retry
await Task.Delay(sleepDuration).ConfigureAwait(false);
retryCount++;
sleepDuration *= 2;
}
}
}
public bool Allowed(Span span)
{
try
{
if (_maxTracesPerInterval == 0)
{
// Rate limit of 0 blocks everything
return(false);
}
if (_maxTracesPerInterval < 0)
{
// Negative rate limit disables rate limiting
return(true);
}
WaitForRefresh();
// This must happen after the wait, because we check for window statistics, modifying this number
Interlocked.Increment(ref _windowChecks);
var count = _intervalQueue.Count;
if (count >= _maxTracesPerInterval)
{
Log.Debug("Dropping trace id {0} with count of {1} for last {2}ms.", span.TraceId, count, _intervalMilliseconds);
return(false);
}
_intervalQueue.Enqueue(DateTime.Now);
Interlocked.Increment(ref _windowAllowed);
return(true);
}
finally
{
// Always set the sample rate metric whether it was allowed or not
// DEV: Setting this allows us to properly compute metrics and debug the
// various sample rates that are getting applied to this span
span.SetMetric(Metrics.SamplingLimitDecision, GetEffectiveRate());
}
}
internal Tracer(TracerSettings settings, IAgentWriter agentWriter, ISampler sampler, IScopeManager scopeManager, IStatsd statsd)
{
// update the count of Tracer instances
Interlocked.Increment(ref _liveTracerCount);
Settings = settings ?? TracerSettings.FromDefaultSources();
// if not configured, try to determine an appropriate service name
DefaultServiceName = Settings.ServiceName ??
GetApplicationName() ??
UnknownServiceName;
// only set DogStatsdClient if tracer metrics are enabled
if (Settings.TracerMetricsEnabled)
{
// Run this first in case the port override is ready
TracingProcessManager.SubscribeToDogStatsDPortOverride(
port =>
{
Log.Debug("Attempting to override dogstatsd port with {0}", port);
Statsd = CreateDogStatsdClient(Settings, DefaultServiceName, port);
});
Statsd = statsd ?? CreateDogStatsdClient(Settings, DefaultServiceName, Settings.DogStatsdPort);
}
IApi apiClient = null;
if (agentWriter == null)
{
if (Settings.ApiType.ToLower().Equals("zipkin"))
{
apiClient = new ZipkinApi(Settings, delegatingHandler: null);
}
}
_agentWriter = agentWriter ?? new AgentWriter(apiClient, Statsd, Settings.SynchronousSend);
_scopeManager = scopeManager ?? new AsyncLocalScopeManager();
Sampler = sampler ?? new RuleBasedSampler(new RateLimiter(Settings.MaxTracesSubmittedPerSecond));
if (!string.IsNullOrWhiteSpace(Settings.CustomSamplingRules))
{
// User has opted in, ensure rate limiter is used
RuleBasedSampler.OptInTracingWithoutLimits();
foreach (var rule in CustomSamplingRule.BuildFromConfigurationString(Settings.CustomSamplingRules))
{
Sampler.RegisterRule(rule);
}
}
if (Settings.GlobalSamplingRate != null)
{
var globalRate = (float)Settings.GlobalSamplingRate;
if (globalRate < 0f || globalRate > 1f)
{
Log.Warning("{0} configuration of {1} is out of range", ConfigurationKeys.GlobalSamplingRate, Settings.GlobalSamplingRate);
}
else
{
Sampler.RegisterRule(new GlobalSamplingRule(globalRate));
}
}
// Register callbacks to make sure we flush the traces before exiting
AppDomain.CurrentDomain.ProcessExit += CurrentDomain_ProcessExit;
AppDomain.CurrentDomain.DomainUnload += CurrentDomain_DomainUnload;
AppDomain.CurrentDomain.UnhandledException += CurrentDomain_UnhandledException;
Console.CancelKeyPress += Console_CancelKeyPress;
// start the heartbeat loop
_heartbeatTimer = new Timer(HeartbeatCallback, state: null, dueTime: TimeSpan.Zero, period: TimeSpan.FromMinutes(1));
// If configured, add/remove the correlation identifiers into the
// LibLog logging context when a scope is activated/closed
if (Settings.LogsInjectionEnabled)
{
InitializeLibLogScopeEventSubscriber(_scopeManager);
}
}
public float GetSamplingRate(Span span)
{
Log.Debug("Using the global sampling rate: {0}", _globalRate);
span.SetMetric(Metrics.SamplingRuleDecision, _globalRate);
return(_globalRate);
}