internal void StartDiagnosticObserversInternal()
{
DiagnosticManager?.Stop();
var observers = new List <DiagnosticObserver>();
#if NETSTANDARD
if (Settings.IsIntegrationEnabled(AspNetCoreDiagnosticObserver.IntegrationName))
{
DatadogLogging.RegisterStartupLog(log => log.Debug("Adding AspNetCoreDiagnosticObserver"));
var aspNetCoreDiagnosticOptions = new AspNetCoreDiagnosticOptions();
observers.Add(new AspNetCoreDiagnosticObserver(this, aspNetCoreDiagnosticOptions));
}
#endif
if (observers.Count == 0)
{
DatadogLogging.RegisterStartupLog(log => log.Debug("DiagnosticManager not started, zero observers added."));
}
else
{
DatadogLogging.RegisterStartupLog(log => log.Debug("Starting DiagnosticManager with {0} observers.", observers.Count));
var diagnosticManager = new DiagnosticManager(observers);
diagnosticManager.Start();
DiagnosticManager = diagnosticManager;
}
}
public static void StartProcesses()
{
try
{
_cancellationTokenSource = new CancellationTokenSource();
foreach (var subProcessMetadata in Processes)
{
var processPath = Environment.GetEnvironmentVariable(subProcessMetadata.ProcessPathKey);
if (!string.IsNullOrWhiteSpace(processPath))
{
var processArgs = Environment.GetEnvironmentVariable(subProcessMetadata.ProcessArgumentsKey);
subProcessMetadata.KeepAliveTask =
StartProcessWithKeepAlive(processPath, processArgs, subProcessMetadata);
}
else
{
DatadogLogging.RegisterStartupLog(log => log.Debug("There is no path configured for {0}.", subProcessMetadata.Name));
}
}
}
catch (Exception ex)
{
DatadogLogging.RegisterStartupLog(log => log.Error(ex, "Error when attempting to start standalone agent processes."));
}
}
private void RunShutdownTasks()
{
try
{
_agentWriter.FlushAndCloseAsync().Wait();
}
catch (Exception ex)
{
DatadogLogging.RegisterStartupLog(log => log.Error(ex, "Error flushing traces on shutdown."));
}
TracingProcessManager.StopProcesses();
}
private static void SafelyKillProcess(ProcessMetadata metadata)
{
try
{
if (metadata.Process != null && !metadata.Process.HasExited)
{
metadata.Process.Kill();
}
}
catch (Exception ex)
{
DatadogLogging.RegisterStartupLog(log => log.Error(ex, "Failed to verify halt of the {0} process.", metadata.Name));
}
}
internal void StartDiagnosticObservers()
{
// instead of adding a hard dependency on DiagnosticSource,
// check if it is available before trying to use it
var type = Type.GetType("System.Diagnostics.DiagnosticSource, System.Diagnostics.DiagnosticSource", throwOnError: false);
if (type == null)
{
DatadogLogging.RegisterStartupLog(log => log.Warning("DiagnosticSource type could not be loaded. Disabling diagnostic observers."));
}
else
{
// don't call this method unless the necessary types are available
StartDiagnosticObserversInternal();
}
}
public static void StopProcesses()
{
try
{
_cancellationTokenSource?.Cancel();
foreach (var subProcessMetadata in Processes)
{
SafelyKillProcess(subProcessMetadata);
}
}
catch (Exception ex)
{
DatadogLogging.RegisterStartupLog(log => log.Error(ex, "Error when cancelling processes."));
}
}
public Api(Uri baseEndpoint, DelegatingHandler delegatingHandler, IStatsd statsd)
{
DatadogLogging.RegisterStartupLog(log => log.Debug("Creating new Api"));
_tracesEndpoint = new Uri(baseEndpoint, TracesPath);
_statsd = statsd;
_client = delegatingHandler == null
? new HttpClient()
: new HttpClient(delegatingHandler);
_client.DefaultRequestHeaders.Add(AgentHttpHeaderNames.Language, ".NET");
// report runtime details
try
{
var frameworkDescription = FrameworkDescription.Create();
if (frameworkDescription != null)
{
_client.DefaultRequestHeaders.Add(AgentHttpHeaderNames.LanguageInterpreter, frameworkDescription.Name);
_client.DefaultRequestHeaders.Add(AgentHttpHeaderNames.LanguageVersion, frameworkDescription.ProductVersion);
}
}
catch (Exception e)
{
Log.Error(e, "Error getting framework description");
}
// report Tracer version
_client.DefaultRequestHeaders.Add(AgentHttpHeaderNames.TracerVersion, TracerConstants.AssemblyVersion);
// report container id (only Linux containers supported for now)
var containerId = ContainerInfo.GetContainerId();
if (containerId != null)
{
_client.DefaultRequestHeaders.Add(AgentHttpHeaderNames.ContainerId, containerId);
}
// don't add automatic instrumentation to requests from this HttpClient
_client.DefaultRequestHeaders.Add(HttpHeaderNames.TracingEnabled, "false");
}
private static int?GetFreeTcpPort()
{
TcpListener tcpListener = null;
try
{
tcpListener = new TcpListener(IPAddress.Loopback, 0);
tcpListener.Start();
var port = ((IPEndPoint)tcpListener.LocalEndpoint).Port;
return(port);
}
catch (Exception ex)
{
DatadogLogging.RegisterStartupLog(log => log.Error(ex, "Error trying to get a free port."));
return(null);
}
finally
{
tcpListener?.Stop();
}
}
/// <summary>
/// Gets an "application name" for the executing application by looking at
/// the hosted app name (.NET Framework on IIS only), assembly name, and process name.
/// </summary>
/// <returns>The default service name.</returns>
private static string GetApplicationName()
{
try
{
#if !NETSTANDARD2_0
// System.Web.dll is only available on .NET Framework
if (System.Web.Hosting.HostingEnvironment.IsHosted)
{
// if this app is an ASP.NET application, return "SiteName/ApplicationVirtualPath".
// note that ApplicationVirtualPath includes a leading slash.
return((System.Web.Hosting.HostingEnvironment.SiteName + System.Web.Hosting.HostingEnvironment.ApplicationVirtualPath).TrimEnd('/'));
}
#endif
return(Assembly.GetEntryAssembly()?.GetName().Name ??
Process.GetCurrentProcess().ProcessName);
}
catch (Exception ex)
{
DatadogLogging.RegisterStartupLog(log => log.Error(ex, "Error creating default service name."));
return(null);
}
}
internal Tracer(TracerSettings settings, IAgentWriter agentWriter, ISampler sampler, IScopeManager scopeManager, IStatsd statsd)
{
// update the count of Tracer instances
Interlocked.Increment(ref _liveTracerCount);
Settings = settings ?? TracerSettings.FromDefaultSources();
// if not configured, try to determine an appropriate service name
DefaultServiceName = Settings.ServiceName ??
GetApplicationName() ??
UnknownServiceName;
// only set DogStatsdClient if tracer metrics are enabled
if (Settings.TracerMetricsEnabled)
{
// Run this first in case the port override is ready
TracingProcessManager.SubscribeToDogStatsDPortOverride(
port =>
{
DatadogLogging.RegisterStartupLog(log => log.Debug("Attempting to override dogstatsd port with {0}", port));
Statsd = CreateDogStatsdClient(Settings, DefaultServiceName, port);
});
Statsd = statsd ?? CreateDogStatsdClient(Settings, DefaultServiceName, Settings.DogStatsdPort);
}
// Run this first in case the port override is ready
TracingProcessManager.SubscribeToTraceAgentPortOverride(
port =>
{
DatadogLogging.RegisterStartupLog(log => log.Debug("Attempting to override trace agent port with {0}", port));
var builder = new UriBuilder(Settings.AgentUri)
{
Port = port
};
var baseEndpoint = builder.Uri;
IApi overridingApiClient = new Api(baseEndpoint, delegatingHandler: null, Statsd);
if (_agentWriter == null)
{
_agentWriter = _agentWriter ?? new AgentWriter(overridingApiClient, Statsd);
}
else
{
_agentWriter.OverrideApi(overridingApiClient);
}
});
// fall back to default implementations of each dependency if not provided
_agentWriter = agentWriter ?? new AgentWriter(new Api(Settings.AgentUri, delegatingHandler: null, Statsd), Statsd);
_scopeManager = scopeManager ?? new AsyncLocalScopeManager();
Sampler = sampler ?? new RuleBasedSampler(new RateLimiter(Settings.MaxTracesSubmittedPerSecond));
if (!string.IsNullOrWhiteSpace(Settings.CustomSamplingRules))
{
// User has opted in, ensure rate limiter is used
RuleBasedSampler.OptInTracingWithoutLimits();
foreach (var rule in CustomSamplingRule.BuildFromConfigurationString(Settings.CustomSamplingRules))
{
Sampler.RegisterRule(rule);
}
}
if (Settings.GlobalSamplingRate != null)
{
var globalRate = (float)Settings.GlobalSamplingRate;
if (globalRate < 0f || globalRate > 1f)
{
DatadogLogging.RegisterStartupLog(log => log.Warning("{0} configuration of {1} is out of range", ConfigurationKeys.GlobalSamplingRate, Settings.GlobalSamplingRate));
}
else
{
Sampler.RegisterRule(new GlobalSamplingRule(globalRate));
}
}
// Register callbacks to make sure we flush the traces before exiting
AppDomain.CurrentDomain.ProcessExit += CurrentDomain_ProcessExit;
AppDomain.CurrentDomain.DomainUnload += CurrentDomain_DomainUnload;
AppDomain.CurrentDomain.UnhandledException += CurrentDomain_UnhandledException;
Console.CancelKeyPress += Console_CancelKeyPress;
// start the heartbeat loop
_heartbeatTimer = new Timer(HeartbeatCallback, state: null, dueTime: TimeSpan.Zero, period: TimeSpan.FromMinutes(1));
// If configured, add/remove the correlation identifiers into the
// LibLog logging context when a scope is activated/closed
if (Settings.LogsInjectionEnabled)
{
InitializeLibLogScopeEventSubscriber(_scopeManager);
}
}
private static Task StartProcessWithKeepAlive(string path, string args, ProcessMetadata metadata)
{
DatadogLogging.RegisterStartupLog(log => log.Debug("Starting keep alive for {0}.", path));
return(Task.Run(
() =>
{
try
{
var circuitBreakerMax = 3;
var sequentialFailures = 0;
while (true)
{
if (_cancellationTokenSource.IsCancellationRequested)
{
DatadogLogging.RegisterStartupLog(log => log.Debug("Shutdown triggered for keep alive {0}.", path));
return;
}
try
{
if (metadata.Process != null && metadata.Process.HasExited == false)
{
DatadogLogging.RegisterStartupLog(log => log.Debug("We already have an active reference to {0}.", path));
continue;
}
if (ProgramIsRunning(path))
{
DatadogLogging.RegisterStartupLog(log => log.Debug("{0} is already running.", path));
continue;
}
var startInfo = new ProcessStartInfo {
FileName = path
};
if (!string.IsNullOrWhiteSpace(args))
{
startInfo.Arguments = args;
}
DatadogLogging.RegisterStartupLog(log => log.Debug("Starting {0}.", path));
metadata.PreStartAction?.Invoke();
metadata.Process = Process.Start(startInfo);
Thread.Sleep(200);
if (metadata.Process == null || metadata.Process.HasExited)
{
DatadogLogging.RegisterStartupLog(log => log.Error("{0} has failed to start.", path));
sequentialFailures++;
}
else
{
DatadogLogging.RegisterStartupLog(log => log.Debug("Successfully started {0}.", path));
sequentialFailures = 0;
foreach (var portSubscriber in metadata.PortSubscribers)
{
portSubscriber(metadata.Port.Value);
}
DatadogLogging.RegisterStartupLog(log => log.Debug("Finished calling port subscribers for {0}.", metadata.Name));
}
}
catch (Exception ex)
{
DatadogLogging.RegisterStartupLog(log => log.Error(ex, "Exception when trying to start an instance of {0}.", path));
sequentialFailures++;
}
finally
{
// Delay for a reasonable amount of time before we check to see if the process is alive again.
Thread.Sleep(20_000);
}
if (sequentialFailures >= circuitBreakerMax)
{
DatadogLogging.RegisterStartupLog(log => log.Error("Circuit breaker triggered for {0}. Max failed retries reached ({1}).", path, sequentialFailures));
break;
}
}
}
finally
{
DatadogLogging.RegisterStartupLog(log => log.Debug("Keep alive is dropping for {0}.", path));
}
}));
}