private static void ListProcesses()
{
foreach (var processId in DiagnosticsClient.GetPublishedProcesses())
{
Console.WriteLine($"Process Id {processId} Name {Process.GetProcessById(processId).ProcessName}");
}
}
/// <summary>
/// Print the current list of available .NET core processes for diagnosis, their statuses and the command line arguments that are passed to them.
/// </summary>
public static void PrintProcessStatus(IConsole console)
{
try
{
StringBuilder sb = new StringBuilder();
var processes = DiagnosticsClient.GetPublishedProcesses()
.Select(GetProcessById)
.Where(process => process != null)
.OrderBy(process => process.ProcessName)
.ThenBy(process => process.Id);
foreach (var process in processes)
{
try
{
String cmdLineArgs = GetArgs(process);
cmdLineArgs = cmdLineArgs == process.MainModule.FileName ? "" : cmdLineArgs;
sb.Append($"{process.Id, 10} {process.ProcessName, -10} {process.MainModule.FileName, -10} {cmdLineArgs, -10}\n");
}
catch (Exception ex) when(ex is Win32Exception || ex is InvalidOperationException)
{
sb.Append($"{process.Id, 10} {process.ProcessName, -10} [Elevated process - cannot determine path] [Elevated process - cannot determine commandline arguments]\n");
}
}
console.Out.WriteLine(sb.ToString());
}
catch (InvalidOperationException ex)
{
console.Out.WriteLine(ex.ToString());
}
}
private static int?SelectProcess(ReplicaInfo replicaInfo)
{
var processIds = DiagnosticsClient.GetPublishedProcesses();
var processes = processIds.Select(pid =>
{
try
{
return(Process.GetProcessById(pid));
}
catch (Exception) // Can fail due to timing.
{
return(null);
}
})
.Where(p => p is object)
.ToArray();
try
{
return(replicaInfo.Selector.Invoke(processes !)?.Id);
}
finally
{
foreach (var process in processes !)
{
process !.Dispose();
}
}
}
public async Task <IEnumerable <IEndpointInfo> > GetEndpointInfoAsync(CancellationToken token)
{
var endpointInfoTasks = new List <Task <EndpointInfo> >();
// Run the EndpointInfo creation parallel. The call to FromProcessId sends
// a GetProcessInfo command to the runtime instance to get additional information.
foreach (int pid in DiagnosticsClient.GetPublishedProcesses())
{
endpointInfoTasks.Add(Task.Run(() =>
{
try
{
return(EndpointInfo.FromProcessId(pid));
}
//Catch when the application is running a more privilaged socket than dotnet-monitor. For example, running a web app as administrator
//while running dotnet-monitor without elevation.
catch (UnauthorizedAccessException)
{
return(null);
}
}, token));
}
await Task.WhenAll(endpointInfoTasks);
return(endpointInfoTasks.Where(t => t.Result != null).Select(t => t.Result));
}
/// <summary>
/// Print the current list of available .NET core processes for diagnosis and their statuses
/// </summary>
public static void PrintProcessStatus(IConsole console)
{
try
{
StringBuilder sb = new StringBuilder();
var processes = DiagnosticsClient.GetPublishedProcesses()
.Select(GetProcessById)
.Where(process => process != null)
.OrderBy(process => process.ProcessName)
.ThenBy(process => process.Id);
foreach (var process in processes)
{
try
{
sb.Append($"{process.Id, 10} {process.ProcessName, -10} {process.MainModule.FileName}\n");
}
catch (InvalidOperationException)
{
sb.Append($"{process.Id, 10} {process.ProcessName, -10} [Elevated process - cannot determine path]\n");
}
}
console.Out.WriteLine(sb.ToString());
}
catch (InvalidOperationException ex)
{
console.Out.WriteLine(ex.ToString());
}
}
private void Refresh()
{
var processes = DiagnosticsClient.GetPublishedProcesses()
.Select(p => Process.GetProcessById(p))
.ToList();
lvProcesses.ItemsSource = processes;
}
public static IList <Process> GetProcessList()
{
return(DiagnosticsClient.GetPublishedProcesses()
.Select(GetProcessById)
.Where(process => process != null)
.OrderBy(process => process.ProcessName)
.ThenBy(process => process.Id)
.ToList());
}
private static void ListProcesses()
{
var selfPid = Process.GetCurrentProcess().Id;
foreach (var pid in DiagnosticsClient.GetPublishedProcesses())
{
var process = Process.GetProcessById(pid);
Console.WriteLine($"{pid,6}{GetSeparator(pid == selfPid)}{process.ProcessName}");
}
}
static void Main(string[] args)
{
// Find the process containing the target EventSource.
var targetProcess = DiagnosticsClient.GetPublishedProcesses()
.Select(Process.GetProcessById)
.FirstOrDefault(process => process?.ProcessName == "Metrics");
if (targetProcess == null)
{
Console.WriteLine("No process named 'Metrics' found. Exiting.");
return;
}
// Define what EventSource and events to listen to.
var providers = new List <EventPipeProvider>()
{
new EventPipeProvider("My-CustomMetricsEventSource-Minimal",
EventLevel.Informational, arguments: new Dictionary <string, string>
{
{ "EventCounterIntervalSec", "1" }
})
};
// Start listening session
var client = new DiagnosticsClient(targetProcess.Id);
using var session = client.StartEventPipeSession(providers, false);
using var source = new EventPipeEventSource(session.EventStream);
// Set up output writer
source.Dynamic.All += obj =>
{
if (obj.EventName == "EventCounters")
{
var payload = (IDictionary <string, object>)obj.PayloadValue(0);
Console.WriteLine(string.Join(", ", payload.Select(p => $"{p.Key}: {p.Value}")));
}
else
{
Console.WriteLine($"{obj.ProviderName}: {obj.EventName}");
}
};
try
{
source.Process();
}
catch (Exception e)
{
Console.WriteLine("Error encountered while processing events");
Console.WriteLine(e.ToString());
}
Console.ReadKey();
}
public static void PrintProcessStatus()
{
var processes = DiagnosticsClient.GetPublishedProcesses()
.Select(Process.GetProcessById)
.Where(process => process != null);
foreach (var process in processes)
{
Console.WriteLine($"{process.ProcessName}");
}
}
private static int FindPidByAppName(string appName)
{
var process = DiagnosticsClient
.GetPublishedProcesses()
.Select(GetProcessById).FirstOrDefault(p => p != null && p.ProcessName.ToLower() == appName.ToLower());
if (process != null)
{
Console.WriteLine("{0} {1} {2}", process.ProcessName, process.Id, process.MainModule.FileName);
}
return(process?.Id ?? -1);
}
public IEnumerable <int> GetProcesses()
{
try
{
//TODO This won't work properly with multi-container scenarios that don't share the process space.
//TODO We will need to use DiagnosticsAgent if we are the server.
return(DiagnosticsClient.GetPublishedProcesses());
}
catch (UnauthorizedAccessException)
{
throw new InvalidOperationException("Unable to enumerate processes.");
}
}
public Task <IEnumerable <IEndpointInfo> > GetEndpointInfoAsync(CancellationToken token)
{
List <IEndpointInfo> endpointInfos = new List <IEndpointInfo>();
foreach (int pid in DiagnosticsClient.GetPublishedProcesses())
{
// CONSIDER: Generate a "runtime instance identifier" based on the pipe name
// e.g. pid + disambiguator in GUID form.
endpointInfos.Add(new EndpointInfo(pid));
}
return(Task.FromResult(endpointInfos.AsEnumerable()));
}
static void PrintProcessStatus()
{
var processes = DiagnosticsClient.GetPublishedProcesses().Select(Process.GetProcessById).Where(process => process != null);
foreach (var process in processes)
{
Console.WriteLine($"ProcessId: {process.Id}");
Console.WriteLine($"ProcessName: {process.ProcessName}");
Console.WriteLine($"StartTime: {process.StartTime}");
Console.WriteLine($"Threads: {process.Threads.Count}");
Console.WriteLine();
Console.WriteLine();
}
}
public async Task <IEnumerable <IEndpointInfo> > GetEndpointInfoAsync(CancellationToken token)
{
var endpointInfoTasks = new List <Task <EndpointInfo> >();
// Run the EndpointInfo creation parallel. The call to FromProcessId sends
// a GetProcessInfo command to the runtime instance to get additional information.
foreach (int pid in DiagnosticsClient.GetPublishedProcesses())
{
endpointInfoTasks.Add(Task.Run(() => EndpointInfo.FromProcessId(pid)));
}
await Task.WhenAll(endpointInfoTasks);
return(endpointInfoTasks.Select(t => t.Result));
}
/// <summary>
/// 获取进程状态：.Net Core 3.0及以上进程
/// </summary>
private void PrintProcessStatus()
{
int row = dgvPros.CurrentCell == null ? 0 : dgvPros.CurrentCell.RowIndex;
int col = dgvPros.CurrentCell == null ? 0 : dgvPros.CurrentCell.ColumnIndex;
var data = DiagnosticsClient.GetPublishedProcesses()
.Select(Process.GetProcessById)
.Where(process => process != null)
.Select(o => { return(new { o.Id, o.ProcessName, o.StartTime, o.Threads.Count }); });
dgvPros.DataSource = data.ToList();
if (dgvPros.Rows.Count > row)
{
dgvPros.CurrentCell = dgvPros.Rows[row].Cells[col];
}
}
public static void PrintProcessStatus(IConsole console)
{
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine("Process with Event Pipe available for Diagnostics");
Console.ResetColor();
// Ignore current process from the list
int currentProcess = Process.GetCurrentProcess().Id;
var processes = DiagnosticsClient.GetPublishedProcesses()
.Select(GetProcessById)
.Where(process => process != null && process.Id != currentProcess);
foreach (var process in processes)
{
Console.WriteLine($"\t{process.Id} {process.ProcessName} {process.MainModule.FileName}");
}
}
public async Task <IEnumerable <IEndpointInfo> > GetEndpointInfoAsync(CancellationToken token)
{
using CancellationTokenSource timeoutTokenSource = new();
using CancellationTokenSource linkedTokenSource =
CancellationTokenSource.CreateLinkedTokenSource(token, timeoutTokenSource.Token);
CancellationToken timeoutToken = timeoutTokenSource.Token;
CancellationToken linkedToken = linkedTokenSource.Token;
var endpointInfoTasks = new List <Task <EndpointInfo> >();
// Run the EndpointInfo creation parallel. The call to FromProcessId sends
// a GetProcessInfo command to the runtime instance to get additional information.
foreach (int pid in DiagnosticsClient.GetPublishedProcesses())
{
endpointInfoTasks.Add(Task.Run(async() =>
{
try
{
return(await EndpointInfo.FromProcessIdAsync(pid, linkedToken));
}
// Catch when timeout on waiting for EndpointInfo creation. Some runtime instances may be
// in a bad state and not respond to any requests on their diagnostic pipe; gracefully abandon
// waiting for these processes.
catch (OperationCanceledException) when(timeoutToken.IsCancellationRequested)
{
_logger.DiagnosticRequestCancelled(pid);
return(null);
}
// Catch all other exceptions and log them.
catch (Exception ex)
{
_logger.DiagnosticRequestFailed(pid, ex);
return(null);
}
}, linkedToken));
}
timeoutTokenSource.CancelAfter(AbandonProcessTimeout);
await Task.WhenAll(endpointInfoTasks);
return(endpointInfoTasks.Where(t => t.Result != null).Select(t => t.Result));
}
/// <summary>
/// Print the current list of available .NET core processes for diagnosis and their statuses
/// </summary>
public static void PrintProcessStatus(IConsole console)
{
try
{
StringBuilder sb = new StringBuilder();
var processes = DiagnosticsClient.GetPublishedProcesses()
.Select(GetProcessById)
.Where(process => process != null)
.OrderBy(process => process.ProcessName)
.ThenBy(process => process.Id);
var currentPid = Process.GetCurrentProcess().Id;
foreach (var process in processes)
{
if (process.Id == currentPid)
{
continue;
}
try
{
sb.Append($"{process.Id, 10} {process.ProcessName, -10} {process.MainModule.FileName}\n");
}
catch (Exception ex)
{
if (ex is System.ComponentModel.Win32Exception || ex is NullReferenceException)
{
sb.Append($"{process.Id, 10} {process.ProcessName, -10} [Elevated process - cannot determine path]\n");
}
else
{
Debug.WriteLine($"[PrintProcessStatus] {ex.ToString()}");
}
}
}
console.Out.WriteLine(sb.ToString());
}
catch (InvalidOperationException ex)
{
console.Out.WriteLine(ex.ToString());
}
}
private static (int pid, string processName) GetProcessInfo(string processIdOrName)
{
if (string.IsNullOrWhiteSpace(processIdOrName))
{
throw new ArgumentException("Missing process ID/name.");
}
if (int.TryParse(processIdOrName, out var parsedId))
{
try
{
using var process = System.Diagnostics.Process.GetProcessById(parsedId);
return(parsedId, process.ProcessName);
}
catch (ArgumentException)
{
throw new ArgumentException($"Could not find process with ID '{parsedId}'.");
}
}
foreach (var pid in DiagnosticsClient.GetPublishedProcesses())
{
try
{
using var process = System.Diagnostics.Process.GetProcessById(pid);
if (process.ProcessName.Contains(processIdOrName, StringComparison.InvariantCultureIgnoreCase))
{
return(pid, process.ProcessName);
}
}
catch (ArgumentException)
{
// ignore - just try the next process
}
}
throw new ArgumentException($"Could not find process with name '{processIdOrName}'.");
}
// Returns processId that matches the given name.
// It also checks whether the process has a diagnostics server port.
// If there are more than 1 process with the given name or there isn't any active process
// with the given name, then this returns -1
public static int FindProcessIdWithName(string name)
{
var publishedProcessesPids = new List <int>(DiagnosticsClient.GetPublishedProcesses());
var processesWithMatchingName = Process.GetProcessesByName(name);
var commonId = -1;
for (int i = 0; i < processesWithMatchingName.Length; i++)
{
if (publishedProcessesPids.Contains(processesWithMatchingName[i].Id))
{
if (commonId != -1)
{
Console.WriteLine("There are more than one active processes with the given name: {0}", name);
return(-1);
}
commonId = processesWithMatchingName[i].Id;
}
}
if (commonId == -1)
{
Console.WriteLine("There is no active process with the given name: {0}", name);
}
return(commonId);
}
public Task <ProcessInfo[]> GetAllProcessesInfo()
{
return(Task.Run(() =>
{
var processes = DiagnosticsClient.GetPublishedProcesses()
.Select(GetProcessById)
.Where(process => process != null)
.OrderBy(process => process.ProcessName)
.ThenBy(process => process.Id);
var procInfos = new List <ProcessInfo>();
foreach (var proc in processes)
{
var procInfo = new ProcessInfo()
{
PID = proc.Id,
Name = proc.ProcessName,
Arguments = GetProcessStartArguments(proc.Id),
StartTime = proc.StartTime
};
procInfos.Add(procInfo);
}
return procInfos.ToArray();
}));
}
/// <summary>
/// Print the current list of available .NET core processes for diagnosis, their statuses and the command line arguments that are passed to them.
/// </summary>
public static void ProcessStatus(IConsole console)
{
int GetColumnWidth(IEnumerable <int> fieldWidths)
{
int consoleWidth = 0;
if (Console.IsOutputRedirected)
{
consoleWidth = Int32.MaxValue;
}
else
{
consoleWidth = Console.WindowWidth;
}
int extra = (int)Math.Ceiling(consoleWidth * 0.05);
int largeLength = consoleWidth / 2 - 16 - extra;
return(Math.Min(fieldWidths.Max(), largeLength));
}
void FormatTableRows(List <ProcessDetails> rows, StringBuilder tableText)
{
if (rows.Count == 0)
{
tableText.Append("No supported .NET processes were found");
return;
}
var processIDs = rows.Select(i => i.ProcessId.ToString().Length);
var processNames = rows.Select(i => i.ProcessName.Length);
var fileNames = rows.Select(i => i.FileName.Length);
var commandLineArgs = rows.Select(i => i.CmdLineArgs.Length);
int iDLength = GetColumnWidth(processIDs);
int nameLength = GetColumnWidth(processNames);
int fileLength = GetColumnWidth(fileNames);
int cmdLength = GetColumnWidth(commandLineArgs);
foreach (var info in rows)
{
MakeFixedWidth(info.ProcessId.ToString(), iDLength, tableText, true, true);
MakeFixedWidth(info.ProcessName, nameLength, tableText, false, true);
MakeFixedWidth(info.FileName, fileLength, tableText, false, true);
MakeFixedWidth(info.CmdLineArgs, cmdLength, tableText, false, true);
tableText.Append("\n");
}
}
try
{
StringBuilder sb = new StringBuilder();
var processes = DiagnosticsClient.GetPublishedProcesses()
.Select(GetProcessById)
.Where(process => process != null)
.OrderBy(process => process.ProcessName)
.ThenBy(process => process.Id);
var currentPid = Process.GetCurrentProcess().Id;
List <Microsoft.Internal.Common.Commands.ProcessStatusCommandHandler.ProcessDetails> printInfo = new ();
foreach (var process in processes)
{
if (process.Id == currentPid)
{
continue;
}
try
{
String cmdLineArgs = GetArgs(process);
cmdLineArgs = cmdLineArgs == process.MainModule?.FileName ? string.Empty : cmdLineArgs;
string fileName = process.MainModule?.FileName ?? string.Empty;
var commandInfo = new ProcessDetails()
{
ProcessId = process.Id,
ProcessName = process.ProcessName,
FileName = fileName,
CmdLineArgs = cmdLineArgs
};
printInfo.Add(commandInfo);
}
catch (Exception ex)
{
if (ex is Win32Exception || ex is InvalidOperationException)
{
var commandInfo = new ProcessDetails()
{
ProcessId = process.Id,
ProcessName = process.ProcessName,
FileName = "[Elevated process - cannot determine path]",
CmdLineArgs = ""
};
printInfo.Add(commandInfo);
}
else
{
Debug.WriteLine($"[PrintProcessStatus] {ex.ToString()}");
}
}
}
FormatTableRows(printInfo, sb);
console.Out.WriteLine(sb.ToString());
}
catch (InvalidOperationException ex)
{
console.Out.WriteLine(ex.ToString());
}
}
public IEnumerable <int> GetProcesses()
{
//TODO This won't work properly with multi-container scenarios that don't share the process space.
//TODO We will need to use DiagnosticsAgent if we are the server.
return(DiagnosticsClient.GetPublishedProcesses());
}
/// <summary>
/// Application to take a process dump if CPU utilization exceeds specified percentage
/// </summary>
/// <param name="processId">Process Id of the the tracee process</param>
/// <param name="cpu">The CPU utilization percentage on which to trigger a dump. The default value is 10%</param>
static async Task Main(int?processId, int cpu = 10)
{
if (processId ! == null)
{
throw new ArgumentNullException(nameof(processId));
}
if (!DiagnosticsClient.GetPublishedProcesses().Contains(processId.Value))
{
throw new ArgumentException($"{nameof(processId)} is not a valid .NET process");
}
var providerList = new List <EventPipeProvider>()
{
new EventPipeProvider(name: "System.Runtime",
keywords: long.MaxValue,
eventLevel: EventLevel.Informational,
arguments: new Dictionary <string, string>()
{
{ "EventCounterIntervalSec", "1" }
}),
new EventPipeProvider(name: "Microsoft-Windows-DotNETRuntime",
keywords: (long)ClrTraceEventParser.Keywords.GC,
eventLevel: EventLevel.Verbose)
};
var diagnosticsClient = new DiagnosticsClient(processId.Value);
var session = diagnosticsClient.StartEventPipeSession(
providers: providerList,
requestRundown: false);
var source = new EventPipeEventSource(session.EventStream);
var tcs = new TaskCompletionSource <bool>(TaskCreationOptions.RunContinuationsAsynchronously);
Console.CancelKeyPress += delegate(object sender, ConsoleCancelEventArgs e)
{
tcs.TrySetCanceled();
};
source.Dynamic.AddCallbackForProviderEvent("System.Runtime", "EventCounters", (traceEvent) =>
{
var counter = traceEvent.GetCounter();
if (counter.GetName() == "cpu-usage")
{
Console.WriteLine($"{counter.GetName()}\t{counter.GetValue()}");
if (Int32.Parse(counter.GetValue()) >= cpu)
{
source.StopProcessing();
tcs.SetResult(true);
}
}
});
_ = Task.Run(() => source.Process());
try
{
_ = await tcs.Task;
}
catch (Exception e) when(e is TaskCanceledException)
{
Console.WriteLine("Cancelled due to Ctrl+C");
}
finally
{
session.Dispose();
source.Dispose();
}
if (tcs.Task.IsCompletedSuccessfully)
{
new Dumper().Collect(processId.Value, false, DumpTypeOption.Mini);
}
}
public async Task ProcessEvents(int processId, CancellationToken cancellationToken)
{
var hasEventPipe = false;
for (int i = 0; i < 10; ++i)
{
if (DiagnosticsClient.GetPublishedProcesses().Contains(processId))
{
hasEventPipe = true;
break;
}
cancellationToken.ThrowIfCancellationRequested();
await Task.Delay(500);
}
if (!hasEventPipe)
{
_logger.LogInformation("Process id {PID}, does not support event pipe", processId);
return;
}
_logger.LogInformation("Listening for event pipe events for {ServiceName} on process id {PID}", _dimValues[1], processId);
while (!cancellationToken.IsCancellationRequested)
{
EventPipeSession session = null;
var client = new DiagnosticsClient(processId);
try
{
session = client.StartEventPipeSession(_sourceConfig.GetProviders());
}
catch (EndOfStreamException)
{
break;
}
catch (Exception ex)
{
if (!cancellationToken.IsCancellationRequested)
{
_logger.LogDebug(0, ex, "Failed to start the event pipe session");
}
// We can't even start the session, wait until the process boots up again to start another metrics thread
break;
}
void StopSession()
{
try
{
session.Stop();
}
catch (EndOfStreamException)
{
// If the app we're monitoring exits abruptly, this may throw in which case we just swallow the exception and exit gracefully.
}
// We may time out if the process ended before we sent StopTracing command. We can just exit in that case.
catch (TimeoutException)
{
}
// On Unix platforms, we may actually get a PNSE since the pipe is gone with the process, and Runtime Client Library
// does not know how to distinguish a situation where there is no pipe to begin with, or where the process has exited
// before dotnet-counters and got rid of a pipe that once existed.
// Since we are catching this in StopMonitor() we know that the pipe once existed (otherwise the exception would've
// been thrown in StartMonitor directly)
catch (PlatformNotSupportedException)
{
}
}
using var _ = cancellationToken.Register(() => StopSession());
try
{
var source = new EventPipeEventSource(session.EventStream);
// Metrics
HandleEventCounters(source);
// Logging
HandleLoggingEvents(source);
source.Process();
}
catch (DiagnosticsClientException ex)
{
_logger.LogDebug(0, ex, "Failed to start the event pipe session");
}
catch (Exception)
{
// This fails if stop is called or if the process dies
}
finally
{
session?.Dispose();
}
}
_logger.LogInformation("Event pipe collection completed for {ServiceName} on process id {PID}", _dimValues[1], processId);
}
public void ProcessEvents(string applicationName,
string serviceName,
int processId,
string replicaName,
ReplicaStatus replica,
CancellationToken cancellationToken)
{
var hasEventPipe = false;
for (int i = 0; i < 10; ++i)
{
if (DiagnosticsClient.GetPublishedProcesses().Contains(processId))
{
hasEventPipe = true;
break;
}
if (cancellationToken.IsCancellationRequested)
{
return;
}
Thread.Sleep(500);
}
if (!hasEventPipe)
{
_logger.LogInformation("Process id {PID}, does not support event pipe", processId);
return;
}
_logger.LogInformation("Listening for event pipe events for {ServiceName} on process id {PID}", replicaName, processId);
// Create the logger factory for this replica
using var loggerFactory = LoggerFactory.Create(builder => ConfigureLogging(serviceName, replicaName, builder));
var processor = new SimpleSpanProcessor(CreateSpanExporter(serviceName, replicaName));
var providers = new List <EventPipeProvider>()
{
// Runtime Metrics
new EventPipeProvider(
SystemRuntimeEventSourceName,
EventLevel.Informational,
(long)ClrTraceEventParser.Keywords.None,
new Dictionary <string, string>()
{
{ "EventCounterIntervalSec", "1" }
}
),
new EventPipeProvider(
MicrosoftAspNetCoreHostingEventSourceName,
EventLevel.Informational,
(long)ClrTraceEventParser.Keywords.None,
new Dictionary <string, string>()
{
{ "EventCounterIntervalSec", "1" }
}
),
new EventPipeProvider(
GrpcAspNetCoreServer,
EventLevel.Informational,
(long)ClrTraceEventParser.Keywords.None,
new Dictionary <string, string>()
{
{ "EventCounterIntervalSec", "1" }
}
),
// Application Metrics
new EventPipeProvider(
applicationName,
EventLevel.Informational,
(long)ClrTraceEventParser.Keywords.None,
new Dictionary <string, string>()
{
{ "EventCounterIntervalSec", "1" }
}
),
// Logging
new EventPipeProvider(
MicrosoftExtensionsLoggingProviderName,
EventLevel.LogAlways,
(long)(LoggingEventSource.Keywords.JsonMessage | LoggingEventSource.Keywords.FormattedMessage)
),
// Distributed Tracing
// Activity correlation
new EventPipeProvider(TplEventSource,
keywords: 0x80,
eventLevel: EventLevel.LogAlways),
// Diagnostic source events
new EventPipeProvider(DiagnosticSourceEventSource,
keywords: 0x1 | 0x2,
eventLevel: EventLevel.Verbose,
arguments: new Dictionary <string, string>
{
{ "FilterAndPayloadSpecs", DiagnosticFilterString }
})
};
while (!cancellationToken.IsCancellationRequested)
{
EventPipeSession session = null;
var client = new DiagnosticsClient(processId);
try
{
session = client.StartEventPipeSession(providers);
}
catch (EndOfStreamException)
{
break;
}
catch (Exception ex)
{
if (!cancellationToken.IsCancellationRequested)
{
_logger.LogDebug(0, ex, "Failed to start the event pipe session");
}
// We can't even start the session, wait until the process boots up again to start another metrics thread
break;
}
void StopSession()
{
try
{
session.Stop();
}
catch (EndOfStreamException)
{
// If the app we're monitoring exits abruptly, this may throw in which case we just swallow the exception and exit gracefully.
}
// We may time out if the process ended before we sent StopTracing command. We can just exit in that case.
catch (TimeoutException)
{
}
// On Unix platforms, we may actually get a PNSE since the pipe is gone with the process, and Runtime Client Library
// does not know how to distinguish a situation where there is no pipe to begin with, or where the process has exited
// before dotnet-counters and got rid of a pipe that once existed.
// Since we are catching this in StopMonitor() we know that the pipe once existed (otherwise the exception would've
// been thrown in StartMonitor directly)
catch (PlatformNotSupportedException)
{
}
}
using var _ = cancellationToken.Register(() => StopSession());
try
{
var source = new EventPipeEventSource(session.EventStream);
var activities = new Dictionary <string, ActivityItem>();
source.Dynamic.All += traceEvent =>
{
try
{
// Uncomment to debug the diagnostics source event source
//if (traceEvent.EventName == "Message")
//{
// _logger.LogTrace("[" + replicaName + "]:" + traceEvent.PayloadValue(0));
//}
//// Distributed tracing
// else
if (traceEvent.EventName == "Activity1Start/Start")
{
var listenerEventName = (string)traceEvent.PayloadByName("EventName");
if (traceEvent.PayloadByName("Arguments") is IDictionary <string, object>[] arguments)
{
string activityId = null;
string parentId = null;
string operationName = null;
string httpMethod = null;
string path = null;
string spanId = null;
string parentSpanId = null;
string traceId = null;
DateTime startTime = default;
ActivityIdFormat idFormat = default;
foreach (var arg in arguments)
{
var key = (string)arg["Key"];
var value = (string)arg["Value"];
if (key == "ActivityId")
{
activityId = value;
}
else if (key == "ActivityParentId")
{
parentId = value;
}
else if (key == "ActivityOperationName")
{
operationName = value;
}
else if (key == "ActivitySpanId")
{
spanId = value;
}
else if (key == "ActivityTraceId")
{
traceId = value;
}
else if (key == "ActivityParentSpanId")
{
parentSpanId = value;
}
else if (key == "Method")
{
httpMethod = value;
}
else if (key == "Path")
{
path = value;
}
else if (key == "ActivityStartTime")
{
startTime = new DateTime(long.Parse(value), DateTimeKind.Utc);
}
else if (key == "ActivityIdFormat")
{
idFormat = Enum.Parse <ActivityIdFormat>(value);
}
}
if (string.IsNullOrEmpty(activityId))
{
// Not a 3.1 application (we can detect this earlier)
return;
}
if (idFormat == ActivityIdFormat.Hierarchical)
{
// We need W3C to make it work
return;
}
// This is what open telemetry currently does
// https://github.com/open-telemetry/opentelemetry-dotnet/blob/4ba732af062ddc2759c02aebbc91335aaa3f7173/src/OpenTelemetry.Collector.AspNetCore/Implementation/HttpInListener.cs#L65-L92
var item = new ActivityItem()
{
Name = path,
SpanId = ActivitySpanId.CreateFromString(spanId),
TraceId = ActivityTraceId.CreateFromString(traceId),
ParentSpanId = parentSpanId == "0000000000000000" ? default : ActivitySpanId.CreateFromString(parentSpanId),
StartTime = startTime,
};
item.Attributes[SpanAttributeConstants.HttpMethodKey] = httpMethod;
item.Attributes[SpanAttributeConstants.HttpPathKey] = path;
activities[activityId] = item;
}
}
else if (traceEvent.EventName == "Activity1Stop/Stop")
{
var listenerEventName = (string)traceEvent.PayloadByName("EventName");
if (traceEvent.PayloadByName("Arguments") is IDictionary <string, object>[] arguments)
{
string activityId = null;
TimeSpan duration = default;
int statusCode = 0;
foreach (var arg in arguments)
{
var key = (string)arg["Key"];
var value = (string)arg["Value"];
if (key == "ActivityId")
{
activityId = value;
}
else if (key == "StatusCode")
{
statusCode = int.Parse(value);
}
else if (key == "ActivityDuration")
{
duration = new TimeSpan(long.Parse(value));
}
}
if (string.IsNullOrEmpty(activityId))
{
// Not a 3.1 application (we can detect this earlier)
return;
}
if (activities.TryGetValue(activityId, out var item))
{
item.Attributes[SpanAttributeConstants.HttpStatusCodeKey] = statusCode;
item.EndTime = item.StartTime + duration;
var spanData = new SpanData(item.Name,
new SpanContext(item.TraceId, item.SpanId, ActivityTraceFlags.Recorded),
item.ParentSpanId,
SpanKind.Server,
item.StartTime,
item.Attributes,
Enumerable.Empty <Event>(),
Enumerable.Empty <Link>(),
null,
Status.Ok,
item.EndTime);
processor.OnEnd(spanData);
activities.Remove(activityId);
}
}
}
else if (traceEvent.EventName == "Activity2Start/Start")
{
var listenerEventName = (string)traceEvent.PayloadByName("EventName");
_logger.LogDebug("[" + replicaName + "]: " + listenerEventName + " fired");
}
else if (traceEvent.EventName == "Activity2Stop/Stop")
{
var listenerEventName = (string)traceEvent.PayloadByName("EventName");
_logger.LogDebug("[" + replicaName + "]: " + listenerEventName + " fired");
}
// Metrics
else if (traceEvent.EventName.Equals("EventCounters"))
{
var payloadVal = (IDictionary <string, object>)traceEvent.PayloadValue(0);
var eventPayload = (IDictionary <string, object>)payloadVal["Payload"];
ICounterPayload payload = CounterPayload.FromPayload(eventPayload);
replica.Metrics[traceEvent.ProviderName + "/" + payload.Name] = payload.Value;
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Error processing counter for {ProviderName}:{EventName}", traceEvent.ProviderName, traceEvent.EventName);
}
};
// Logging
string lastFormattedMessage = "";
var logActivities = new Dictionary <Guid, LogActivityItem>();
var stack = new Stack <Guid>();
source.Dynamic.AddCallbackForProviderEvent(MicrosoftExtensionsLoggingProviderName, "ActivityJsonStart/Start", (traceEvent) =>
{
var factoryId = (int)traceEvent.PayloadByName("FactoryID");
var categoryName = (string)traceEvent.PayloadByName("LoggerName");
var argsJson = (string)traceEvent.PayloadByName("ArgumentsJson");
// TODO: Store this information by logger factory id
var item = new LogActivityItem
{
ActivityID = traceEvent.ActivityID,
ScopedObject = new LogObject(JsonDocument.Parse(argsJson).RootElement),
};
if (stack.TryPeek(out var parentId) && logActivities.TryGetValue(parentId, out var parentItem))
{
item.Parent = parentItem;
}
stack.Push(traceEvent.ActivityID);
logActivities[traceEvent.ActivityID] = item;
});
source.Dynamic.AddCallbackForProviderEvent(MicrosoftExtensionsLoggingProviderName, "ActivityJsonStop/Stop", (traceEvent) =>
{
var factoryId = (int)traceEvent.PayloadByName("FactoryID");
var categoryName = (string)traceEvent.PayloadByName("LoggerName");
stack.Pop();
logActivities.Remove(traceEvent.ActivityID);
});
source.Dynamic.AddCallbackForProviderEvent(MicrosoftExtensionsLoggingProviderName, "MessageJson", (traceEvent) =>
{
// Level, FactoryID, LoggerName, EventID, EventName, ExceptionJson, ArgumentsJson
var logLevel = (LogLevel)traceEvent.PayloadByName("Level");
var factoryId = (int)traceEvent.PayloadByName("FactoryID");
var categoryName = (string)traceEvent.PayloadByName("LoggerName");
var eventId = (int)traceEvent.PayloadByName("EventId");
var eventName = (string)traceEvent.PayloadByName("EventName");
var exceptionJson = (string)traceEvent.PayloadByName("ExceptionJson");
var argsJson = (string)traceEvent.PayloadByName("ArgumentsJson");
// There's a bug that causes some of the columns to get mixed up
if (eventName.StartsWith("{"))
{
argsJson = exceptionJson;
exceptionJson = eventName;
eventName = null;
}
if (string.IsNullOrEmpty(argsJson))
{
return;
}
Exception exception = null;
var logger = loggerFactory.CreateLogger(categoryName);
var scopes = new List <IDisposable>();
if (logActivities.TryGetValue(traceEvent.ActivityID, out var logActivityItem))
{
// REVIEW: Does order matter here? We're combining everything anyways.
while (logActivityItem != null)
{
scopes.Add(logger.BeginScope(logActivityItem.ScopedObject));
logActivityItem = logActivityItem.Parent;
}
}
try
{
if (exceptionJson != "{}")
{
var exceptionMessage = JsonSerializer.Deserialize <JsonElement>(exceptionJson);
exception = new LoggerException(exceptionMessage);
}
var message = JsonSerializer.Deserialize <JsonElement>(argsJson);
if (message.TryGetProperty("{OriginalFormat}", out var formatElement))
{
var formatString = formatElement.GetString();
var formatter = new LogValuesFormatter(formatString);
object[] args = new object[formatter.ValueNames.Count];
for (int i = 0; i < args.Length; i++)
{
args[i] = message.GetProperty(formatter.ValueNames[i]).GetString();
}
logger.Log(logLevel, new EventId(eventId, eventName), exception, formatString, args);
}
else
{
var obj = new LogObject(message, lastFormattedMessage);
logger.Log(logLevel, new EventId(eventId, eventName), obj, exception, LogObject.Callback);
}
}
catch (Exception ex)
{
_logger.LogDebug(ex, "Error processing log entry for {ServiceName}", replicaName);
}
finally
{
scopes.ForEach(d => d.Dispose());
}
});
source.Dynamic.AddCallbackForProviderEvent(MicrosoftExtensionsLoggingProviderName, "FormattedMessage", (traceEvent) =>
{
// Level, FactoryID, LoggerName, EventID, EventName, FormattedMessage
var logLevel = (LogLevel)traceEvent.PayloadByName("Level");
var factoryId = (int)traceEvent.PayloadByName("FactoryID");
var categoryName = (string)traceEvent.PayloadByName("LoggerName");
var eventId = (int)traceEvent.PayloadByName("EventId");
var eventName = (string)traceEvent.PayloadByName("EventName");
var formattedMessage = (string)traceEvent.PayloadByName("FormattedMessage");
if (string.IsNullOrEmpty(formattedMessage))
{
formattedMessage = eventName;
eventName = "";
}
lastFormattedMessage = formattedMessage;
});
source.Process();
}
public void ProcessEvents(
string applicationName,
string serviceName,
int processId,
string replicaName,
IDictionary <string, string> metrics,
CancellationToken cancellationToken)
{
var hasEventPipe = false;
for (var i = 0; i < 10; ++i)
{
if (DiagnosticsClient.GetPublishedProcesses().Contains(processId))
{
hasEventPipe = true;
break;
}
if (cancellationToken.IsCancellationRequested)
{
return;
}
Thread.Sleep(500);
}
if (!hasEventPipe)
{
_logger.LogInformation("Process id {PID}, does not support event pipe", processId);
return;
}
_logger.LogInformation("Listening for event pipe events for {ServiceName} on process id {PID}", replicaName, processId);
// Create the logger factory for this replica
using var loggerFactory = LoggerFactory.Create(builder => ConfigureLogging(serviceName, replicaName, builder));
var processor = new SimpleSpanProcessor(CreateSpanExporter(serviceName, replicaName));
var providers = new List <EventPipeProvider>()
{
// Runtime Metrics
new EventPipeProvider(
SystemRuntimeEventSourceName,
EventLevel.Informational,
(long)ClrTraceEventParser.Keywords.None,
new Dictionary <string, string>()
{
{ "EventCounterIntervalSec", "1" }
}
),
new EventPipeProvider(
MicrosoftAspNetCoreHostingEventSourceName,
EventLevel.Informational,
(long)ClrTraceEventParser.Keywords.None,
new Dictionary <string, string>()
{
{ "EventCounterIntervalSec", "1" }
}
),
new EventPipeProvider(
GrpcAspNetCoreServer,
EventLevel.Informational,
(long)ClrTraceEventParser.Keywords.None,
new Dictionary <string, string>()
{
{ "EventCounterIntervalSec", "1" }
}
),
// Application Metrics
new EventPipeProvider(
applicationName,
EventLevel.Informational,
(long)ClrTraceEventParser.Keywords.None,
new Dictionary <string, string>()
{
{ "EventCounterIntervalSec", "1" }
}
),
// Logging
new EventPipeProvider(
MicrosoftExtensionsLoggingProviderName,
EventLevel.LogAlways,
(long)(LoggingEventSource.Keywords.JsonMessage | LoggingEventSource.Keywords.FormattedMessage)
),
// Distributed Tracing
// Activity correlation
new EventPipeProvider(TplEventSource,
keywords: 0x80,
eventLevel: EventLevel.LogAlways),
// Diagnostic source events
new EventPipeProvider(DiagnosticSourceEventSource,
keywords: 0x1 | 0x2,
eventLevel: EventLevel.Verbose,
arguments: new Dictionary <string, string>
{
{ "FilterAndPayloadSpecs", DiagnosticFilterString }
})
};
while (!cancellationToken.IsCancellationRequested)
{
EventPipeSession session = null;
var client = new DiagnosticsClient(processId);
try
{
session = client.StartEventPipeSession(providers);
}
catch (EndOfStreamException)
{
break;
}
// If the process has already exited, a ServerNotAvailableException will be thrown.
catch (ServerNotAvailableException)
{
break;
}
catch (Exception ex)
{
if (!cancellationToken.IsCancellationRequested)
{
_logger.LogDebug(0, ex, "Failed to start the event pipe session");
}
// We can't even start the session, wait until the process boots up again to start another metrics thread
break;
}
void StopSession()
{
try
{
session.Stop();
}
catch (EndOfStreamException)
{
// If the app we're monitoring exits abruptly, this may throw in which case we just swallow the exception and exit gracefully.
}
// We may time out if the process ended before we sent StopTracing command. We can just exit in that case.
catch (TimeoutException)
{
}
// On Unix platforms, we may actually get a PNSE since the pipe is gone with the process, and Runtime Client Library
// does not know how to distinguish a situation where there is no pipe to begin with, or where the process has exited
// before dotnet-counters and got rid of a pipe that once existed.
// Since we are catching this in StopMonitor() we know that the pipe once existed (otherwise the exception would've
// been thrown in StartMonitor directly)
catch (PlatformNotSupportedException)
{
}
// If the process has already exited, a ServerNotAvailableException will be thrown.
// This can always race with tye shutting down and a process being restarted on exiting.
catch (ServerNotAvailableException)
{
}
}
using var _ = cancellationToken.Register(() => StopSession());
try
{
var source = new EventPipeEventSource(session.EventStream);
// Distribued Tracing
HandleDistributedTracingEvents(source, processor);
// Metrics
HandleEventCounters(source, metrics);
// Logging
HandleLoggingEvents(source, loggerFactory, replicaName);
source.Process();
}
catch (DiagnosticsClientException ex)
{
_logger.LogDebug(0, ex, "Failed to start the event pipe session");
}
catch (Exception)
{
// This fails if stop is called or if the process dies
}
finally
{
session?.Dispose();
}
}
_logger.LogInformation("Event pipe collection completed for {ServiceName} on process id {PID}", replicaName, processId);
}
