public Task LogsWildcardTest(DiagnosticPortConnectionMode mode, LogFormat logFormat)
 {
     return(ValidateLogsAsync(
                mode,
                new LogsConfiguration()
     {
         FilterSpecs = new Dictionary <string, LogLevel?>()
         {
             { "*", LogLevel.Trace },
             { TestAppScenarios.Logger.Categories.LoggerCategory2, LogLevel.Warning }
         },
         LogLevel = LogLevel.Information,
         UseAppFilters = false
     },
                async reader =>
     {
         LogsTestUtilities.ValidateEntry(LogsTestUtilities.Category1TraceEntry, await reader.ReadAsync());
         LogsTestUtilities.ValidateEntry(LogsTestUtilities.Category1DebugEntry, await reader.ReadAsync());
         LogsTestUtilities.ValidateEntry(LogsTestUtilities.Category1InformationEntry, await reader.ReadAsync());
         LogsTestUtilities.ValidateEntry(LogsTestUtilities.Category1WarningEntry, await reader.ReadAsync());
         LogsTestUtilities.ValidateEntry(LogsTestUtilities.Category1ErrorEntry, await reader.ReadAsync());
         LogsTestUtilities.ValidateEntry(LogsTestUtilities.Category1CriticalEntry, await reader.ReadAsync());
         LogsTestUtilities.ValidateEntry(LogsTestUtilities.Category2WarningEntry, await reader.ReadAsync());
         LogsTestUtilities.ValidateEntry(LogsTestUtilities.Category2ErrorEntry, await reader.ReadAsync());
         LogsTestUtilities.ValidateEntry(LogsTestUtilities.Category2CriticalEntry, await reader.ReadAsync());
         LogsTestUtilities.ValidateEntry(LogsTestUtilities.Category3TraceEntry, await reader.ReadAsync());
         LogsTestUtilities.ValidateEntry(LogsTestUtilities.Category3DebugEntry, await reader.ReadAsync());
         LogsTestUtilities.ValidateEntry(LogsTestUtilities.Category3InformationEntry, await reader.ReadAsync());
         LogsTestUtilities.ValidateEntry(LogsTestUtilities.Category3WarningEntry, await reader.ReadAsync());
         LogsTestUtilities.ValidateEntry(LogsTestUtilities.Category3ErrorEntry, await reader.ReadAsync());
         LogsTestUtilities.ValidateEntry(LogsTestUtilities.Category3CriticalEntry, await reader.ReadAsync());
         Assert.False(await reader.WaitToReadAsync());
     },
                logFormat));
 }
 public Task LogsUseAppFiltersViaBodyTest(DiagnosticPortConnectionMode mode, LogFormat logFormat)
 {
     return(ValidateLogsAsync(
                mode,
                new LogsConfiguration()
     {
         LogLevel = LogLevel.Trace,
         UseAppFilters = true
     },
                async reader =>
     {
         LogsTestUtilities.ValidateEntry(LogsTestUtilities.Category1DebugEntry, await reader.ReadAsync());
         LogsTestUtilities.ValidateEntry(LogsTestUtilities.Category1InformationEntry, await reader.ReadAsync());
         LogsTestUtilities.ValidateEntry(LogsTestUtilities.Category1WarningEntry, await reader.ReadAsync());
         LogsTestUtilities.ValidateEntry(LogsTestUtilities.Category1ErrorEntry, await reader.ReadAsync());
         LogsTestUtilities.ValidateEntry(LogsTestUtilities.Category1CriticalEntry, await reader.ReadAsync());
         LogsTestUtilities.ValidateEntry(LogsTestUtilities.Category2InformationEntry, await reader.ReadAsync());
         LogsTestUtilities.ValidateEntry(LogsTestUtilities.Category2WarningEntry, await reader.ReadAsync());
         LogsTestUtilities.ValidateEntry(LogsTestUtilities.Category2ErrorEntry, await reader.ReadAsync());
         LogsTestUtilities.ValidateEntry(LogsTestUtilities.Category2CriticalEntry, await reader.ReadAsync());
         LogsTestUtilities.ValidateEntry(LogsTestUtilities.Category3WarningEntry, await reader.ReadAsync());
         LogsTestUtilities.ValidateEntry(LogsTestUtilities.Category3ErrorEntry, await reader.ReadAsync());
         LogsTestUtilities.ValidateEntry(LogsTestUtilities.Category3CriticalEntry, await reader.ReadAsync());
         Assert.False(await reader.WaitToReadAsync());
     },
                logFormat));
 }
        public Task LogsDefaultLevelNoneNotSupportedViaBodyTest(DiagnosticPortConnectionMode mode, LogFormat logFormat)
        {
            return(ScenarioRunner.SingleTarget(
                       _outputHelper,
                       _httpClientFactory,
                       mode,
                       TestAppScenarios.Logger.Name,
                       appValidate: async(runner, client) =>
            {
                ValidationProblemDetailsException exception = await Assert.ThrowsAsync <ValidationProblemDetailsException>(
                    async() =>
                {
                    using ResponseStreamHolder _ = await client.CaptureLogsAsync(
                              await runner.ProcessIdTask,
                              CommonTestTimeouts.LogsDuration,
                              new LogsConfiguration()
                    {
                        LogLevel = LogLevel.None
                    },
                              logFormat);
                });
                Assert.Equal(HttpStatusCode.BadRequest, exception.StatusCode);
                Assert.Equal(StatusCodes.Status400BadRequest, exception.Details.Status);

                // Allow test app to gracefully exit by continuing the scenario.
                await runner.SendCommandAsync(TestAppScenarios.Logger.Commands.StartLogging);
            }));
        }
        public async Task CollectionRule_ConfigurationChangeTest(DiagnosticPortConnectionMode mode)
        {
            const string firstRuleName  = "FirstRule";
            const string secondRuleName = "SecondRule";

            DiagnosticPortHelper.Generate(
                mode,
                out DiagnosticPortConnectionMode appConnectionMode,
                out string diagnosticPortPath);

            await using MonitorCollectRunner toolRunner = new(_outputHelper);
            toolRunner.ConnectionMode        = mode;
            toolRunner.DiagnosticPortPath    = diagnosticPortPath;
            toolRunner.DisableAuthentication = true;

            // Create a rule with some settings
            RootOptions originalOptions = new();

            originalOptions.CreateCollectionRule(firstRuleName)
            .SetStartupTrigger();

            await toolRunner.WriteUserSettingsAsync(originalOptions);

            await toolRunner.StartAsync();

            AppRunner appRunner = new(_outputHelper, Assembly.GetExecutingAssembly());

            appRunner.ConnectionMode     = appConnectionMode;
            appRunner.DiagnosticPortPath = diagnosticPortPath;
            appRunner.ScenarioName       = TestAppScenarios.AsyncWait.Name;

            Task originalActionsCompletedTask = toolRunner.WaitForCollectionRuleActionsCompletedAsync(firstRuleName);

            await appRunner.ExecuteAsync(async() =>
            {
                // Validate that the first rule is observed and its actions are run.
                await originalActionsCompletedTask;

                // Set up new observers for the first and second rule.
                originalActionsCompletedTask = toolRunner.WaitForCollectionRuleActionsCompletedAsync(firstRuleName);
                Task newActionsCompletedTask = toolRunner.WaitForCollectionRuleActionsCompletedAsync(secondRuleName);

                // Change collection rule configuration to only contain the second rule.
                RootOptions newOptions = new();
                newOptions.CreateCollectionRule(secondRuleName)
                .SetStartupTrigger();

                await toolRunner.WriteUserSettingsAsync(newOptions);

                // Validate that only the second rule is observed.
                await newActionsCompletedTask;
                Assert.False(originalActionsCompletedTask.IsCompleted);

                await appRunner.SendCommandAsync(TestAppScenarios.AsyncWait.Commands.Continue);
            });

            Assert.Equal(0, appRunner.ExitCode);
        }
Exemple #5
0
        private static void ConfigureEndpointInfoSource(IConfigurationBuilder builder, string diagnosticPort)
        {
            DiagnosticPortConnectionMode connectionMode = GetConnectionMode(diagnosticPort);

            builder.AddInMemoryCollection(new Dictionary <string, string>
            {
                { ConfigurationPath.Combine(ConfigurationKeys.DiagnosticPort, nameof(DiagnosticPortOptions.ConnectionMode)), connectionMode.ToString() },
                { ConfigurationPath.Combine(ConfigurationKeys.DiagnosticPort, nameof(DiagnosticPortOptions.EndpointName)), diagnosticPort }
            });
        }
        private static void ConfigureEndpointInfoSource(IConfigurationBuilder builder, string diagnosticPort)
        {
            DiagnosticPortConnectionMode connectionMode = string.IsNullOrEmpty(diagnosticPort) ? DiagnosticPortConnectionMode.Connect : DiagnosticPortConnectionMode.Listen;

            builder.AddInMemoryCollection(new Dictionary <string, string>
            {
                { ConfigurationHelper.MakeKey(ConfigurationKeys.DiagnosticPort, nameof(DiagnosticPortOptions.ConnectionMode)), connectionMode.ToString() },
                { ConfigurationHelper.MakeKey(ConfigurationKeys.DiagnosticPort, nameof(DiagnosticPortOptions.EndpointName)), diagnosticPort }
            });
        }
Exemple #7
0
        public Task DumpTest(DiagnosticPortConnectionMode mode, DumpType type)
        {
#if !NET6_0_OR_GREATER
            // Capturing non-full dumps via diagnostic command works inconsistently
            // on Alpine for .NET 5 and lower (the dump command will return successfully, but)
            // the dump file will not exist). Only test other dump types on .NET 6+
            if (DistroInformation.IsAlpineLinux && type != DumpType.Full)
            {
                _outputHelper.WriteLine("Skipped on Alpine for .NET 5 and lower.");
                return(Task.CompletedTask);
            }
#endif

            return(ScenarioRunner.SingleTarget(
                       _outputHelper,
                       _httpClientFactory,
                       mode,
                       TestAppScenarios.AsyncWait.Name,
                       appValidate: async(runner, client) =>
            {
                int processId = await runner.ProcessIdTask;

                using ResponseStreamHolder holder = await client.CaptureDumpAsync(processId, type);
                Assert.NotNull(holder);

                // The dump operation may still be in progress but the process should still be discoverable.
                // If this check fails, then the dump operation is causing dotnet-monitor to not be able
                // to observe the process any more.
                ProcessInfo processInfo = await client.GetProcessAsync(processId);
                Assert.NotNull(processInfo);

                await DumpTestUtilities.ValidateDump(runner.Environment.ContainsKey(DumpTestUtilities.EnableElfDumpOnMacOS), holder.Stream);

                await runner.SendCommandAsync(TestAppScenarios.AsyncWait.Commands.Continue);
            },
                       configureApp: runner =>
            {
                // MachO not supported on .NET 5, only ELF: https://github.com/dotnet/runtime/blob/main/docs/design/coreclr/botr/xplat-minidump-generation.md#os-x
                if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX) && DotNetHost.RuntimeVersion.Major == 5)
                {
                    runner.Environment.Add(DumpTestUtilities.EnableElfDumpOnMacOS, "1");
                }
            },
                       configureTool: runner =>
            {
                string dumpTempFolder = Path.Combine(runner.TempPath, "Dumps");

                // The dump temp folder should not exist in order to test that capturing dumps into the folder
                // will work since dotnet-monitor should ensure the folder is created before issuing the dump command.
                Assert.False(Directory.Exists(dumpTempFolder), "The dump temp folder should not exist.");

                runner.ConfigurationFromEnvironment.SetDumpTempFolder(dumpTempFolder);
            }));
        }
        public async Task CollectionRule_StoppedOnExitTest(DiagnosticPortConnectionMode mode)
        {
            DiagnosticPortHelper.Generate(
                mode,
                out DiagnosticPortConnectionMode appConnectionMode,
                out string diagnosticPortPath);

            await using MonitorCollectRunner toolRunner = new(_outputHelper);
            toolRunner.ConnectionMode        = mode;
            toolRunner.DiagnosticPortPath    = diagnosticPortPath;
            toolRunner.DisableAuthentication = true;

            // Create a rule with some settings
            RootOptions originalOptions = new();

            originalOptions.CreateCollectionRule(DefaultRuleName)
            .SetEventCounterTrigger(options =>
            {
                options.ProviderName          = "System.Runtime";
                options.CounterName           = "cpu-usage";
                options.GreaterThan           = 1000; // Intentionally unobtainable
                options.SlidingWindowDuration = TimeSpan.FromSeconds(1);
            });

            await toolRunner.WriteUserSettingsAsync(originalOptions);

            await toolRunner.StartAsync();

            AppRunner appRunner = new(_outputHelper, Assembly.GetExecutingAssembly());

            appRunner.ConnectionMode     = appConnectionMode;
            appRunner.DiagnosticPortPath = diagnosticPortPath;
            appRunner.ScenarioName       = TestAppScenarios.AsyncWait.Name;

            Task ruleStartedTask  = toolRunner.WaitForCollectionRuleStartedAsync(DefaultRuleName);
            Task rulesStoppedTask = toolRunner.WaitForCollectionRulesStoppedAsync();

            await appRunner.ExecuteAsync(async() =>
            {
                await ruleStartedTask;

                await appRunner.SendCommandAsync(TestAppScenarios.AsyncWait.Commands.Continue);
            });

            Assert.Equal(0, appRunner.ExitCode);

            // All of the rules for the process should have stopped. Note that dotnet-monitor has
            // not yet exited at this point in time; this is verification that the rules have stopped
            // for the target process before dotnet-monitor shuts down.
            await rulesStoppedTask;
        }
        public FilteredEndpointInfoSource(
            ServerEndpointInfoSource serverEndpointInfoSource,
            IOptions <DiagnosticPortOptions> portOptions,
            ILogger <ClientEndpointInfoSource> clientSourceLogger)
        {
            _portOptions = portOptions.Value;

            DiagnosticPortConnectionMode connectionMode = _portOptions.GetConnectionMode();

            switch (connectionMode)
            {
            case DiagnosticPortConnectionMode.Connect:
                _source = new ClientEndpointInfoSource(clientSourceLogger);
                break;

            case DiagnosticPortConnectionMode.Listen:
                _source = serverEndpointInfoSource;
                break;

            default:
                throw new InvalidOperationException(string.Format(CultureInfo.CurrentCulture, Strings.ErrorMessage_UnhandledConnectionMode, connectionMode));
            }

            // Filter out the current process based on the connection mode.
            if (RuntimeInfo.IsDiagnosticsEnabled)
            {
                int pid = Process.GetCurrentProcess().Id;

                // Regardless of connection mode, can use the runtime instance cookie to filter self out.
                try
                {
                    var  client = new DiagnosticsClient(pid);
                    Guid runtimeInstanceCookie = client.GetProcessInfo().RuntimeInstanceCookie;
                    if (Guid.Empty != runtimeInstanceCookie)
                    {
                        _runtimeInstanceCookieToFilterOut = runtimeInstanceCookie;
                    }
                }
                catch (Exception)
                {
                }

                // If connecting to runtime instances, filter self out. In listening mode, it's likely
                // that multiple processes have the same PID in multi-container scenarios.
                if (DiagnosticPortConnectionMode.Connect == connectionMode)
                {
                    _processIdToFilterOut = pid;
                }
            }
        }
        public static async Task SingleTarget(
            ITestOutputHelper outputHelper,
            IHttpClientFactory httpClientFactory,
            DiagnosticPortConnectionMode mode,
            string scenarioName,
            Func <AppRunner, ApiClient, Task> appValidate,
            Func <ApiClient, int, Task> postAppValidate = null,
            Action <AppRunner> configureApp             = null,
            Action <MonitorCollectRunner> configureTool = null,
            bool disableHttpEgress = false)
        {
            DiagnosticPortHelper.Generate(
                mode,
                out DiagnosticPortConnectionMode appConnectionMode,
                out string diagnosticPortPath);

            await using MonitorCollectRunner toolRunner = new(outputHelper);
            toolRunner.ConnectionMode        = mode;
            toolRunner.DiagnosticPortPath    = diagnosticPortPath;
            toolRunner.DisableAuthentication = true;
            toolRunner.DisableHttpEgress     = disableHttpEgress;

            configureTool?.Invoke(toolRunner);

            await toolRunner.StartAsync();

            using HttpClient httpClient = await toolRunner.CreateHttpClientDefaultAddressAsync(httpClientFactory);

            ApiClient apiClient = new(outputHelper, httpClient);

            AppRunner appRunner = new(outputHelper, Assembly.GetExecutingAssembly());

            appRunner.ConnectionMode     = appConnectionMode;
            appRunner.DiagnosticPortPath = diagnosticPortPath;
            appRunner.ScenarioName       = scenarioName;

            configureApp?.Invoke(appRunner);

            await appRunner.ExecuteAsync(async() =>
            {
                await appValidate(appRunner, apiClient);
            });

            Assert.Equal(0, appRunner.ExitCode);

            if (null != postAppValidate)
            {
                await postAppValidate(apiClient, await appRunner.ProcessIdTask);
            }
        }
Exemple #11
0
        /// <summary>
        /// Calculates the app's diagnostic port mode and generates a port path
        /// if <paramref name="monitorConnectionMode"/> is <see cref="DiagnosticPortConnectionMode.Listen"/>.
        /// </summary>
        public static void Generate(
            DiagnosticPortConnectionMode monitorConnectionMode,
            out DiagnosticPortConnectionMode appConnectionMode,
            out string diagnosticPortPath)
        {
            appConnectionMode  = DiagnosticPortConnectionMode.Listen;
            diagnosticPortPath = null;

            if (DiagnosticPortConnectionMode.Listen == monitorConnectionMode)
            {
                appConnectionMode = DiagnosticPortConnectionMode.Connect;

                string fileName = Guid.NewGuid().ToString("D");
                diagnosticPortPath = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ?
                                     fileName : Path.Combine(Path.GetTempPath(), fileName);
            }
        }
Exemple #12
0
 public Task LogsDefaultLevelTest(DiagnosticPortConnectionMode mode, LogFormat logFormat)
 {
     return(ValidateLogsAsync(
                mode,
                LogLevel.Warning,
                async reader =>
     {
         LogsTestUtilities.ValidateEntry(LogsTestUtilities.Category1WarningEntry, await reader.ReadAsync());
         LogsTestUtilities.ValidateEntry(LogsTestUtilities.Category1ErrorEntry, await reader.ReadAsync());
         LogsTestUtilities.ValidateEntry(LogsTestUtilities.Category1CriticalEntry, await reader.ReadAsync());
         LogsTestUtilities.ValidateEntry(LogsTestUtilities.Category2WarningEntry, await reader.ReadAsync());
         LogsTestUtilities.ValidateEntry(LogsTestUtilities.Category2ErrorEntry, await reader.ReadAsync());
         LogsTestUtilities.ValidateEntry(LogsTestUtilities.Category2CriticalEntry, await reader.ReadAsync());
         LogsTestUtilities.ValidateEntry(LogsTestUtilities.Category3WarningEntry, await reader.ReadAsync());
         LogsTestUtilities.ValidateEntry(LogsTestUtilities.Category3ErrorEntry, await reader.ReadAsync());
         LogsTestUtilities.ValidateEntry(LogsTestUtilities.Category3CriticalEntry, await reader.ReadAsync());
         Assert.False(await reader.WaitToReadAsync());
     },
                logFormat));
 }
        public async Task CollectionRule_ActionLimitTest(DiagnosticPortConnectionMode mode)
        {
            using TemporaryDirectory tempDirectory = new(_outputHelper);
            string ExpectedFilePath    = Path.Combine(tempDirectory.FullName, "file.txt");
            string ExpectedFileContent = Guid.NewGuid().ToString("N");

            Task ruleCompletedTask = null;

            await ScenarioRunner.SingleTarget(
                _outputHelper,
                _httpClientFactory,
                mode,
                TestAppScenarios.SpinWait.Name,
                appValidate : async(runner, client) =>
            {
                await runner.SendCommandAsync(TestAppScenarios.SpinWait.Commands.StartSpin);

                await ruleCompletedTask;

                await runner.SendCommandAsync(TestAppScenarios.SpinWait.Commands.StopSpin);

                Assert.True(File.Exists(ExpectedFilePath));
                Assert.Equal(ExpectedFileContent, File.ReadAllText(ExpectedFilePath));
            },
                configureTool : runner =>
            {
                runner.ConfigurationFromEnvironment.CreateCollectionRule(DefaultRuleName)
                .SetEventCounterTrigger(options =>
                {
                    // cpu usage greater that 5% for 2 seconds
                    options.ProviderName          = "System.Runtime";
                    options.CounterName           = "cpu-usage";
                    options.GreaterThan           = 5;
                    options.SlidingWindowDuration = TimeSpan.FromSeconds(2);
                })
                .AddExecuteActionAppAction("TextFileOutput", ExpectedFilePath, ExpectedFileContent)
                .SetActionLimits(count: 1);

                ruleCompletedTask = runner.WaitForCollectionRuleCompleteAsync(DefaultRuleName);
            });
        }
        public ActionResult <Models.DotnetMonitorInfo> GetInfo()
        {
            return(this.InvokeService(() =>
            {
                string version = GetDotnetMonitorVersion();
                string runtimeVersion = Environment.Version.ToString();
                DiagnosticPortConnectionMode diagnosticPortMode = _diagnosticPortOptions.Value.GetConnectionMode();
                string diagnosticPortName = GetDiagnosticPortName();

                Models.DotnetMonitorInfo dotnetMonitorInfo = new Models.DotnetMonitorInfo()
                {
                    Version = version,
                    RuntimeVersion = runtimeVersion,
                    DiagnosticPortMode = diagnosticPortMode,
                    DiagnosticPortName = diagnosticPortName
                };

                _logger.WrittenToHttpStream();
                return new ActionResult <Models.DotnetMonitorInfo>(dotnetMonitorInfo);
            }, _logger));
        }
Exemple #15
0
        public Task SingleProcessIdentificationTest(DiagnosticPortConnectionMode mode)
        {
            string expectedEnvVarValue = Guid.NewGuid().ToString("D");

            return(ScenarioRunner.SingleTarget(
                       _outputHelper,
                       _httpClientFactory,
                       mode,
                       TestAppScenarios.AsyncWait.Name,
                       appValidate: async(runner, client) =>
            {
                int processId = await runner.ProcessIdTask;

                // GET /processes and filter to just the single process
                IEnumerable <ProcessIdentifier> identifiers = await client.GetProcessesWithRetryAsync(
                    _outputHelper,
                    new[] { processId });
                Assert.NotNull(identifiers);
                Assert.Single(identifiers);

                await VerifyProcessAsync(client, identifiers, processId, expectedEnvVarValue);

                await runner.SendCommandAsync(TestAppScenarios.AsyncWait.Commands.Continue);
            },
                       postAppValidate: async(client, processId) =>
            {
                // GET /processes and filter to just the single process
                IEnumerable <ProcessIdentifier> identifiers = await client.GetProcessesWithRetryAsync(
                    _outputHelper,
                    new[] { processId });

                // Verify app is no longer reported
                Assert.NotNull(identifiers);
                Assert.Empty(identifiers);
            },
                       configureApp: runner =>
            {
                runner.Environment[ExpectedEnvVarName] = expectedEnvVarValue;
            }));
        }
Exemple #16
0
 private Task ValidateLogsAsync(
     DiagnosticPortConnectionMode mode,
     LogsConfiguration configuration,
     Func <ChannelReader <LogEntry>, Task> callback,
     LogFormat logFormat)
 {
     return(ScenarioRunner.SingleTarget(
                _outputHelper,
                _httpClientFactory,
                mode,
                TestAppScenarios.Logger.Name,
                appValidate: async(runner, client) =>
                await ValidateResponseStream(
                    runner,
                    client.CaptureLogsAsync(
                        await runner.ProcessIdTask,
                        CommonTestTimeouts.LogsDuration,
                        configuration,
                        logFormat),
                    callback,
                    logFormat)));
 }
        public async Task CollectionRule_ProcessNameFilterNoMatchTest(DiagnosticPortConnectionMode mode)
        {
            Task filteredTask = null;

            await ScenarioRunner.SingleTarget(
                _outputHelper,
                _httpClientFactory,
                mode,
                TestAppScenarios.AsyncWait.Name,
                appValidate : async(runner, client) =>
            {
                await filteredTask;

                await runner.SendCommandAsync(TestAppScenarios.AsyncWait.Commands.Continue);
            },
                configureTool : runner =>
            {
                runner.ConfigurationFromEnvironment.CreateCollectionRule(DefaultRuleName)
                .SetStartupTrigger()
                .AddProcessNameFilter("UmatchedName");

                filteredTask = runner.WaitForCollectionRuleUnmatchedFiltersAsync(DefaultRuleName);
            });
        }
Exemple #18
0
        public async Task MultiProcessIdentificationTest(DiagnosticPortConnectionMode mode)
        {
            DiagnosticPortHelper.Generate(
                mode,
                out DiagnosticPortConnectionMode appConnectionMode,
                out string diagnosticPortPath);

            await using MonitorCollectRunner toolRunner = new(_outputHelper);
            toolRunner.ConnectionMode        = mode;
            toolRunner.DiagnosticPortPath    = diagnosticPortPath;
            toolRunner.DisableAuthentication = true;
            await toolRunner.StartAsync();

            using HttpClient httpClient = await toolRunner.CreateHttpClientDefaultAddressAsync(_httpClientFactory);

            ApiClient apiClient = new(_outputHelper, httpClient);

            const int appCount = 3;

            AppRunner[] appRunners = new AppRunner[appCount];

            for (int i = 0; i < appCount; i++)
            {
                AppRunner runner = new(_outputHelper, Assembly.GetExecutingAssembly(), appId : i + 1);
                runner.ConnectionMode     = appConnectionMode;
                runner.DiagnosticPortPath = diagnosticPortPath;
                runner.ScenarioName       = TestAppScenarios.AsyncWait.Name;
                runner.Environment[ExpectedEnvVarName] = Guid.NewGuid().ToString("D");
                appRunners[i] = runner;
            }

            IList <ProcessIdentifier> identifiers;
            await appRunners.ExecuteAsync(async() =>
            {
                // Scope to only the processes that were launched by the test
                IList <int> unmatchedPids = new List <int>();
                foreach (AppRunner runner in appRunners)
                {
                    unmatchedPids.Add(await runner.ProcessIdTask);
                }

                // Query for process identifiers
                identifiers = (await apiClient.GetProcessesWithRetryAsync(
                                   _outputHelper,
                                   unmatchedPids.ToArray())).ToList();
                Assert.NotNull(identifiers);

                _outputHelper.WriteLine("Start enumerating discovered processes.");
                foreach (ProcessIdentifier identifier in identifiers.ToList())
                {
                    _outputHelper.WriteLine($"- PID:  {identifier.Pid}");
                    _outputHelper.WriteLine($"  UID:  {identifier.Uid}");
                    _outputHelper.WriteLine($"  Name: {identifier.Name}");

                    unmatchedPids.Remove(identifier.Pid);
                }
                _outputHelper.WriteLine("End enumerating discovered processes");

                Assert.Empty(unmatchedPids);
                Assert.Equal(appRunners.Length, identifiers.Count);

                foreach (ProcessIdentifier processIdentifier in identifiers)
                {
                    int pid     = processIdentifier.Pid;
                    Guid uid    = processIdentifier.Uid;
                    string name = processIdentifier.Name;
#if NET5_0_OR_GREATER
                    // CHECK 1: Get response for processes using PID, UID, and Name and check for consistency

                    List <ProcessInfo> processInfoQueriesCheck1 = new List <ProcessInfo>();

                    processInfoQueriesCheck1.Add(await apiClient.GetProcessWithRetryAsync(_outputHelper, pid: pid));
                    // Only check with uid if it is non-empty; this can happen in connect mode if the ProcessInfo command fails
                    // to respond within the short period of time that is used to get the additional process information.
                    if (uid == Guid.Empty)
                    {
                        _outputHelper.WriteLine("Skipped uid-only check because it is empty GUID.");
                    }
                    else
                    {
                        processInfoQueriesCheck1.Add(await apiClient.GetProcessWithRetryAsync(_outputHelper, uid: uid));
                    }

                    VerifyProcessInfoEquality(processInfoQueriesCheck1);
#endif
                    // CHECK 2: Get response for requests using PID | PID and UID | PID, UID, and Name and check for consistency

                    List <ProcessInfo> processInfoQueriesCheck2 = new List <ProcessInfo>();

                    processInfoQueriesCheck2.Add(await apiClient.GetProcessWithRetryAsync(_outputHelper, pid: pid));
                    processInfoQueriesCheck2.Add(await apiClient.GetProcessWithRetryAsync(_outputHelper, pid: pid, uid: uid));
                    processInfoQueriesCheck2.Add(await apiClient.GetProcessWithRetryAsync(_outputHelper, pid: pid, uid: uid, name: name));

                    VerifyProcessInfoEquality(processInfoQueriesCheck2);

                    // CHECK 3: Get response for processes using PID and an unassociated (randomly generated) UID and ensure the proper exception is thrown

                    await VerifyInvalidRequestException(apiClient, pid, Guid.NewGuid(), null);
                }

                // CHECK 4: Get response for processes using invalid PID, UID, or Name and ensure the proper exception is thrown

                await VerifyInvalidRequestException(apiClient, -1, null, null);
                await VerifyInvalidRequestException(apiClient, null, Guid.NewGuid(), null);
                await VerifyInvalidRequestException(apiClient, null, null, "");

                // Verify each app instance is reported and shut them down.
                foreach (AppRunner runner in appRunners)
                {
                    Assert.True(runner.Environment.TryGetValue(ExpectedEnvVarName, out string expectedEnvVarValue));

                    await VerifyProcessAsync(apiClient, identifiers, await runner.ProcessIdTask, expectedEnvVarValue);

                    await runner.SendCommandAsync(TestAppScenarios.AsyncWait.Commands.Continue);
                }
            });

            for (int i = 0; i < appCount; i++)
            {
                Assert.True(0 == appRunners[i].ExitCode, $"App {i} exit code is non-zero.");
            }

            // Query for process identifiers
            identifiers = (await apiClient.GetProcessesAsync()).ToList();
            Assert.NotNull(identifiers);

            // Verify none of the apps are reported
            List <int> runnerProcessIds = new(appCount);

            for (int i = 0; i < appCount; i++)
            {
                runnerProcessIds.Add(await appRunners[i].ProcessIdTask);
            }

            foreach (ProcessIdentifier identifier in identifiers)
            {
                Assert.DoesNotContain(identifier.Pid, runnerProcessIds);
            }
        }