private Task StartContainerAsync(Application application, Service service) { if (service.Description.DockerImage == null) { return(Task.CompletedTask); } if (!_dockerInstalled.Value) { _logger.LogError("Unable to start docker container for service {ServiceName}, Docker is not installed.", service.Description.Name); service.Logs.OnNext($"Unable to start docker container for service {service.Description.Name}, Docker is not installed."); return(Task.CompletedTask); } var serviceDescription = service.Description; var environmentArguments = ""; var dockerInfo = new DockerInformation() { Threads = new Thread[service.Description.Replicas.Value] }; void RunDockerContainer(IEnumerable <(int Port, int BindingPort, string Protocol)> ports) { var hasPorts = ports.Any(); var replica = service.Description.Name.ToLower() + "_" + Guid.NewGuid().ToString().Substring(0, 10).ToLower(); var status = new DockerStatus(); service.Replicas[replica] = status; var environment = new Dictionary <string, string> { // Default to development environment ["DOTNET_ENVIRONMENT"] = "Development", // Remove the color codes from the console output ["DOTNET_LOGGING__CONSOLE__DISABLECOLORS"] = "true" }; var portString = ""; if (hasPorts) { status.Ports = ports.Select(p => p.Port); portString = string.Join(" ", ports.Select(p => $"-p {p.Port}:{p.Port}")); // These ports should also be passed in not assuming ASP.NET Core environment["ASPNETCORE_URLS"] = string.Join(";", ports.Select(p => $"{p.Protocol ?? "http"}://*:{p.Port}")); foreach (var p in ports) { environment[$"{p.Protocol?.ToUpper() ?? "HTTP"}_PORT"] = p.BindingPort.ToString(); } } application.PopulateEnvironment(service, (key, value) => environment[key] = value, "host.docker.internal"); environment["APP_INSTANCE"] = replica; foreach (var pair in environment) { environmentArguments += $"-e {pair.Key}={pair.Value} "; } var command = $"run -d {environmentArguments} {portString} --name {replica} --restart=unless-stopped {service.Description.DockerImage} {service.Description.Args ?? ""}"; _logger.LogInformation("Running docker command {Command}", command); service.Logs.OnNext($"[{replica}]: {command}"); status.DockerCommand = command; var result = ProcessUtil.Run("docker", command, throwOnError: false, cancellationToken: dockerInfo.StoppingTokenSource.Token); if (result.ExitCode != 0) { _logger.LogError("docker run failed for {ServiceName} with exit code {ExitCode}:" + result.StandardError, service.Description.Name, result.ExitCode); service.Replicas.TryRemove(replica, out _); PrintStdOutAndErr(service, replica, result); return; } var containerId = result.StandardOutput.Trim(); // There's a race condition that sometimes makes us miss the output // so keep trying to get the container id while (string.IsNullOrEmpty(containerId)) { // Try to get the ID of the container result = ProcessUtil.Run("docker", $"ps --no-trunc -f name={replica} --format " + "{{.ID}}"); containerId = result.ExitCode == 0 ? result.StandardOutput.Trim() : null; } var shortContainerId = containerId.Substring(0, Math.Min(12, containerId.Length)); status.ContainerId = shortContainerId; _logger.LogInformation("Running container {ContainerName} with ID {ContainerId}", replica, shortContainerId); _logger.LogInformation("Collecting docker logs for {ContainerName}.", replica); ProcessUtil.Run("docker", $"logs -f {containerId}", outputDataReceived: data => service.Logs.OnNext($"[{replica}]: {data}"), onStart: pid => { status.DockerLogsPid = pid; }, throwOnError: false, cancellationToken: dockerInfo.StoppingTokenSource.Token); _logger.LogInformation("docker logs collection for {ContainerName} complete with exit code {ExitCode}", replica, result.ExitCode); // Docker has a tendency to hang so we're going to timeout this shutdown process var timeoutCts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); _logger.LogInformation("Stopping container {ContainerName} with ID {ContainerId}", replica, shortContainerId); result = ProcessUtil.Run("docker", $"stop {containerId}", throwOnError: false, cancellationToken: timeoutCts.Token); PrintStdOutAndErr(service, replica, result); _logger.LogInformation("Stopped container {ContainerName} with ID {ContainerId} exited with {ExitCode}", replica, shortContainerId, result.ExitCode); result = ProcessUtil.Run("docker", $"rm {containerId}", throwOnError: false, cancellationToken: timeoutCts.Token); PrintStdOutAndErr(service, replica, result); _logger.LogInformation("Removed container {ContainerName} with ID {ContainerId} exited with {ExitCode}", replica, shortContainerId, result.ExitCode); service.Replicas.TryRemove(replica, out _); }; if (serviceDescription.Bindings.Count > 0) { // Each replica is assigned a list of internal ports, one mapped to each external // port for (int i = 0; i < serviceDescription.Replicas; i++) { var ports = new List <(int, int, string)>(); foreach (var binding in serviceDescription.Bindings) { if (binding.Port == null) { continue; } ports.Add((service.PortMap[binding.Port.Value][i], binding.Port.Value, binding.Protocol)); } dockerInfo.Threads[i] = new Thread(() => RunDockerContainer(ports)); } } else { for (int i = 0; i < service.Description.Replicas; i++) { dockerInfo.Threads[i] = new Thread(() => RunDockerContainer(Enumerable.Empty <(int, int, string)>())); } } for (int i = 0; i < service.Description.Replicas; i++) { dockerInfo.Threads[i].Start(); } service.Items[typeof(DockerInformation)] = dockerInfo; return(Task.CompletedTask); }
private Task LaunchService(Application application, Service service) { var serviceDescription = service.Description; if (serviceDescription.DockerImage != null) { return(Docker.RunAsync(_logger, service)); } var serviceName = serviceDescription.Name; var path = ""; var workingDirectory = ""; var args = service.Description.Args ?? ""; if (serviceDescription.Project != null) { var fullProjectPath = Path.GetFullPath(Path.Combine(application.ContextDirectory, serviceDescription.Project)); path = GetExePath(fullProjectPath); workingDirectory = Path.GetDirectoryName(fullProjectPath); service.Status["projectFilePath"] = fullProjectPath; } else { path = Path.GetFullPath(Path.Combine(application.ContextDirectory, serviceDescription.Executable)); workingDirectory = serviceDescription.WorkingDirectory != null? Path.GetFullPath(Path.Combine(application.ContextDirectory, serviceDescription.WorkingDirectory)) : Path.GetDirectoryName(path); // If this is a dll then use dotnet to run it if (Path.GetExtension(path) == ".dll") { args = $"\"{path}\" {args}".Trim(); path = "dotnet"; } } service.Status["executablePath"] = path; service.Status["workingDirectory"] = workingDirectory; service.Status["args"] = args; service.Status["debugMode"] = _debugMode; var processInfo = new ProcessInfo { Threads = new Thread[service.Description.Replicas.Value] }; void RunApplication(IEnumerable <(int Port, string Protocol)> ports) { var hasPorts = ports.Any(); var restarts = 0; var environment = new Dictionary <string, string> { // Default to development environment ["DOTNET_ENVIRONMENT"] = "Development" }; application.PopulateEnvironment(service, (k, v) => environment[k] = v); if (_debugMode) { environment["DOTNET_STARTUP_HOOKS"] = typeof(Hosting.Runtime.HostingRuntimeHelpers).Assembly.Location; } if (hasPorts) { // These ports should also be passed in not assuming ASP.NET Core environment["ASPNETCORE_URLS"] = string.Join(";", ports.Select(p => $"{p.Protocol ?? "http"}://localhost:{p.Port}")); } while (!processInfo.StoppedTokenSource.IsCancellationRequested) { var replica = serviceName + "_" + Guid.NewGuid().ToString().Substring(0, 10).ToLower(); var status = service.Replicas[replica] = new ServiceReplica(); // This isn't your host name environment["APP_INSTANCE"] = replica; status["exitCode"] = null; status["pid"] = null; status["env"] = environment; if (hasPorts) { status["ports"] = ports; } service.Status["restarts"] = restarts; _logger.LogInformation("Launching service {ServiceName} from {ExePath} {args}", replica, path, args); var metricsTokenSource = CancellationTokenSource.CreateLinkedTokenSource(processInfo.StoppedTokenSource.Token); var metricsThread = new Thread(state => CollectMetrics((int)state, replica, status, metricsTokenSource.Token)); try { var result = ProcessUtil.Run(path, args, environmentVariables: environment, workingDirectory: workingDirectory, outputDataReceived: data => { if (data == null) { return; } service.Logs.Add("[" + replica + "]: " + data); }, onStart: pid => { if (hasPorts) { _logger.LogInformation("{ServiceName} running on process id {PID} bound to {Address}", replica, pid, string.Join(", ", ports.Select(p => $"{p.Protocol ?? "http"}://localhost:{p.Port}"))); } else { _logger.LogInformation("{ServiceName} running on process id {PID}", replica, pid); } status["pid"] = pid; metricsThread.Start(pid); }, throwOnError: false, cancellationToken: processInfo.StoppedTokenSource.Token); status["exitCode"] = result.ExitCode; if (status["pid"] != null) { metricsTokenSource.Cancel(); metricsThread.Join(); } } catch (Exception ex) { _logger.LogError(0, ex, "Failed to launch process for service {ServiceName}", replica); Thread.Sleep(5000); } restarts++; if (status["exitCode"] != null) { _logger.LogInformation("{ServiceName} process exited with exit code {ExitCode}", replica, status["exitCode"]); } // Remove the replica from the set service.Replicas.TryRemove(replica, out _); } } if (serviceDescription.Bindings.Count > 0) { // Each replica is assigned a list of internal ports, one mapped to each external // port for (int i = 0; i < serviceDescription.Replicas; i++) { var ports = new List <(int, string)>(); foreach (var binding in serviceDescription.Bindings) { if (binding.Port == null) { continue; } ports.Add((service.PortMap[binding.Port.Value][i], binding.Protocol)); } processInfo.Threads[i] = new Thread(() => RunApplication(ports)); } } else { for (int i = 0; i < service.Description.Replicas; i++) { processInfo.Threads[i] = new Thread(() => RunApplication(Enumerable.Empty <(int, string)>())); } } for (int i = 0; i < service.Description.Replicas; i++) { processInfo.Threads[i].Start(); } service.Items[typeof(ProcessInfo)] = processInfo; return(Task.CompletedTask); }
private async Task LaunchService(Application application, Service service) { var serviceDescription = service.Description; if (serviceDescription.DockerImage != null) { return; } var serviceName = serviceDescription.Name; var path = ""; var workingDirectory = ""; var args = service.Description.Args ?? ""; if (serviceDescription.Project != null) { var expandedProject = Environment.ExpandEnvironmentVariables(serviceDescription.Project); var fullProjectPath = Path.GetFullPath(Path.Combine(application.ContextDirectory, expandedProject)); path = GetExePath(fullProjectPath); workingDirectory = Path.GetDirectoryName(fullProjectPath); service.Status.ProjectFilePath = fullProjectPath; } else { var expandedExecutable = Environment.ExpandEnvironmentVariables(serviceDescription.Executable); path = Path.GetFullPath(Path.Combine(application.ContextDirectory, expandedExecutable)); workingDirectory = serviceDescription.WorkingDirectory != null? Path.GetFullPath(Path.Combine(application.ContextDirectory, Environment.ExpandEnvironmentVariables(serviceDescription.WorkingDirectory))) : Path.GetDirectoryName(path); } // If this is a dll then use dotnet to run it if (Path.GetExtension(path) == ".dll") { args = $"\"{path}\" {args}".Trim(); path = "dotnet"; } service.Status.ExecutablePath = path; service.Status.WorkingDirectory = workingDirectory; service.Status.Args = args; var processInfo = new ProcessInfo { Tasks = new Task[service.Description.Replicas.Value] }; if (service.Status.ProjectFilePath != null && service.Description.Build.GetValueOrDefault() && _buildProjects) { // Sometimes building can fail because of file locking (like files being open in VS) _logger.LogInformation("Building project {ProjectFile}", service.Status.ProjectFilePath); service.Logs.OnNext($"dotnet build \"{service.Status.ProjectFilePath}\" /nologo"); var buildResult = await ProcessUtil.RunAsync("dotnet", $"build \"{service.Status.ProjectFilePath}\" /nologo", outputDataReceived : data => service.Logs.OnNext(data), throwOnError : false); if (buildResult.ExitCode != 0) { _logger.LogInformation("Building {ProjectFile} failed with exit code {ExitCode}: " + buildResult.StandardOutput + buildResult.StandardError, service.Status.ProjectFilePath, buildResult.ExitCode); return; } } async Task RunApplicationAsync(IEnumerable <(int Port, int BindingPort, string Protocol)> ports) { var hasPorts = ports.Any(); var environment = new Dictionary <string, string> { // Default to development environment ["DOTNET_ENVIRONMENT"] = "Development" }; application.PopulateEnvironment(service, (k, v) => environment[k] = v); if (_debugMode) { environment["DOTNET_STARTUP_HOOKS"] = typeof(Hosting.Runtime.HostingRuntimeHelpers).Assembly.Location; } if (hasPorts) { // These ports should also be passed in not assuming ASP.NET Core environment["ASPNETCORE_URLS"] = string.Join(";", ports.Select(p => $"{p.Protocol ?? "http"}://localhost:{p.Port}")); foreach (var p in ports) { environment[$"{p.Protocol?.ToUpper() ?? "HTTP"}_PORT"] = p.BindingPort.ToString(); } } while (!processInfo.StoppedTokenSource.IsCancellationRequested) { var replica = serviceName + "_" + Guid.NewGuid().ToString().Substring(0, 10).ToLower(); var status = new ProcessStatus(service, replica); service.Replicas[replica] = status; service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Added, status)); // This isn't your host name environment["APP_INSTANCE"] = replica; status.ExitCode = null; status.Pid = null; status.Environment = environment; if (hasPorts) { status.Ports = ports.Select(p => p.Port); } _logger.LogInformation("Launching service {ServiceName}: {ExePath} {args}", replica, path, args); try { service.Logs.OnNext($"[{replica}]:{path} {args}"); var result = await ProcessUtil.RunAsync(path, args, environmentVariables : environment, workingDirectory : workingDirectory, outputDataReceived : data => service.Logs.OnNext($"[{replica}]: {data}"), onStart : pid => { if (hasPorts) { _logger.LogInformation("{ServiceName} running on process id {PID} bound to {Address}", replica, pid, string.Join(", ", ports.Select(p => $"{p.Protocol ?? "http"}://localhost:{p.Port}"))); } else { _logger.LogInformation("{ServiceName} running on process id {PID}", replica, pid); } status.Pid = pid; service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Started, status)); }, throwOnError : false, cancellationToken : processInfo.StoppedTokenSource.Token); status.ExitCode = result.ExitCode; if (status.Pid != null) { service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Stopped, status)); } } catch (Exception ex) { _logger.LogError(0, ex, "Failed to launch process for service {ServiceName}", replica); Thread.Sleep(5000); } service.Restarts++; if (status.ExitCode != null) { _logger.LogInformation("{ServiceName} process exited with exit code {ExitCode}", replica, status.ExitCode); } // Remove the replica from the set service.Replicas.TryRemove(replica, out _); service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Removed, status)); } } if (serviceDescription.Bindings.Count > 0) { // Each replica is assigned a list of internal ports, one mapped to each external // port for (int i = 0; i < serviceDescription.Replicas; i++) { var ports = new List <(int, int, string)>(); foreach (var binding in serviceDescription.Bindings) { if (binding.Port == null) { continue; } ports.Add((service.PortMap[binding.Port.Value][i], binding.Port.Value, binding.Protocol)); } processInfo.Tasks[i] = RunApplicationAsync(ports); } } else { for (int i = 0; i < service.Description.Replicas; i++) { processInfo.Tasks[i] = RunApplicationAsync(Enumerable.Empty <(int, int, string)>()); } } service.Items[typeof(ProcessInfo)] = processInfo; }
private Task LaunchService(Application application, Service service) { var serviceDescription = service.Description; if (serviceDescription.DockerImage != null) { return(Docker.RunAsync(_logger, service)); } var serviceName = serviceDescription.Name; var path = ""; var workingDirectory = ""; var args = service.Description.Args ?? ""; var applicationName = ""; if (serviceDescription.Project != null) { var fullProjectPath = Path.GetFullPath(Path.Combine(application.ContextDirectory, serviceDescription.Project)); path = GetExePath(fullProjectPath); workingDirectory = Path.GetDirectoryName(fullProjectPath); // TODO: Requires msbuild applicationName = Path.GetFileNameWithoutExtension(fullProjectPath); service.Status.ProjectFilePath = fullProjectPath; } else { applicationName = Path.GetFileNameWithoutExtension(serviceDescription.Executable); path = Path.GetFullPath(Path.Combine(application.ContextDirectory, serviceDescription.Executable)); workingDirectory = serviceDescription.WorkingDirectory != null? Path.GetFullPath(Path.Combine(application.ContextDirectory, serviceDescription.WorkingDirectory)) : Path.GetDirectoryName(path); } // If this is a dll then use dotnet to run it if (Path.GetExtension(path) == ".dll") { applicationName = Path.GetFileNameWithoutExtension(path); args = $"\"{path}\" {args}".Trim(); path = "dotnet"; } service.Status.ExecutablePath = path; service.Status.WorkingDirectory = workingDirectory; service.Status.Args = args; var processInfo = new ProcessInfo { Threads = new Thread[service.Description.Replicas.Value] }; if (service.Status.ProjectFilePath != null && service.Description.Build && _buildProjects) { _logger.LogInformation("Building project {ProjectFile}", service.Status.ProjectFilePath); service.Logs.OnNext("======================BUILDING===================="); var buildResult = ProcessUtil.Run("dotnet", $"build \"{service.Status.ProjectFilePath}\" /nologo", outputDataReceived: data => service.Logs.OnNext(data), throwOnError: false); service.Logs.OnNext(""); if (buildResult.ExitCode != 0) { _logger.LogInformation("Building {ProjectFile} failed with exit code {ExitCode}: " + buildResult.StandardError, service.Status.ProjectFilePath, buildResult.ExitCode); return(Task.CompletedTask); } } void RunApplication(IEnumerable <(int Port, int BindingPort, string Protocol)> ports) { var hasPorts = ports.Any(); var environment = new Dictionary <string, string> { // Default to development environment ["DOTNET_ENVIRONMENT"] = "Development" }; application.PopulateEnvironment(service, (k, v) => environment[k] = v); if (_debugMode) { environment["DOTNET_STARTUP_HOOKS"] = typeof(Hosting.Runtime.HostingRuntimeHelpers).Assembly.Location; } if (hasPorts) { // These ports should also be passed in not assuming ASP.NET Core environment["ASPNETCORE_URLS"] = string.Join(";", ports.Select(p => $"{p.Protocol ?? "http"}://localhost:{p.Port}")); foreach (var p in ports) { environment[$"{p.Protocol?.ToUpper() ?? "HTTP"}_PORT"] = p.BindingPort.ToString(); } } while (!processInfo.StoppedTokenSource.IsCancellationRequested) { var replica = serviceName + "_" + Guid.NewGuid().ToString().Substring(0, 10).ToLower(); var status = new ProcessStatus(); service.Replicas[replica] = status; // This isn't your host name environment["APP_INSTANCE"] = replica; status.ExitCode = null; status.Pid = null; status.Environment = environment; if (hasPorts) { status.Ports = ports.Select(p => p.Port); } _logger.LogInformation("Launching service {ServiceName}: {ExePath} {args}", replica, path, args); var metricsTokenSource = CancellationTokenSource.CreateLinkedTokenSource(processInfo.StoppedTokenSource.Token); // This is the thread that will collect diagnostics from the running process // - Logs - I'll collect structured logs from Microsoft.Extensions.Logging // - Metrics - It'll collect EventCounters // - Distribued Traces - It'll create spans var diagnosticsThread = new Thread(state => { _diagnosticsCollector.ProcessEvents( applicationName, service.Description.Name, (int)state, replica, status, metricsTokenSource.Token); }); try { var result = ProcessUtil.Run(path, args, environmentVariables: environment, workingDirectory: workingDirectory, outputDataReceived: data => service.Logs.OnNext("[" + replica + "]: " + data), onStart: pid => { if (hasPorts) { _logger.LogInformation("{ServiceName} running on process id {PID} bound to {Address}", replica, pid, string.Join(", ", ports.Select(p => $"{p.Protocol ?? "http"}://localhost:{p.Port}"))); } else { _logger.LogInformation("{ServiceName} running on process id {PID}", replica, pid); } status.Pid = pid; diagnosticsThread.Start(pid); }, throwOnError: false, cancellationToken: processInfo.StoppedTokenSource.Token); status.ExitCode = result.ExitCode; if (status.Pid != null) { metricsTokenSource.Cancel(); diagnosticsThread.Join(); } } catch (Exception ex) { _logger.LogError(0, ex, "Failed to launch process for service {ServiceName}", replica); Thread.Sleep(5000); } service.Restarts++; if (status.ExitCode != null) { _logger.LogInformation("{ServiceName} process exited with exit code {ExitCode}", replica, status.ExitCode); } // Remove the replica from the set service.Replicas.TryRemove(replica, out _); } } if (serviceDescription.Bindings.Count > 0) { // Each replica is assigned a list of internal ports, one mapped to each external // port for (int i = 0; i < serviceDescription.Replicas; i++) { var ports = new List <(int, int, string)>(); foreach (var binding in serviceDescription.Bindings) { if (binding.Port == null) { continue; } ports.Add((service.PortMap[binding.Port.Value][i], binding.Port.Value, binding.Protocol)); } processInfo.Threads[i] = new Thread(() => RunApplication(ports)); } } else { for (int i = 0; i < service.Description.Replicas; i++) { processInfo.Threads[i] = new Thread(() => RunApplication(Enumerable.Empty <(int, int, string)>())); } } for (int i = 0; i < service.Description.Replicas; i++) { processInfo.Threads[i].Start(); } service.Items[typeof(ProcessInfo)] = processInfo; return(Task.CompletedTask); }