private async Task StopContainerAsync(Tye.Hosting.Model.Service service)
{
if (service.Items.TryGetValue(typeof(DockerInformation), out var value) && value is DockerInformation di)
{
di.StoppingTokenSource.Cancel();
await Task.WhenAll(di.Tasks);
}
}
static async Task KillProcessAsync(Tye.Hosting.Model.Service service)
{
if (service.Items.TryGetValue(typeof(ProcessInfo), out var stateObj) && stateObj is ProcessInfo state)
{
// Cancel the token before stopping the process
state.StoppedTokenSource.Cancel();
await Task.WhenAll(state.Tasks);
}
}
private static void PrintStdOutAndErr(Tye.Hosting.Model.Service service, string replica, ProcessResult result)
{
if (result.ExitCode != 0)
{
if (result.StandardOutput != null)
{
service.Logs.OnNext($"[{replica}]: {result.StandardOutput}");
}
if (result.StandardError != null)
{
service.Logs.OnNext($"[{replica}]: {result.StandardError}");
}
}
}
private async Task LaunchService(Tye.Hosting.Model.Application application, Tye.Hosting.Model.Service service)
{
var serviceDescription = service.Description;
var serviceName = serviceDescription.Name;
var path = "";
var workingDirectory = "";
var args = "";
if (serviceDescription.RunInfo is ProjectRunInfo project)
{
var expandedProject = Environment.ExpandEnvironmentVariables(project.Project);
var fullProjectPath = Path.GetFullPath(Path.Combine(application.ContextDirectory, expandedProject));
path = GetExePath(fullProjectPath);
workingDirectory = Path.GetDirectoryName(fullProjectPath) !;
args = project.Args ?? "";
service.Status.ProjectFilePath = fullProjectPath;
}
else if (serviceDescription.RunInfo is ExecutableRunInfo executable)
{
var expandedExecutable = Environment.ExpandEnvironmentVariables(executable.Executable);
path = Path.GetExtension(expandedExecutable) == ".dll" ?
Path.GetFullPath(Path.Combine(application.ContextDirectory, expandedExecutable)) :
expandedExecutable;
workingDirectory = executable.WorkingDirectory != null?
Path.GetFullPath(Path.Combine(application.ContextDirectory, Environment.ExpandEnvironmentVariables(executable.WorkingDirectory))) :
Path.GetDirectoryName(path) !;
args = executable.Args ?? "";
}
else
{
throw new InvalidOperationException("Unsupported ServiceType.");
}
// If this is a dll then use dotnet to run it
if (Path.GetExtension(path) == ".dll")
{
args = $"\"{path}\" {args}".Trim();
path = "dotnet";
}
service.Status.ExecutablePath = path;
service.Status.WorkingDirectory = workingDirectory;
service.Status.Args = args;
var processInfo = new ProcessInfo(new Task[service.Description.Replicas]);
if (service.Status.ProjectFilePath != null &&
service.Description.RunInfo is ProjectRunInfo project2 &&
project2.Build &&
_options.BuildProjects)
{
// Sometimes building can fail because of file locking (like files being open in VS)
_logger.LogInformation("Building project {ProjectFile}", service.Status.ProjectFilePath);
service.Logs.OnNext($"dotnet build \"{service.Status.ProjectFilePath}\" /nologo");
var buildResult = await ProcessUtil.RunAsync("dotnet", $"build \"{service.Status.ProjectFilePath}\" /nologo", throwOnError : false);
service.Logs.OnNext(buildResult.StandardOutput);
if (buildResult.ExitCode != 0)
{
_logger.LogInformation("Building {ProjectFile} failed with exit code {ExitCode}: \r\n" + buildResult.StandardOutput, service.Status.ProjectFilePath, buildResult.ExitCode);
return;
}
}
async Task RunApplicationAsync(IEnumerable <(int Port, int BindingPort, string?Protocol)> ports)
{
// Make sure we yield before trying to start the process, this is important so we don't hang startup
await Task.Yield();
var hasPorts = ports.Any();
var environment = new Dictionary <string, string>
{
// Default to development environment
["DOTNET_ENVIRONMENT"] = "Development"
};
// Set up environment variables to use the version of dotnet we're using to run
// this is important for tests where we're not using a globally-installed dotnet.
var dotnetRoot = GetDotnetRoot();
if (dotnetRoot is object)
{
environment["DOTNET_ROOT"] = dotnetRoot;
environment["DOTNET_MULTILEVEL_LOOKUP"] = "0";
environment["PATH"] = $"{dotnetRoot};{Environment.GetEnvironmentVariable("PATH")}";
}
application.PopulateEnvironment(service, (k, v) => environment[k] = v);
if (_options.DebugMode && (_options.DebugAllServices || _options.ServicesToDebug.Contains(serviceName, StringComparer.OrdinalIgnoreCase)))
{
environment["DOTNET_STARTUP_HOOKS"] = typeof(Hosting.Runtime.HostingRuntimeHelpers).Assembly.Location;
}
if (hasPorts)
{
// These are the ports that the application should use for binding
// 1. Configure ASP.NET Core to bind to those same ports
environment["ASPNETCORE_URLS"] = string.Join(";", ports.Select(p => $"{p.Protocol ?? "http"}://localhost:{p.Port}"));
// Set the HTTPS port for the redirect middleware
foreach (var p in ports)
{
if (string.Equals(p.Protocol, "https", StringComparison.OrdinalIgnoreCase))
{
// We need to set the redirect URL to the exposed port so the redirect works cleanly
environment["HTTPS_PORT"] = p.BindingPort.ToString();
}
}
// 3. For non-ASP.NET Core apps, pass the same information in the PORT env variable as a semicolon separated list.
environment["PORT"] = string.Join(";", ports.Select(p => $"{p.Port}"));
}
while (!processInfo.StoppedTokenSource.IsCancellationRequested)
{
var replica = serviceName + "_" + Guid.NewGuid().ToString().Substring(0, 10).ToLower();
var status = new ProcessStatus(service, replica);
service.Replicas[replica] = status;
service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Added, status));
// This isn't your host name
environment["APP_INSTANCE"] = replica;
status.ExitCode = null;
status.Pid = null;
status.Environment = environment;
if (hasPorts)
{
status.Ports = ports.Select(p => p.Port);
}
_logger.LogInformation("Launching service {ServiceName}: {ExePath} {args}", replica, path, args);
try
{
service.Logs.OnNext($"[{replica}]:{path} {args}");
var result = await ProcessUtil.RunAsync(path, args,
environmentVariables : environment,
workingDirectory : workingDirectory,
outputDataReceived : data => service.Logs.OnNext($"[{replica}]: {data}"),
onStart : pid =>
{
if (hasPorts)
{
_logger.LogInformation("{ServiceName} running on process id {PID} bound to {Address}", replica, pid, string.Join(", ", ports.Select(p => $"{p.Protocol ?? "http"}://localhost:{p.Port}")));
}
else
{
_logger.LogInformation("{ServiceName} running on process id {PID}", replica, pid);
}
status.Pid = pid;
service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Started, status));
},
throwOnError : false,
cancellationToken : processInfo.StoppedTokenSource.Token);
status.ExitCode = result.ExitCode;
if (status.Pid != null)
{
service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Stopped, status));
}
}
catch (Exception ex)
{
_logger.LogError(0, ex, "Failed to launch process for service {ServiceName}", replica);
try
{
await Task.Delay(5000, processInfo.StoppedTokenSource.Token);
}
catch (OperationCanceledException)
{
// Swallow cancellation exceptions and continue
}
}
service.Restarts++;
if (status.ExitCode != null)
{
_logger.LogInformation("{ServiceName} process exited with exit code {ExitCode}", replica, status.ExitCode);
}
// Remove the replica from the set
service.Replicas.TryRemove(replica, out _);
service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Removed, status));
}
}
if (serviceDescription.Bindings.Count > 0)
{
// Each replica is assigned a list of internal ports, one mapped to each external
// port
for (int i = 0; i < serviceDescription.Replicas; i++)
{
var ports = new List <(int, int, string?)>();
foreach (var binding in serviceDescription.Bindings)
{
if (binding.Port == null)
{
continue;
}
ports.Add((service.PortMap[binding.Port.Value][i], binding.Port.Value, binding.Protocol));
}
processInfo.Tasks[i] = RunApplicationAsync(ports);
}
}
else
{
for (int i = 0; i < service.Description.Replicas; i++)
{
processInfo.Tasks[i] = RunApplicationAsync(Enumerable.Empty <(int, int, string?)>());
}
}
service.Items[typeof(ProcessInfo)] = processInfo;
}
private async Task StartContainerAsync(Tye.Hosting.Model.Application application, Tye.Hosting.Model.Service service, DockerRunInfo docker)
{
if (!await DockerDetector.Instance.IsDockerInstalled.Value)
{
_logger.LogError("Unable to start docker container for service {ServiceName}, Docker is not installed.", service.Description.Name);
service.Logs.OnNext($"Unable to start docker container for service {service.Description.Name}, Docker is not installed.");
return;
}
if (!await DockerDetector.Instance.IsDockerConnectedToDaemon.Value)
{
_logger.LogError("Unable to start docker container for service {ServiceName}, Docker is not running.", service.Description.Name);
service.Logs.OnNext($"Unable to start docker container for service {service.Description.Name}, Docker is not running.");
return;
}
var serviceDescription = service.Description;
var environmentArguments = "";
var volumes = "";
var workingDirectory = docker.WorkingDirectory != null ? $"-w {docker.WorkingDirectory}" : "";
// This is .NET specific
var userSecretStore = GetUserSecretsPathFromSecrets();
if (!string.IsNullOrEmpty(userSecretStore))
{
// Map the user secrets on this drive to user secrets
docker.VolumeMappings[userSecretStore] = "/root/.microsoft/usersecrets:ro";
}
var dockerInfo = new DockerInformation(new Task[service.Description.Replicas]);
async Task RunDockerContainer(IEnumerable <(int Port, int?InternalPort, int BindingPort, string?Protocol)> ports)
{
var hasPorts = ports.Any();
var replica = service.Description.Name.ToLower() + "_" + Guid.NewGuid().ToString().Substring(0, 10).ToLower();
var status = new DockerStatus(service, replica);
service.Replicas[replica] = status;
service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Added, status));
var environment = new Dictionary <string, string>
{
// Default to development environment
["DOTNET_ENVIRONMENT"] = "Development",
// Remove the color codes from the console output
["DOTNET_LOGGING__CONSOLE__DISABLECOLORS"] = "true"
};
var portString = "";
if (hasPorts)
{
status.Ports = ports.Select(p => p.Port);
// These are the ports that the application should use for binding
// 1. Tell the docker container what port to bind to
portString = string.Join(" ", ports.Select(p => $"-p {p.Port}:{p.InternalPort ?? p.Port}"));
// 2. Configure ASP.NET Core to bind to those same ports
environment["ASPNETCORE_URLS"] = string.Join(";", ports.Select(p => $"{p.Protocol ?? "http"}://*:{p.InternalPort ?? p.Port}"));
// Set the HTTPS port for the redirect middleware
foreach (var p in ports)
{
if (string.Equals(p.Protocol, "https", StringComparison.OrdinalIgnoreCase))
{
// We need to set the redirect URL to the exposed port so the redirect works cleanly
environment["HTTPS_PORT"] = p.BindingPort.ToString();
}
}
// 3. For non-ASP.NET Core apps, pass the same information in the PORT env variable as a semicolon separated list.
environment["PORT"] = string.Join(";", ports.Select(p => $"{p.InternalPort ?? p.Port}"));
}
// See: https://github.com/docker/for-linux/issues/264
//
// The way we do proxying here doesn't really work for multi-container scenarios on linux
// without some more setup.
application.PopulateEnvironment(service, (key, value) => environment[key] = value, "host.docker.internal");
environment["APP_INSTANCE"] = replica;
foreach (var pair in environment)
{
environmentArguments += $"-e {pair.Key}={pair.Value} ";
}
foreach (var pair in docker.VolumeMappings)
{
var sourcePath = Path.GetFullPath(Path.Combine(application.ContextDirectory, pair.Key.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar)));
volumes += $"-v {sourcePath}:{pair.Value} ";
}
var command = $"run -d {workingDirectory} {volumes} {environmentArguments} {portString} --name {replica} --restart=unless-stopped {docker.Image} {docker.Args ?? ""}";
_logger.LogInformation("Running docker command {Command}", command);
service.Logs.OnNext($"[{replica}]: {command}");
status.DockerCommand = command;
var result = await ProcessUtil.RunAsync(
"docker",
command,
throwOnError : false,
cancellationToken : dockerInfo.StoppingTokenSource.Token,
outputDataReceived : data => service.Logs.OnNext($"[{replica}]: {data}"));
if (result.ExitCode != 0)
{
_logger.LogError("docker run failed for {ServiceName} with exit code {ExitCode}:" + result.StandardError, service.Description.Name, result.ExitCode);
service.Replicas.TryRemove(replica, out _);
service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Removed, status));
PrintStdOutAndErr(service, replica, result);
return;
}
var containerId = (string?)result.StandardOutput.Trim();
// There's a race condition that sometimes makes us miss the output
// so keep trying to get the container id
while (string.IsNullOrEmpty(containerId))
{
// Try to get the ID of the container
result = await ProcessUtil.RunAsync("docker", $"ps --no-trunc -f name={replica} --format " + "{{.ID}}");
containerId = result.ExitCode == 0 ? result.StandardOutput.Trim() : null;
}
var shortContainerId = containerId.Substring(0, Math.Min(12, containerId.Length));
status.ContainerId = shortContainerId;
_logger.LogInformation("Running container {ContainerName} with ID {ContainerId}", replica, shortContainerId);
service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Started, status));
_logger.LogInformation("Collecting docker logs for {ContainerName}.", replica);
await ProcessUtil.RunAsync("docker", $"logs -f {containerId}",
outputDataReceived : data => service.Logs.OnNext($"[{replica}]: {data}"),
onStart : pid =>
{
status.DockerLogsPid = pid;
},
throwOnError : false,
cancellationToken : dockerInfo.StoppingTokenSource.Token);
_logger.LogInformation("docker logs collection for {ContainerName} complete with exit code {ExitCode}", replica, result.ExitCode);
// Docker has a tendency to get stuck so we're going to timeout this shutdown process
var timeoutCts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
_logger.LogInformation("Stopping container {ContainerName} with ID {ContainerId}", replica, shortContainerId);
result = await ProcessUtil.RunAsync("docker", $"stop {containerId}", throwOnError : false, cancellationToken : timeoutCts.Token);
PrintStdOutAndErr(service, replica, result);
service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Stopped, status));
_logger.LogInformation("Stopped container {ContainerName} with ID {ContainerId} exited with {ExitCode}", replica, shortContainerId, result.ExitCode);
result = await ProcessUtil.RunAsync("docker", $"rm {containerId}", throwOnError : false, cancellationToken : timeoutCts.Token);
PrintStdOutAndErr(service, replica, result);
_logger.LogInformation("Removed container {ContainerName} with ID {ContainerId} exited with {ExitCode}", replica, shortContainerId, result.ExitCode);
service.Replicas.TryRemove(replica, out _);
service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Removed, status));
};
if (serviceDescription.Bindings.Count > 0)
{
// Each replica is assigned a list of internal ports, one mapped to each external
// port
for (var i = 0; i < serviceDescription.Replicas; i++)
{
var ports = new List <(int, int?, int, string?)>();
foreach (var binding in serviceDescription.Bindings)
{
if (binding.Port == null)
{
continue;
}
ports.Add((service.PortMap[binding.Port.Value][i], binding.InternalPort, binding.Port.Value, binding.Protocol));
}
dockerInfo.Tasks[i] = RunDockerContainer(ports);
}
}
else
{
for (var i = 0; i < service.Description.Replicas; i++)
{
dockerInfo.Tasks[i] = RunDockerContainer(Enumerable.Empty <(int, int?, int, string?)>());
}
}
service.Items[typeof(DockerInformation)] = dockerInfo;
}
private async Task StartContainerAsync(Tye.Hosting.Model.Application application, Tye.Hosting.Model.Service service, DockerRunInfo docker)
{
if (!await DockerDetector.Instance.IsDockerInstalled.Value)
{
_logger.LogError("Unable to start docker container for service {ServiceName}, Docker is not installed.", service.Description.Name);
service.Logs.OnNext($"Unable to start docker container for service {service.Description.Name}, Docker is not installed.");
return;
}
if (!await DockerDetector.Instance.IsDockerConnectedToDaemon.Value)
{
_logger.LogError("Unable to start docker container for service {ServiceName}, Docker is not running.", service.Description.Name);
service.Logs.OnNext($"Unable to start docker container for service {service.Description.Name}, Docker is not running.");
return;
}
var serviceDescription = service.Description;
var environmentArguments = "";
var dockerInfo = new DockerInformation(new Task[service.Description.Replicas]);
async Task RunDockerContainer(IEnumerable <(int Port, int?InternalPort, int BindingPort, string?Protocol)> ports)
{
var hasPorts = ports.Any();
var replica = service.Description.Name.ToLower() + "_" + Guid.NewGuid().ToString().Substring(0, 10).ToLower();
var status = new DockerStatus(service, replica);
service.Replicas[replica] = status;
service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Added, status));
var environment = new Dictionary <string, string>
{
// Default to development environment
["DOTNET_ENVIRONMENT"] = "Development",
// Remove the color codes from the console output
["DOTNET_LOGGING__CONSOLE__DISABLECOLORS"] = "true"
};
var portString = "";
if (hasPorts)
{
status.Ports = ports.Select(p => p.Port);
portString = string.Join(" ", ports.Select(p => $"-p {p.Port}:{p.InternalPort ?? p.Port}"));
foreach (var p in ports)
{
environment[$"{p.Protocol?.ToUpper() ?? "HTTP"}_PORT"] = p.BindingPort.ToString();
}
}
application.PopulateEnvironment(service, (key, value) => environment[key] = value, "host.docker.internal");
environment["APP_INSTANCE"] = replica;
foreach (var pair in environment)
{
environmentArguments += $"-e {pair.Key}={pair.Value} ";
}
var command = $"run -d {environmentArguments} {portString} --name {replica} --restart=unless-stopped {docker.Image} {docker.Args ?? ""}";
_logger.LogInformation("Running docker command {Command}", command);
service.Logs.OnNext($"[{replica}]: {command}");
status.DockerCommand = command;
var result = await ProcessUtil.RunAsync(
"docker",
command,
throwOnError : false,
cancellationToken : dockerInfo.StoppingTokenSource.Token,
outputDataReceived : data => service.Logs.OnNext($"[{replica}]: {data}"));
if (result.ExitCode != 0)
{
_logger.LogError("docker run failed for {ServiceName} with exit code {ExitCode}:" + result.StandardError, service.Description.Name, result.ExitCode);
service.Replicas.TryRemove(replica, out _);
service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Removed, status));
PrintStdOutAndErr(service, replica, result);
return;
}
var containerId = (string?)result.StandardOutput.Trim();
// There's a race condition that sometimes makes us miss the output
// so keep trying to get the container id
while (string.IsNullOrEmpty(containerId))
{
// Try to get the ID of the container
result = await ProcessUtil.RunAsync("docker", $"ps --no-trunc -f name={replica} --format " + "{{.ID}}");
containerId = result.ExitCode == 0 ? result.StandardOutput.Trim() : null;
}
var shortContainerId = containerId.Substring(0, Math.Min(12, containerId.Length));
status.ContainerId = shortContainerId;
_logger.LogInformation("Running container {ContainerName} with ID {ContainerId}", replica, shortContainerId);
service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Started, status));
_logger.LogInformation("Collecting docker logs for {ContainerName}.", replica);
await ProcessUtil.RunAsync("docker", $"logs -f {containerId}",
outputDataReceived : data => service.Logs.OnNext($"[{replica}]: {data}"),
onStart : pid =>
{
status.DockerLogsPid = pid;
},
throwOnError : false,
cancellationToken : dockerInfo.StoppingTokenSource.Token);
_logger.LogInformation("docker logs collection for {ContainerName} complete with exit code {ExitCode}", replica, result.ExitCode);
// Docker has a tendency to get stuck so we're going to timeout this shutdown process
var timeoutCts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
_logger.LogInformation("Stopping container {ContainerName} with ID {ContainerId}", replica, shortContainerId);
result = await ProcessUtil.RunAsync("docker", $"stop {containerId}", throwOnError : false, cancellationToken : timeoutCts.Token);
PrintStdOutAndErr(service, replica, result);
service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Stopped, status));
_logger.LogInformation("Stopped container {ContainerName} with ID {ContainerId} exited with {ExitCode}", replica, shortContainerId, result.ExitCode);
result = await ProcessUtil.RunAsync("docker", $"rm {containerId}", throwOnError : false, cancellationToken : timeoutCts.Token);
PrintStdOutAndErr(service, replica, result);
_logger.LogInformation("Removed container {ContainerName} with ID {ContainerId} exited with {ExitCode}", replica, shortContainerId, result.ExitCode);
service.Replicas.TryRemove(replica, out _);
service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Removed, status));
};
if (serviceDescription.Bindings.Count > 0)
{
// Each replica is assigned a list of internal ports, one mapped to each external
// port
for (var i = 0; i < serviceDescription.Replicas; i++)
{
var ports = new List <(int, int?, int, string?)>();
foreach (var binding in serviceDescription.Bindings)
{
if (binding.Port == null)
{
continue;
}
ports.Add((service.PortMap[binding.Port.Value][i], binding.InternalPort, binding.Port.Value, binding.Protocol));
}
dockerInfo.Tasks[i] = RunDockerContainer(ports);
}
}
else
{
for (var i = 0; i < service.Description.Replicas; i++)
{
dockerInfo.Tasks[i] = RunDockerContainer(Enumerable.Empty <(int, int?, int, string?)>());
}
}
service.Items[typeof(DockerInformation)] = dockerInfo;
}