public async Task WhenConnectingAndServerIsUp_ThenServerStatusIsPopulated()
{
var output = new StringBuilder();
output.AppendLine("Client: ");
output.AppendLine(" Version: 17.12.0-ce");
output.AppendLine(" API version: 1.35");
output.AppendLine(" Go version: go1.9.2");
output.AppendLine(" Git commit: c97c6d6");
output.AppendLine(" Built: Wed Dec 27 20:05:22 2017");
output.AppendLine(" OS/Arch: windows/amd64");
output.AppendLine("");
output.AppendLine("Server: ");
output.AppendLine(" Engine:");
output.AppendLine(" Version: 17.12.0-ce");
output.AppendLine(" API version: 1.35 (minimum version 1.24)");
output.AppendLine(" Go version: go1.9.2");
output.AppendLine(" Git commit: c97c6d6");
output.AppendLine(" Built: Wed Dec 27 20:15:52 2017");
output.AppendLine(" OS/Arch: windows/amd64");
output.AppendLine(" Experimental: true");
var shellExecutor = new Mock <IShellExecutor>();
shellExecutor
.Setup(e => e.Execute(Commands.Docker, "version"))
.ReturnsAsync(new ShellExecuteResult(true, output.ToString(), ""));
var status = new DockerStatus(shellExecutor.Object);
await status.Connect();
status.ClientDetails.IsRunning.Should().BeTrue();
status.ClientDetails.Error.Should().BeNullOrEmpty();
status.ClientDetails.Version.Should().Be("17.12.0-ce");
status.ClientDetails.ApiVersion.Should().Be("1.35");
status.ClientDetails.GoVersion.Should().Be("go1.9.2");
status.ClientDetails.GitCommit.Should().Be("c97c6d6");
status.ClientDetails.Built.Should().Be("Wed Dec 27 20:05:22 2017");
status.ClientDetails.OsArch.Should().Be("windows/amd64");
status.ClientDetails.Experimental.Should().Be("");
status.ServerDetails.IsRunning.Should().BeTrue();
status.ServerDetails.Error.Should().BeNullOrEmpty();
status.ServerDetails.Version.Should().Be("17.12.0-ce");
status.ServerDetails.ApiVersion.Should().Be("1.35 (minimum version 1.24)");
status.ServerDetails.GoVersion.Should().Be("go1.9.2");
status.ServerDetails.GitCommit.Should().Be("c97c6d6");
status.ServerDetails.Built.Should().Be("Wed Dec 27 20:15:52 2017");
status.ServerDetails.OsArch.Should().Be("windows/amd64");
status.ServerDetails.Experimental.Should().Be("true");
}
public async Task WhenConnectingAndServerIsDown_ThenServerStatusIsNotRunning()
{
var output = new StringBuilder();
output.AppendLine("Client: ");
output.AppendLine(" Version: 17.12.0-ce");
output.AppendLine(" API version: 1.35");
output.AppendLine(" Go version: go1.9.2");
output.AppendLine(" Git commit: c97c6d6");
output.AppendLine(" Built: Wed Dec 27 20:05:22 2017");
output.AppendLine(" OS/Arch: windows/amd64");
var shellExecutor = new Mock <IShellExecutor>();
shellExecutor
.Setup(e => e.Execute(Commands.Docker, "version"))
.ReturnsAsync(new ShellExecuteResult(true, output.ToString(), "error during connect: Get http://%2F%2F.%2Fpipe%2Fdocker_engine/v1.35/version: open //./pipe/docker_engine: The system cannot find the file specified. In the default daemon configuration on Windows, the docker client must be run elevated to connect. This error may also indicate that the docker daemon is not running."));
var status = new DockerStatus(shellExecutor.Object);
await status.Connect();
status.ClientDetails.IsRunning.Should().BeTrue();
status.ClientDetails.Error.Should().BeNullOrEmpty();
status.ClientDetails.Version.Should().Be("17.12.0-ce");
status.ClientDetails.ApiVersion.Should().Be("1.35");
status.ClientDetails.GoVersion.Should().Be("go1.9.2");
status.ClientDetails.GitCommit.Should().Be("c97c6d6");
status.ClientDetails.Built.Should().Be("Wed Dec 27 20:05:22 2017");
status.ClientDetails.OsArch.Should().Be("windows/amd64");
status.ClientDetails.Experimental.Should().Be("");
status.ServerDetails.IsRunning.Should().BeFalse();
status.ServerDetails.Error.Should().Contain("error during connect: Get http://%2F%2F.%2Fpipe%2Fdocker_engine/v1.35/version: open //./pipe/docker_engine: The system cannot find the file specified. In the default daemon configuration on Windows, the docker client must be run elevated to connect. This error may also indicate that the docker daemon is not running.");
status.ServerDetails.Version.Should().BeNullOrEmpty();
status.ServerDetails.ApiVersion.Should().BeNullOrEmpty();
status.ServerDetails.GoVersion.Should().BeNullOrEmpty();
status.ServerDetails.GitCommit.Should().BeNullOrEmpty();
status.ServerDetails.Built.Should().BeNullOrEmpty();
status.ServerDetails.OsArch.Should().BeNullOrEmpty();
status.ServerDetails.Experimental.Should().BeNullOrEmpty();
}
private async Task StartContainerAsync(Tye.Hosting.Model.Application application, Tye.Hosting.Model.Service service, DockerRunInfo docker)
{
if (!await DockerDetector.Instance.IsDockerInstalled.Value)
{
_logger.LogError("Unable to start docker container for service {ServiceName}, Docker is not installed.", service.Description.Name);
service.Logs.OnNext($"Unable to start docker container for service {service.Description.Name}, Docker is not installed.");
return;
}
if (!await DockerDetector.Instance.IsDockerConnectedToDaemon.Value)
{
_logger.LogError("Unable to start docker container for service {ServiceName}, Docker is not running.", service.Description.Name);
service.Logs.OnNext($"Unable to start docker container for service {service.Description.Name}, Docker is not running.");
return;
}
var serviceDescription = service.Description;
var environmentArguments = "";
var volumes = "";
var workingDirectory = docker.WorkingDirectory != null ? $"-w {docker.WorkingDirectory}" : "";
// This is .NET specific
var userSecretStore = GetUserSecretsPathFromSecrets();
if (!string.IsNullOrEmpty(userSecretStore))
{
// Map the user secrets on this drive to user secrets
docker.VolumeMappings[userSecretStore] = "/root/.microsoft/usersecrets:ro";
}
var dockerInfo = new DockerInformation(new Task[service.Description.Replicas]);
async Task RunDockerContainer(IEnumerable <(int Port, int?InternalPort, int BindingPort, string?Protocol)> ports)
{
var hasPorts = ports.Any();
var replica = service.Description.Name.ToLower() + "_" + Guid.NewGuid().ToString().Substring(0, 10).ToLower();
var status = new DockerStatus(service, replica);
service.Replicas[replica] = status;
service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Added, status));
var environment = new Dictionary <string, string>
{
// Default to development environment
["DOTNET_ENVIRONMENT"] = "Development",
// Remove the color codes from the console output
["DOTNET_LOGGING__CONSOLE__DISABLECOLORS"] = "true"
};
var portString = "";
if (hasPorts)
{
status.Ports = ports.Select(p => p.Port);
// These are the ports that the application should use for binding
// 1. Tell the docker container what port to bind to
portString = string.Join(" ", ports.Select(p => $"-p {p.Port}:{p.InternalPort ?? p.Port}"));
// 2. Configure ASP.NET Core to bind to those same ports
environment["ASPNETCORE_URLS"] = string.Join(";", ports.Select(p => $"{p.Protocol ?? "http"}://*:{p.InternalPort ?? p.Port}"));
// Set the HTTPS port for the redirect middleware
foreach (var p in ports)
{
if (string.Equals(p.Protocol, "https", StringComparison.OrdinalIgnoreCase))
{
// We need to set the redirect URL to the exposed port so the redirect works cleanly
environment["HTTPS_PORT"] = p.BindingPort.ToString();
}
}
// 3. For non-ASP.NET Core apps, pass the same information in the PORT env variable as a semicolon separated list.
environment["PORT"] = string.Join(";", ports.Select(p => $"{p.InternalPort ?? p.Port}"));
}
// See: https://github.com/docker/for-linux/issues/264
//
// The way we do proxying here doesn't really work for multi-container scenarios on linux
// without some more setup.
application.PopulateEnvironment(service, (key, value) => environment[key] = value, "host.docker.internal");
environment["APP_INSTANCE"] = replica;
foreach (var pair in environment)
{
environmentArguments += $"-e {pair.Key}={pair.Value} ";
}
foreach (var pair in docker.VolumeMappings)
{
var sourcePath = Path.GetFullPath(Path.Combine(application.ContextDirectory, pair.Key.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar)));
volumes += $"-v {sourcePath}:{pair.Value} ";
}
var command = $"run -d {workingDirectory} {volumes} {environmentArguments} {portString} --name {replica} --restart=unless-stopped {docker.Image} {docker.Args ?? ""}";
_logger.LogInformation("Running docker command {Command}", command);
service.Logs.OnNext($"[{replica}]: {command}");
status.DockerCommand = command;
var result = await ProcessUtil.RunAsync(
"docker",
command,
throwOnError : false,
cancellationToken : dockerInfo.StoppingTokenSource.Token,
outputDataReceived : data => service.Logs.OnNext($"[{replica}]: {data}"));
if (result.ExitCode != 0)
{
_logger.LogError("docker run failed for {ServiceName} with exit code {ExitCode}:" + result.StandardError, service.Description.Name, result.ExitCode);
service.Replicas.TryRemove(replica, out _);
service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Removed, status));
PrintStdOutAndErr(service, replica, result);
return;
}
var containerId = (string?)result.StandardOutput.Trim();
// There's a race condition that sometimes makes us miss the output
// so keep trying to get the container id
while (string.IsNullOrEmpty(containerId))
{
// Try to get the ID of the container
result = await ProcessUtil.RunAsync("docker", $"ps --no-trunc -f name={replica} --format " + "{{.ID}}");
containerId = result.ExitCode == 0 ? result.StandardOutput.Trim() : null;
}
var shortContainerId = containerId.Substring(0, Math.Min(12, containerId.Length));
status.ContainerId = shortContainerId;
_logger.LogInformation("Running container {ContainerName} with ID {ContainerId}", replica, shortContainerId);
service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Started, status));
_logger.LogInformation("Collecting docker logs for {ContainerName}.", replica);
await ProcessUtil.RunAsync("docker", $"logs -f {containerId}",
outputDataReceived : data => service.Logs.OnNext($"[{replica}]: {data}"),
onStart : pid =>
{
status.DockerLogsPid = pid;
},
throwOnError : false,
cancellationToken : dockerInfo.StoppingTokenSource.Token);
_logger.LogInformation("docker logs collection for {ContainerName} complete with exit code {ExitCode}", replica, result.ExitCode);
// Docker has a tendency to get stuck so we're going to timeout this shutdown process
var timeoutCts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
_logger.LogInformation("Stopping container {ContainerName} with ID {ContainerId}", replica, shortContainerId);
result = await ProcessUtil.RunAsync("docker", $"stop {containerId}", throwOnError : false, cancellationToken : timeoutCts.Token);
PrintStdOutAndErr(service, replica, result);
service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Stopped, status));
_logger.LogInformation("Stopped container {ContainerName} with ID {ContainerId} exited with {ExitCode}", replica, shortContainerId, result.ExitCode);
result = await ProcessUtil.RunAsync("docker", $"rm {containerId}", throwOnError : false, cancellationToken : timeoutCts.Token);
PrintStdOutAndErr(service, replica, result);
_logger.LogInformation("Removed container {ContainerName} with ID {ContainerId} exited with {ExitCode}", replica, shortContainerId, result.ExitCode);
service.Replicas.TryRemove(replica, out _);
service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Removed, status));
};
if (serviceDescription.Bindings.Count > 0)
{
// Each replica is assigned a list of internal ports, one mapped to each external
// port
for (var i = 0; i < serviceDescription.Replicas; i++)
{
var ports = new List <(int, int?, int, string?)>();
foreach (var binding in serviceDescription.Bindings)
{
if (binding.Port == null)
{
continue;
}
ports.Add((service.PortMap[binding.Port.Value][i], binding.InternalPort, binding.Port.Value, binding.Protocol));
}
dockerInfo.Tasks[i] = RunDockerContainer(ports);
}
}
else
{
for (var i = 0; i < service.Description.Replicas; i++)
{
dockerInfo.Tasks[i] = RunDockerContainer(Enumerable.Empty <(int, int?, int, string?)>());
}
}
service.Items[typeof(DockerInformation)] = dockerInfo;
}
private async Task StartContainerAsync(Application application, Service service, DockerRunInfo docker, string?dockerNetwork)
{
var serviceDescription = service.Description;
var environmentArguments = "";
var volumes = "";
var workingDirectory = docker.WorkingDirectory != null ? $"-w {docker.WorkingDirectory}" : "";
var hostname = "host.docker.internal";
var dockerImage = docker.Image ?? service.Description.Name;
if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
{
// See: https://github.com/docker/for-linux/issues/264
//
// host.docker.internal is making it's way into linux docker but doesn't work yet
// instead we use the machine IP
var addresses = await Dns.GetHostAddressesAsync(Dns.GetHostName());
hostname = addresses[0].ToString();
}
async Task RunDockerContainer(IEnumerable <(int ExternalPort, int Port, int?ContainerPort, string?Protocol)> ports, CancellationToken cancellationToken)
{
var hasPorts = ports.Any();
var replica = service.Description.Name.ToLower() + "_" + Guid.NewGuid().ToString().Substring(0, 10).ToLower();
var status = new DockerStatus(service, replica);
service.Replicas[replica] = status;
service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Added, status));
var environment = new Dictionary <string, string>();
var portString = "";
if (hasPorts)
{
status.Ports = ports.Select(p => p.Port);
status.Bindings = ports.Select(p => new ReplicaBinding()
{
Port = p.Port, ExternalPort = p.ExternalPort, Protocol = p.Protocol
}).ToList();
// These are the ports that the application should use for binding
// 1. Tell the docker container what port to bind to
portString = docker.Private ? "" : string.Join(" ", ports.Select(p => $"-p {p.Port}:{p.ContainerPort ?? p.Port}"));
if (docker.IsAspNet)
{
// 2. Configure ASP.NET Core to bind to those same ports
environment["ASPNETCORE_URLS"] = string.Join(";", ports.Select(p => $"{p.Protocol ?? "http"}://*:{p.ContainerPort ?? p.Port}"));
// Set the HTTPS port for the redirect middleware
foreach (var p in ports)
{
if (string.Equals(p.Protocol, "https", StringComparison.OrdinalIgnoreCase))
{
// We need to set the redirect URL to the exposed port so the redirect works cleanly
environment["HTTPS_PORT"] = p.ExternalPort.ToString();
}
}
}
// 3. For non-ASP.NET Core apps, pass the same information in the PORT env variable as a semicolon separated list.
environment["PORT"] = string.Join(";", ports.Select(p => $"{p.ContainerPort ?? p.Port}"));
// This the port for the container proxy (containerport:externalport)
environment["PROXY_PORT"] = string.Join(";", ports.Select(p => $"{p.ContainerPort ?? p.Port}:{p.ExternalPort}"));
}
// See: https://github.com/docker/for-linux/issues/264
//
// The way we do proxying here doesn't really work for multi-container scenarios on linux
// without some more setup.
application.PopulateEnvironment(service, (key, value) => environment[key] = value, hostname !);
environment["APP_INSTANCE"] = replica;
environment["CONTAINER_HOST"] = hostname !;
status.Environment = environment;
foreach (var pair in environment)
{
environmentArguments += $"-e \"{pair.Key}={pair.Value}\" ";
}
foreach (var volumeMapping in docker.VolumeMappings)
{
if (volumeMapping.Source != null)
{
var sourcePath = Path.GetFullPath(Path.Combine(application.ContextDirectory, volumeMapping.Source));
volumes += $"-v {sourcePath}:{volumeMapping.Target} ";
}
else if (volumeMapping.Name != null)
{
volumes += $"-v {volumeMapping.Name}:{volumeMapping.Target} ";
}
}
var command = $"run -d {workingDirectory} {volumes} {environmentArguments} {portString} --name {replica} --restart=unless-stopped {dockerImage} {docker.Args ?? ""}";
_logger.LogInformation("Running image {Image} for {Replica}", docker.Image, replica);
service.Logs.OnNext($"[{replica}]: docker {command}");
status.DockerCommand = command;
status.DockerNetwork = dockerNetwork;
WriteReplicaToStore(replica);
var result = await ProcessUtil.RunAsync(
"docker",
command,
throwOnError : false,
cancellationToken : cancellationToken,
outputDataReceived : data => service.Logs.OnNext($"[{replica}]: {data}"),
errorDataReceived : data => service.Logs.OnNext($"[{replica}]: {data}"));
if (result.ExitCode != 0)
{
_logger.LogError("docker run failed for {ServiceName} with exit code {ExitCode}:" + result.StandardError, service.Description.Name, result.ExitCode);
service.Replicas.TryRemove(replica, out var _);
service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Removed, status));
PrintStdOutAndErr(service, replica, result);
return;
}
var containerId = (string?)result.StandardOutput.Trim();
// There's a race condition that sometimes makes us miss the output
// so keep trying to get the container id
while (string.IsNullOrEmpty(containerId))
{
// Try to get the ID of the container
result = await ProcessUtil.RunAsync("docker", $"ps --no-trunc -f name={replica} --format " + "{{.ID}}");
containerId = result.ExitCode == 0 ? result.StandardOutput.Trim() : null;
}
var shortContainerId = containerId.Substring(0, Math.Min(12, containerId.Length));
status.ContainerId = shortContainerId;
_logger.LogInformation("Running container {ContainerName} with ID {ContainerId}", replica, shortContainerId);
if (!string.IsNullOrEmpty(dockerNetwork))
{
status.DockerNetworkAlias = docker.NetworkAlias ?? serviceDescription !.Name;
var networkCommand = $"network connect {dockerNetwork} {replica} --alias {status.DockerNetworkAlias}";
service.Logs.OnNext($"[{replica}]: docker {networkCommand}");
_logger.LogInformation("Running docker command {Command}", networkCommand);
result = await ProcessUtil.RunAsync("docker", networkCommand);
PrintStdOutAndErr(service, replica, result);
}
var sentStartedEvent = false;
while (!cancellationToken.IsCancellationRequested)
{
if (sentStartedEvent)
{
using var restartCts = new CancellationTokenSource(DockerStopTimeout);
result = await ProcessUtil.RunAsync("docker", $"restart {containerId}", throwOnError : false, cancellationToken : restartCts.Token);
if (restartCts.IsCancellationRequested)
{
_logger.LogWarning($"Failed to restart container after {DockerStopTimeout.Seconds} seconds.", replica, shortContainerId);
break; // implement retry mechanism?
}
else if (result.ExitCode != 0)
{
_logger.LogWarning($"Failed to restart container due to exit code {result.ExitCode}.", replica, shortContainerId);
break;
}
service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Stopped, status));
}
using var stoppingCts = new CancellationTokenSource();
status.StoppingTokenSource = stoppingCts;
service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Started, status));
sentStartedEvent = true;
await using var _ = cancellationToken.Register(() => status.StoppingTokenSource.Cancel());
_logger.LogInformation("Collecting docker logs for {ContainerName}.", replica);
var backOff = TimeSpan.FromSeconds(5);
while (!status.StoppingTokenSource.Token.IsCancellationRequested)
{
var logsRes = await ProcessUtil.RunAsync("docker", $"logs -f {containerId}",
outputDataReceived : data => service.Logs.OnNext($"[{replica}]: {data}"),
errorDataReceived : data => service.Logs.OnNext($"[{replica}]: {data}"),
throwOnError : false,
cancellationToken : status.StoppingTokenSource.Token);
if (logsRes.ExitCode != 0)
{
break;
}
if (!status.StoppingTokenSource.IsCancellationRequested)
{
try
{
// Avoid spamming logs if restarts are happening
await Task.Delay(backOff, status.StoppingTokenSource.Token);
}
catch (OperationCanceledException)
{
break;
}
}
backOff *= 2;
}
_logger.LogInformation("docker logs collection for {ContainerName} complete with exit code {ExitCode}", replica, result.ExitCode);
status.StoppingTokenSource = null;
}
// Docker has a tendency to get stuck so we're going to timeout this shutdown process
var timeoutCts = new CancellationTokenSource(DockerStopTimeout);
_logger.LogInformation("Stopping container {ContainerName} with ID {ContainerId}", replica, shortContainerId);
result = await ProcessUtil.RunAsync("docker", $"stop {containerId}", throwOnError : false, cancellationToken : timeoutCts.Token);
if (timeoutCts.IsCancellationRequested)
{
_logger.LogWarning($"Failed to stop container after {DockerStopTimeout.Seconds} seconds, container will most likely be running.", replica, shortContainerId);
}
PrintStdOutAndErr(service, replica, result);
if (sentStartedEvent)
{
service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Stopped, status));
}
_logger.LogInformation("Stopped container {ContainerName} with ID {ContainerId} exited with {ExitCode}", replica, shortContainerId, result.ExitCode);
result = await ProcessUtil.RunAsync("docker", $"rm {containerId}", throwOnError : false, cancellationToken : timeoutCts.Token);
if (timeoutCts.IsCancellationRequested)
{
_logger.LogWarning($"Failed to remove container after {DockerStopTimeout.Seconds} seconds, container will most likely still exist.", replica, shortContainerId);
}
PrintStdOutAndErr(service, replica, result);
_logger.LogInformation("Removed container {ContainerName} with ID {ContainerId} exited with {ExitCode}", replica, shortContainerId, result.ExitCode);
service.Replicas.TryRemove(replica, out var _);
service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Removed, status));
};
async Task DockerBuildAsync(CancellationToken cancellationToken)
{
if (docker.DockerFile != null)
{
_logger.LogInformation("Building docker image {Image} from docker file", dockerImage);
void Log(string data)
{
_logger.LogInformation("[" + serviceDescription !.Name + "]:" + data);
service.Logs.OnNext(data);
}
var dockerBuildResult = await ProcessUtil.RunAsync(
$"docker",
$"build \"{docker.DockerFileContext?.FullName}\" -t {dockerImage} -f \"{docker.DockerFile}\"",
outputDataReceived : Log,
errorDataReceived : Log,
workingDirectory : docker.WorkingDirectory,
cancellationToken : cancellationToken,
throwOnError : false);
if (dockerBuildResult.ExitCode != 0)
{
throw new CommandException("'docker build' failed.");
}
}
}
Task DockerRunAsync(CancellationToken cancellationToken)
{
var tasks = new Task[serviceDescription !.Replicas];
if (serviceDescription.Bindings.Count > 0)
{
// Each replica is assigned a list of internal ports, one mapped to each external
// port
for (var i = 0; i < serviceDescription.Replicas; i++)
{
var ports = new List <(int, int, int?, string?)>();
foreach (var binding in serviceDescription.Bindings)
{
if (binding.Port == null)
{
continue;
}
ports.Add((binding.Port.Value, binding.ReplicaPorts[i], binding.ContainerPort, binding.Protocol));
}
tasks[i] = RunDockerContainer(ports, cancellationToken);
}
}
else
{
for (var i = 0; i < service.Description.Replicas; i++)
{
tasks[i] = RunDockerContainer(Enumerable.Empty <(int, int, int?, string?)>(), cancellationToken);
}
}
return(Task.WhenAll(tasks));
}
private Task StartContainerAsync(Application application, Service service)
{
if (service.Description.DockerImage == null)
{
return(Task.CompletedTask);
}
if (!_dockerInstalled.Value)
{
_logger.LogError("Unable to start docker container for service {ServiceName}, Docker is not installed.", service.Description.Name);
service.Logs.OnNext($"Unable to start docker container for service {service.Description.Name}, Docker is not installed.");
return(Task.CompletedTask);
}
var serviceDescription = service.Description;
var environmentArguments = "";
var dockerInfo = new DockerInformation()
{
Threads = new Thread[service.Description.Replicas.Value]
};
void RunDockerContainer(IEnumerable <(int Port, int BindingPort, string Protocol)> ports)
{
var hasPorts = ports.Any();
var replica = service.Description.Name.ToLower() + "_" + Guid.NewGuid().ToString().Substring(0, 10).ToLower();
var status = new DockerStatus();
service.Replicas[replica] = status;
var environment = new Dictionary <string, string>
{
// Default to development environment
["DOTNET_ENVIRONMENT"] = "Development",
// Remove the color codes from the console output
["DOTNET_LOGGING__CONSOLE__DISABLECOLORS"] = "true"
};
var portString = "";
if (hasPorts)
{
status.Ports = ports.Select(p => p.Port);
portString = string.Join(" ", ports.Select(p => $"-p {p.Port}:{p.Port}"));
// These ports should also be passed in not assuming ASP.NET Core
environment["ASPNETCORE_URLS"] = string.Join(";", ports.Select(p => $"{p.Protocol ?? "http"}://*:{p.Port}"));
foreach (var p in ports)
{
environment[$"{p.Protocol?.ToUpper() ?? "HTTP"}_PORT"] = p.BindingPort.ToString();
}
}
application.PopulateEnvironment(service, (key, value) => environment[key] = value, "host.docker.internal");
environment["APP_INSTANCE"] = replica;
foreach (var pair in environment)
{
environmentArguments += $"-e {pair.Key}={pair.Value} ";
}
var command = $"run -d {environmentArguments} {portString} --name {replica} --restart=unless-stopped {service.Description.DockerImage} {service.Description.Args ?? ""}";
_logger.LogInformation("Running docker command {Command}", command);
service.Logs.OnNext($"[{replica}]: {command}");
status.DockerCommand = command;
var result = ProcessUtil.Run("docker", command, throwOnError: false, cancellationToken: dockerInfo.StoppingTokenSource.Token);
if (result.ExitCode != 0)
{
_logger.LogError("docker run failed for {ServiceName} with exit code {ExitCode}:" + result.StandardError, service.Description.Name, result.ExitCode);
service.Replicas.TryRemove(replica, out _);
PrintStdOutAndErr(service, replica, result);
return;
}
var containerId = result.StandardOutput.Trim();
// There's a race condition that sometimes makes us miss the output
// so keep trying to get the container id
while (string.IsNullOrEmpty(containerId))
{
// Try to get the ID of the container
result = ProcessUtil.Run("docker", $"ps --no-trunc -f name={replica} --format " + "{{.ID}}");
containerId = result.ExitCode == 0 ? result.StandardOutput.Trim() : null;
}
var shortContainerId = containerId.Substring(0, Math.Min(12, containerId.Length));
status.ContainerId = shortContainerId;
_logger.LogInformation("Running container {ContainerName} with ID {ContainerId}", replica, shortContainerId);
_logger.LogInformation("Collecting docker logs for {ContainerName}.", replica);
ProcessUtil.Run("docker", $"logs -f {containerId}",
outputDataReceived: data => service.Logs.OnNext($"[{replica}]: {data}"),
onStart: pid =>
{
status.DockerLogsPid = pid;
},
throwOnError: false,
cancellationToken: dockerInfo.StoppingTokenSource.Token);
_logger.LogInformation("docker logs collection for {ContainerName} complete with exit code {ExitCode}", replica, result.ExitCode);
// Docker has a tendency to hang so we're going to timeout this shutdown process
var timeoutCts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
_logger.LogInformation("Stopping container {ContainerName} with ID {ContainerId}", replica, shortContainerId);
result = ProcessUtil.Run("docker", $"stop {containerId}", throwOnError: false, cancellationToken: timeoutCts.Token);
PrintStdOutAndErr(service, replica, result);
_logger.LogInformation("Stopped container {ContainerName} with ID {ContainerId} exited with {ExitCode}", replica, shortContainerId, result.ExitCode);
result = ProcessUtil.Run("docker", $"rm {containerId}", throwOnError: false, cancellationToken: timeoutCts.Token);
PrintStdOutAndErr(service, replica, result);
_logger.LogInformation("Removed container {ContainerName} with ID {ContainerId} exited with {ExitCode}", replica, shortContainerId, result.ExitCode);
service.Replicas.TryRemove(replica, out _);
};
if (serviceDescription.Bindings.Count > 0)
{
// Each replica is assigned a list of internal ports, one mapped to each external
// port
for (int i = 0; i < serviceDescription.Replicas; i++)
{
var ports = new List <(int, int, string)>();
foreach (var binding in serviceDescription.Bindings)
{
if (binding.Port == null)
{
continue;
}
ports.Add((service.PortMap[binding.Port.Value][i], binding.Port.Value, binding.Protocol));
}
dockerInfo.Threads[i] = new Thread(() => RunDockerContainer(ports));
}
}
else
{
for (int i = 0; i < service.Description.Replicas; i++)
{
dockerInfo.Threads[i] = new Thread(() => RunDockerContainer(Enumerable.Empty <(int, int, string)>()));
}
}
for (int i = 0; i < service.Description.Replicas; i++)
{
dockerInfo.Threads[i].Start();
}
service.Items[typeof(DockerInformation)] = dockerInfo;
return(Task.CompletedTask);
}
public static Task RunAsync(ILogger logger, Service service)
{
if (service.Description.DockerImage == null)
{
return(Task.CompletedTask);
}
var serviceDescription = service.Description;
var environmentArguments = "";
if (serviceDescription.Configuration != null)
{
foreach (var env in serviceDescription.Configuration)
{
environmentArguments += $"--env {env.Name}={env.Value} ";
}
}
var dockerInfo = new DockerInformation()
{
Threads = new Thread[service.Description.Replicas.Value]
};
void RunDockerContainer(Dictionary <int, int> ports)
{
var replica = service.Description.Name.ToLower() + "_" + Guid.NewGuid().ToString().Substring(0, 10).ToLower();
var status = new DockerStatus();
service.Replicas[replica] = status;
var hasPorts = ports?.Any() ?? false;
var portString = hasPorts ? string.Join(" ", ports.Select(p => $"-p {p.Value}:{p.Key}")) : "";
var command = $"run -d {environmentArguments} {portString} --name {replica} --restart=unless-stopped {service.Description.DockerImage}";
logger.LogInformation("Running docker command {Command}", command);
status.DockerCommand = command;
if (hasPorts)
{
status.Ports = ports.Values;
}
var result = ProcessUtil.Run("docker", command, throwOnError: false, cancellationToken: dockerInfo.StoppingTokenSource.Token);
if (result.ExitCode != 0)
{
logger.LogError("docker run failed for {ServiceName} with exit code {ExitCode}:" + result.StandardError, service.Description.Name, result.ExitCode);
service.Replicas.TryRemove(replica, out _);
return;
}
var containerId = result.StandardOutput.Trim();
// There's a race condition that sometimes makes us miss the output
// so keep trying to get the container id
while (string.IsNullOrEmpty(containerId))
{
// Try to get the ID of the container
result = ProcessUtil.Run("docker", $"ps --no-trunc -f name={replica} --format " + "{{.ID}}");
containerId = result.ExitCode == 0 ? result.StandardOutput.Trim() : null;
}
var shortContainerId = containerId.Substring(0, Math.Min(12, containerId.Length));
status.ContainerId = shortContainerId;
logger.LogInformation("Running container {ContainerName} with ID {ContainerId}", replica, shortContainerId);
logger.LogInformation("Collecting docker logs for {ContainerName}.", replica);
ProcessUtil.Run("docker", $"logs -f {containerId}",
outputDataReceived: service.Logs.OnNext,
onStart: pid =>
{
status.DockerLogsPid = pid;
},
throwOnError: false,
cancellationToken: dockerInfo.StoppingTokenSource.Token);
logger.LogInformation("docker logs collection for {ContainerName} complete with exit code {ExitCode}", replica, result.ExitCode);
// Docker has a tendency to hang so we're going to timeout this shutdown process
var timeoutCts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
logger.LogInformation("Stopping container {ContainerName} with ID {ContainerId}", replica, shortContainerId);
result = ProcessUtil.Run("docker", $"stop {containerId}", throwOnError: false, cancellationToken: timeoutCts.Token);
if (result.ExitCode != 0)
{
service.Logs.OnNext(result.StandardOutput);
service.Logs.OnNext(result.StandardError);
}
logger.LogInformation("Stopped container {ContainerName} with ID {ContainerId} exited with {ExitCode}", replica, shortContainerId, result.ExitCode);
result = ProcessUtil.Run("docker", $"rm {containerId}", throwOnError: false, cancellationToken: timeoutCts.Token);
if (result.ExitCode != 0)
{
service.Logs.OnNext(result.StandardOutput);
service.Logs.OnNext(result.StandardError);
}
logger.LogInformation("Removed container {ContainerName} with ID {ContainerId} exited with {ExitCode}", replica, shortContainerId, result.ExitCode);
};
if (serviceDescription.Bindings.Count > 0)
{
// Each replica is assigned a list of internal ports, one mapped to each external
// port
for (int i = 0; i < serviceDescription.Replicas; i++)
{
var ports = new Dictionary <int, int>();
foreach (var binding in serviceDescription.Bindings)
{
if (binding.Port == null)
{
continue;
}
ports[binding.Port.Value] = service.PortMap[binding.Port.Value][i];
}
dockerInfo.Threads[i] = new Thread(() => RunDockerContainer(ports));
}
}
else
{
for (int i = 0; i < service.Description.Replicas; i++)
{
dockerInfo.Threads[i] = new Thread(() => RunDockerContainer(null));
}
}
for (int i = 0; i < service.Description.Replicas; i++)
{
dockerInfo.Threads[i].Start();
}
service.Items[typeof(DockerInformation)] = dockerInfo;
return(Task.CompletedTask);
}
private async Task StartContainerAsync(Application application, Service service, DockerRunInfo docker, string?dockerNetwork)
{
var serviceDescription = service.Description;
var environmentArguments = "";
var volumes = "";
var workingDirectory = docker.WorkingDirectory != null ? $"-w {docker.WorkingDirectory}" : "";
var hostname = "host.docker.internal";
if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
{
// See: https://github.com/docker/for-linux/issues/264
//
// host.docker.internal is making it's way into linux docker but doesn't work yet
// instead we use the machine IP
var addresses = await Dns.GetHostAddressesAsync(Dns.GetHostName());
hostname = addresses[0].ToString();
}
// This is .NET specific
var userSecretStore = GetUserSecretsPathFromSecrets();
if (!string.IsNullOrEmpty(userSecretStore))
{
// Map the user secrets on this drive to user secrets
docker.VolumeMappings.Add(new DockerVolume(source: userSecretStore, name: null, target: "/root/.microsoft/usersecrets:ro"));
}
var dockerInfo = new DockerInformation(new Task[service.Description.Replicas]);
async Task RunDockerContainer(IEnumerable <(int ExternalPort, int Port, int?ContainerPort, string?Protocol)> ports)
{
var hasPorts = ports.Any();
var replica = service.Description.Name.ToLower() + "_" + Guid.NewGuid().ToString().Substring(0, 10).ToLower();
var status = new DockerStatus(service, replica);
service.Replicas[replica] = status;
service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Added, status));
var environment = new Dictionary <string, string>
{
// Default to development environment
["DOTNET_ENVIRONMENT"] = "Development",
["ASPNETCORE_ENVIRONMENT"] = "Development",
// Remove the color codes from the console output
["DOTNET_LOGGING__CONSOLE__DISABLECOLORS"] = "true",
["ASPNETCORE_LOGGING__CONSOLE__DISABLECOLORS"] = "true"
};
var portString = "";
if (hasPorts)
{
status.Ports = ports.Select(p => p.Port);
// These are the ports that the application should use for binding
// 1. Tell the docker container what port to bind to
portString = string.Join(" ", ports.Select(p => $"-p {p.Port}:{p.ContainerPort ?? p.Port}"));
// 2. Configure ASP.NET Core to bind to those same ports
environment["ASPNETCORE_URLS"] = string.Join(";", ports.Select(p => $"{p.Protocol ?? "http"}://*:{p.ContainerPort ?? p.Port}"));
// Set the HTTPS port for the redirect middleware
foreach (var p in ports)
{
if (string.Equals(p.Protocol, "https", StringComparison.OrdinalIgnoreCase))
{
// We need to set the redirect URL to the exposed port so the redirect works cleanly
environment["HTTPS_PORT"] = p.ExternalPort.ToString();
}
}
// 3. For non-ASP.NET Core apps, pass the same information in the PORT env variable as a semicolon separated list.
environment["PORT"] = string.Join(";", ports.Select(p => $"{p.ContainerPort ?? p.Port}"));
}
// See: https://github.com/docker/for-linux/issues/264
//
// The way we do proxying here doesn't really work for multi-container scenarios on linux
// without some more setup.
application.PopulateEnvironment(service, (key, value) => environment[key] = value, hostname);
environment["APP_INSTANCE"] = replica;
status.Environment = environment;
foreach (var pair in environment)
{
environmentArguments += $"-e {pair.Key}={pair.Value} ";
}
foreach (var volumeMapping in docker.VolumeMappings)
{
if (volumeMapping.Source != null)
{
var sourcePath = Path.GetFullPath(Path.Combine(application.ContextDirectory, volumeMapping.Source));
volumes += $"-v {sourcePath}:{volumeMapping.Target} ";
}
else if (volumeMapping.Name != null)
{
volumes += $"-v {volumeMapping.Name}:{volumeMapping.Target} ";
}
}
var command = $"run -d {workingDirectory} {volumes} {environmentArguments} {portString} --name {replica} --restart=unless-stopped {docker.Image} {docker.Args ?? ""}";
_logger.LogInformation("Running image {Image} for {Replica}", docker.Image, replica);
service.Logs.OnNext($"[{replica}]: docker {command}");
status.DockerCommand = command;
status.DockerNetwork = dockerNetwork;
WriteReplicaToStore(replica);
var result = await ProcessUtil.RunAsync(
"docker",
command,
throwOnError : false,
cancellationToken : dockerInfo.StoppingTokenSource.Token,
outputDataReceived : data => service.Logs.OnNext($"[{replica}]: {data}"),
errorDataReceived : data => service.Logs.OnNext($"[{replica}]: {data}"));
if (result.ExitCode != 0)
{
_logger.LogError("docker run failed for {ServiceName} with exit code {ExitCode}:" + result.StandardError, service.Description.Name, result.ExitCode);
service.Replicas.TryRemove(replica, out _);
service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Removed, status));
PrintStdOutAndErr(service, replica, result);
return;
}
var containerId = (string?)result.StandardOutput.Trim();
// There's a race condition that sometimes makes us miss the output
// so keep trying to get the container id
while (string.IsNullOrEmpty(containerId))
{
// Try to get the ID of the container
result = await ProcessUtil.RunAsync("docker", $"ps --no-trunc -f name={replica} --format " + "{{.ID}}");
containerId = result.ExitCode == 0 ? result.StandardOutput.Trim() : null;
}
var shortContainerId = containerId.Substring(0, Math.Min(12, containerId.Length));
status.ContainerId = shortContainerId;
_logger.LogInformation("Running container {ContainerName} with ID {ContainerId}", replica, shortContainerId);
if (!string.IsNullOrEmpty(dockerNetwork))
{
status.DockerNetworkAlias = serviceDescription.Name;
var networkCommand = $"network connect {dockerNetwork} {replica} --alias {serviceDescription.Name}";
service.Logs.OnNext($"[{replica}]: docker {networkCommand}");
_logger.LogInformation("Running docker command {Command}", networkCommand);
result = await ProcessUtil.RunAsync("docker", networkCommand);
PrintStdOutAndErr(service, replica, result);
}
service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Started, status));
_logger.LogInformation("Collecting docker logs for {ContainerName}.", replica);
while (!dockerInfo.StoppingTokenSource.Token.IsCancellationRequested)
{
var logsRes = await ProcessUtil.RunAsync("docker", $"logs -f {containerId}",
outputDataReceived : data => service.Logs.OnNext($"[{replica}]: {data}"),
errorDataReceived : data => service.Logs.OnNext($"[{replica}]: {data}"),
throwOnError : false,
cancellationToken : dockerInfo.StoppingTokenSource.Token);
if (logsRes.ExitCode != 0)
{
break;
}
if (!dockerInfo.StoppingTokenSource.IsCancellationRequested)
{
try
{
// Avoid spamming logs if restarts are happening
await Task.Delay(5000, dockerInfo.StoppingTokenSource.Token);
}
catch (OperationCanceledException)
{
break;
}
}
}
_logger.LogInformation("docker logs collection for {ContainerName} complete with exit code {ExitCode}", replica, result.ExitCode);
// Docker has a tendency to get stuck so we're going to timeout this shutdown process
var timeoutCts = new CancellationTokenSource(DockerStopTimeout);
_logger.LogInformation("Stopping container {ContainerName} with ID {ContainerId}", replica, shortContainerId);
result = await ProcessUtil.RunAsync("docker", $"stop {containerId}", throwOnError : false, cancellationToken : timeoutCts.Token);
if (timeoutCts.IsCancellationRequested)
{
_logger.LogWarning($"Failed to stop container after {DockerStopTimeout.Seconds} seconds, container will most likely be running.", replica, shortContainerId);
}
PrintStdOutAndErr(service, replica, result);
service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Stopped, status));
_logger.LogInformation("Stopped container {ContainerName} with ID {ContainerId} exited with {ExitCode}", replica, shortContainerId, result.ExitCode);
result = await ProcessUtil.RunAsync("docker", $"rm {containerId}", throwOnError : false, cancellationToken : timeoutCts.Token);
if (timeoutCts.IsCancellationRequested)
{
_logger.LogWarning($"Failed to remove container after {DockerStopTimeout.Seconds} seconds, container will most likely still exist.", replica, shortContainerId);
}
PrintStdOutAndErr(service, replica, result);
_logger.LogInformation("Removed container {ContainerName} with ID {ContainerId} exited with {ExitCode}", replica, shortContainerId, result.ExitCode);
service.Replicas.TryRemove(replica, out _);
service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Removed, status));
};
if (serviceDescription.Bindings.Count > 0)
{
// Each replica is assigned a list of internal ports, one mapped to each external
// port
for (var i = 0; i < serviceDescription.Replicas; i++)
{
var ports = new List <(int, int, int?, string?)>();
foreach (var binding in serviceDescription.Bindings)
{
if (binding.Port == null)
{
continue;
}
ports.Add((binding.Port.Value, binding.ReplicaPorts[i], binding.ContainerPort, binding.Protocol));
}
dockerInfo.Tasks[i] = RunDockerContainer(ports);
}
}
else
{
for (var i = 0; i < service.Description.Replicas; i++)
{
dockerInfo.Tasks[i] = RunDockerContainer(Enumerable.Empty <(int, int, int?, string?)>());
}
}
service.Items[typeof(DockerInformation)] = dockerInfo;
}
