private async Task StartContainerAsync(Tye.Hosting.Model.Application application, Tye.Hosting.Model.Service service, DockerRunInfo docker)
{
if (!await DockerDetector.Instance.IsDockerInstalled.Value)
{
_logger.LogError("Unable to start docker container for service {ServiceName}, Docker is not installed.", service.Description.Name);
service.Logs.OnNext($"Unable to start docker container for service {service.Description.Name}, Docker is not installed.");
return;
}
if (!await DockerDetector.Instance.IsDockerConnectedToDaemon.Value)
{
_logger.LogError("Unable to start docker container for service {ServiceName}, Docker is not running.", service.Description.Name);
service.Logs.OnNext($"Unable to start docker container for service {service.Description.Name}, Docker is not running.");
return;
}
var serviceDescription = service.Description;
var environmentArguments = "";
var volumes = "";
var workingDirectory = docker.WorkingDirectory != null ? $"-w {docker.WorkingDirectory}" : "";
// This is .NET specific
var userSecretStore = GetUserSecretsPathFromSecrets();
if (!string.IsNullOrEmpty(userSecretStore))
{
// Map the user secrets on this drive to user secrets
docker.VolumeMappings[userSecretStore] = "/root/.microsoft/usersecrets:ro";
}
var dockerInfo = new DockerInformation(new Task[service.Description.Replicas]);
async Task RunDockerContainer(IEnumerable <(int Port, int?InternalPort, int BindingPort, string?Protocol)> ports)
{
var hasPorts = ports.Any();
var replica = service.Description.Name.ToLower() + "_" + Guid.NewGuid().ToString().Substring(0, 10).ToLower();
var status = new DockerStatus(service, replica);
service.Replicas[replica] = status;
service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Added, status));
var environment = new Dictionary <string, string>
{
// Default to development environment
["DOTNET_ENVIRONMENT"] = "Development",
// Remove the color codes from the console output
["DOTNET_LOGGING__CONSOLE__DISABLECOLORS"] = "true"
};
var portString = "";
if (hasPorts)
{
status.Ports = ports.Select(p => p.Port);
// These are the ports that the application should use for binding
// 1. Tell the docker container what port to bind to
portString = string.Join(" ", ports.Select(p => $"-p {p.Port}:{p.InternalPort ?? p.Port}"));
// 2. Configure ASP.NET Core to bind to those same ports
environment["ASPNETCORE_URLS"] = string.Join(";", ports.Select(p => $"{p.Protocol ?? "http"}://*:{p.InternalPort ?? p.Port}"));
// Set the HTTPS port for the redirect middleware
foreach (var p in ports)
{
if (string.Equals(p.Protocol, "https", StringComparison.OrdinalIgnoreCase))
{
// We need to set the redirect URL to the exposed port so the redirect works cleanly
environment["HTTPS_PORT"] = p.BindingPort.ToString();
}
}
// 3. For non-ASP.NET Core apps, pass the same information in the PORT env variable as a semicolon separated list.
environment["PORT"] = string.Join(";", ports.Select(p => $"{p.InternalPort ?? p.Port}"));
}
// See: https://github.com/docker/for-linux/issues/264
//
// The way we do proxying here doesn't really work for multi-container scenarios on linux
// without some more setup.
application.PopulateEnvironment(service, (key, value) => environment[key] = value, "host.docker.internal");
environment["APP_INSTANCE"] = replica;
foreach (var pair in environment)
{
environmentArguments += $"-e {pair.Key}={pair.Value} ";
}
foreach (var pair in docker.VolumeMappings)
{
var sourcePath = Path.GetFullPath(Path.Combine(application.ContextDirectory, pair.Key.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar)));
volumes += $"-v {sourcePath}:{pair.Value} ";
}
var command = $"run -d {workingDirectory} {volumes} {environmentArguments} {portString} --name {replica} --restart=unless-stopped {docker.Image} {docker.Args ?? ""}";
_logger.LogInformation("Running docker command {Command}", command);
service.Logs.OnNext($"[{replica}]: {command}");
status.DockerCommand = command;
var result = await ProcessUtil.RunAsync(
"docker",
command,
throwOnError : false,
cancellationToken : dockerInfo.StoppingTokenSource.Token,
outputDataReceived : data => service.Logs.OnNext($"[{replica}]: {data}"));
if (result.ExitCode != 0)
{
_logger.LogError("docker run failed for {ServiceName} with exit code {ExitCode}:" + result.StandardError, service.Description.Name, result.ExitCode);
service.Replicas.TryRemove(replica, out _);
service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Removed, status));
PrintStdOutAndErr(service, replica, result);
return;
}
var containerId = (string?)result.StandardOutput.Trim();
// There's a race condition that sometimes makes us miss the output
// so keep trying to get the container id
while (string.IsNullOrEmpty(containerId))
{
// Try to get the ID of the container
result = await ProcessUtil.RunAsync("docker", $"ps --no-trunc -f name={replica} --format " + "{{.ID}}");
containerId = result.ExitCode == 0 ? result.StandardOutput.Trim() : null;
}
var shortContainerId = containerId.Substring(0, Math.Min(12, containerId.Length));
status.ContainerId = shortContainerId;
_logger.LogInformation("Running container {ContainerName} with ID {ContainerId}", replica, shortContainerId);
service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Started, status));
_logger.LogInformation("Collecting docker logs for {ContainerName}.", replica);
await ProcessUtil.RunAsync("docker", $"logs -f {containerId}",
outputDataReceived : data => service.Logs.OnNext($"[{replica}]: {data}"),
onStart : pid =>
{
status.DockerLogsPid = pid;
},
throwOnError : false,
cancellationToken : dockerInfo.StoppingTokenSource.Token);
_logger.LogInformation("docker logs collection for {ContainerName} complete with exit code {ExitCode}", replica, result.ExitCode);
// Docker has a tendency to get stuck so we're going to timeout this shutdown process
var timeoutCts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
_logger.LogInformation("Stopping container {ContainerName} with ID {ContainerId}", replica, shortContainerId);
result = await ProcessUtil.RunAsync("docker", $"stop {containerId}", throwOnError : false, cancellationToken : timeoutCts.Token);
PrintStdOutAndErr(service, replica, result);
service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Stopped, status));
_logger.LogInformation("Stopped container {ContainerName} with ID {ContainerId} exited with {ExitCode}", replica, shortContainerId, result.ExitCode);
result = await ProcessUtil.RunAsync("docker", $"rm {containerId}", throwOnError : false, cancellationToken : timeoutCts.Token);
PrintStdOutAndErr(service, replica, result);
_logger.LogInformation("Removed container {ContainerName} with ID {ContainerId} exited with {ExitCode}", replica, shortContainerId, result.ExitCode);
service.Replicas.TryRemove(replica, out _);
service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Removed, status));
};
if (serviceDescription.Bindings.Count > 0)
{
// Each replica is assigned a list of internal ports, one mapped to each external
// port
for (var i = 0; i < serviceDescription.Replicas; i++)
{
var ports = new List <(int, int?, int, string?)>();
foreach (var binding in serviceDescription.Bindings)
{
if (binding.Port == null)
{
continue;
}
ports.Add((service.PortMap[binding.Port.Value][i], binding.InternalPort, binding.Port.Value, binding.Protocol));
}
dockerInfo.Tasks[i] = RunDockerContainer(ports);
}
}
else
{
for (var i = 0; i < service.Description.Replicas; i++)
{
dockerInfo.Tasks[i] = RunDockerContainer(Enumerable.Empty <(int, int?, int, string?)>());
}
}
service.Items[typeof(DockerInformation)] = dockerInfo;
}
public static Task RunAsync(ILogger logger, Service service)
{
if (service.Description.DockerImage == null)
{
return(Task.CompletedTask);
}
var serviceDescription = service.Description;
var environmentArguments = "";
if (serviceDescription.Configuration != null)
{
foreach (var env in serviceDescription.Configuration)
{
environmentArguments += $"--env {env.Name}={env.Value} ";
}
}
var dockerInfo = new DockerInformation()
{
Threads = new Thread[service.Description.Replicas.Value]
};
void RunDockerContainer(Dictionary <int, int> ports)
{
var replica = service.Description.Name.ToLower() + "_" + Guid.NewGuid().ToString().Substring(0, 10).ToLower();
var status = new DockerStatus();
service.Replicas[replica] = status;
var hasPorts = ports?.Any() ?? false;
var portString = hasPorts ? string.Join(" ", ports.Select(p => $"-p {p.Value}:{p.Key}")) : "";
var command = $"run -d {environmentArguments} {portString} --name {replica} --restart=unless-stopped {service.Description.DockerImage}";
logger.LogInformation("Running docker command {Command}", command);
status.DockerCommand = command;
if (hasPorts)
{
status.Ports = ports.Values;
}
var result = ProcessUtil.Run("docker", command, throwOnError: false, cancellationToken: dockerInfo.StoppingTokenSource.Token);
if (result.ExitCode != 0)
{
logger.LogError("docker run failed for {ServiceName} with exit code {ExitCode}:" + result.StandardError, service.Description.Name, result.ExitCode);
service.Replicas.TryRemove(replica, out _);
return;
}
var containerId = result.StandardOutput.Trim();
// There's a race condition that sometimes makes us miss the output
// so keep trying to get the container id
while (string.IsNullOrEmpty(containerId))
{
// Try to get the ID of the container
result = ProcessUtil.Run("docker", $"ps --no-trunc -f name={replica} --format " + "{{.ID}}");
containerId = result.ExitCode == 0 ? result.StandardOutput.Trim() : null;
}
var shortContainerId = containerId.Substring(0, Math.Min(12, containerId.Length));
status.ContainerId = shortContainerId;
logger.LogInformation("Running container {ContainerName} with ID {ContainerId}", replica, shortContainerId);
logger.LogInformation("Collecting docker logs for {ContainerName}.", replica);
ProcessUtil.Run("docker", $"logs -f {containerId}",
outputDataReceived: service.Logs.OnNext,
onStart: pid =>
{
status.DockerLogsPid = pid;
},
throwOnError: false,
cancellationToken: dockerInfo.StoppingTokenSource.Token);
logger.LogInformation("docker logs collection for {ContainerName} complete with exit code {ExitCode}", replica, result.ExitCode);
// Docker has a tendency to hang so we're going to timeout this shutdown process
var timeoutCts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
logger.LogInformation("Stopping container {ContainerName} with ID {ContainerId}", replica, shortContainerId);
result = ProcessUtil.Run("docker", $"stop {containerId}", throwOnError: false, cancellationToken: timeoutCts.Token);
if (result.ExitCode != 0)
{
service.Logs.OnNext(result.StandardOutput);
service.Logs.OnNext(result.StandardError);
}
logger.LogInformation("Stopped container {ContainerName} with ID {ContainerId} exited with {ExitCode}", replica, shortContainerId, result.ExitCode);
result = ProcessUtil.Run("docker", $"rm {containerId}", throwOnError: false, cancellationToken: timeoutCts.Token);
if (result.ExitCode != 0)
{
service.Logs.OnNext(result.StandardOutput);
service.Logs.OnNext(result.StandardError);
}
logger.LogInformation("Removed container {ContainerName} with ID {ContainerId} exited with {ExitCode}", replica, shortContainerId, result.ExitCode);
};
if (serviceDescription.Bindings.Count > 0)
{
// Each replica is assigned a list of internal ports, one mapped to each external
// port
for (int i = 0; i < serviceDescription.Replicas; i++)
{
var ports = new Dictionary <int, int>();
foreach (var binding in serviceDescription.Bindings)
{
if (binding.Port == null)
{
continue;
}
ports[binding.Port.Value] = service.PortMap[binding.Port.Value][i];
}
dockerInfo.Threads[i] = new Thread(() => RunDockerContainer(ports));
}
}
else
{
for (int i = 0; i < service.Description.Replicas; i++)
{
dockerInfo.Threads[i] = new Thread(() => RunDockerContainer(null));
}
}
for (int i = 0; i < service.Description.Replicas; i++)
{
dockerInfo.Threads[i].Start();
}
service.Items[typeof(DockerInformation)] = dockerInfo;
return(Task.CompletedTask);
}
private Task StartContainerAsync(Application application, Service service)
{
if (service.Description.DockerImage == null)
{
return(Task.CompletedTask);
}
if (!_dockerInstalled.Value)
{
_logger.LogError("Unable to start docker container for service {ServiceName}, Docker is not installed.", service.Description.Name);
service.Logs.OnNext($"Unable to start docker container for service {service.Description.Name}, Docker is not installed.");
return(Task.CompletedTask);
}
var serviceDescription = service.Description;
var environmentArguments = "";
var dockerInfo = new DockerInformation()
{
Threads = new Thread[service.Description.Replicas.Value]
};
void RunDockerContainer(IEnumerable <(int Port, int BindingPort, string Protocol)> ports)
{
var hasPorts = ports.Any();
var replica = service.Description.Name.ToLower() + "_" + Guid.NewGuid().ToString().Substring(0, 10).ToLower();
var status = new DockerStatus();
service.Replicas[replica] = status;
var environment = new Dictionary <string, string>
{
// Default to development environment
["DOTNET_ENVIRONMENT"] = "Development",
// Remove the color codes from the console output
["DOTNET_LOGGING__CONSOLE__DISABLECOLORS"] = "true"
};
var portString = "";
if (hasPorts)
{
status.Ports = ports.Select(p => p.Port);
portString = string.Join(" ", ports.Select(p => $"-p {p.Port}:{p.Port}"));
// These ports should also be passed in not assuming ASP.NET Core
environment["ASPNETCORE_URLS"] = string.Join(";", ports.Select(p => $"{p.Protocol ?? "http"}://*:{p.Port}"));
foreach (var p in ports)
{
environment[$"{p.Protocol?.ToUpper() ?? "HTTP"}_PORT"] = p.BindingPort.ToString();
}
}
application.PopulateEnvironment(service, (key, value) => environment[key] = value, "host.docker.internal");
environment["APP_INSTANCE"] = replica;
foreach (var pair in environment)
{
environmentArguments += $"-e {pair.Key}={pair.Value} ";
}
var command = $"run -d {environmentArguments} {portString} --name {replica} --restart=unless-stopped {service.Description.DockerImage} {service.Description.Args ?? ""}";
_logger.LogInformation("Running docker command {Command}", command);
service.Logs.OnNext($"[{replica}]: {command}");
status.DockerCommand = command;
var result = ProcessUtil.Run("docker", command, throwOnError: false, cancellationToken: dockerInfo.StoppingTokenSource.Token);
if (result.ExitCode != 0)
{
_logger.LogError("docker run failed for {ServiceName} with exit code {ExitCode}:" + result.StandardError, service.Description.Name, result.ExitCode);
service.Replicas.TryRemove(replica, out _);
PrintStdOutAndErr(service, replica, result);
return;
}
var containerId = result.StandardOutput.Trim();
// There's a race condition that sometimes makes us miss the output
// so keep trying to get the container id
while (string.IsNullOrEmpty(containerId))
{
// Try to get the ID of the container
result = ProcessUtil.Run("docker", $"ps --no-trunc -f name={replica} --format " + "{{.ID}}");
containerId = result.ExitCode == 0 ? result.StandardOutput.Trim() : null;
}
var shortContainerId = containerId.Substring(0, Math.Min(12, containerId.Length));
status.ContainerId = shortContainerId;
_logger.LogInformation("Running container {ContainerName} with ID {ContainerId}", replica, shortContainerId);
_logger.LogInformation("Collecting docker logs for {ContainerName}.", replica);
ProcessUtil.Run("docker", $"logs -f {containerId}",
outputDataReceived: data => service.Logs.OnNext($"[{replica}]: {data}"),
onStart: pid =>
{
status.DockerLogsPid = pid;
},
throwOnError: false,
cancellationToken: dockerInfo.StoppingTokenSource.Token);
_logger.LogInformation("docker logs collection for {ContainerName} complete with exit code {ExitCode}", replica, result.ExitCode);
// Docker has a tendency to hang so we're going to timeout this shutdown process
var timeoutCts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
_logger.LogInformation("Stopping container {ContainerName} with ID {ContainerId}", replica, shortContainerId);
result = ProcessUtil.Run("docker", $"stop {containerId}", throwOnError: false, cancellationToken: timeoutCts.Token);
PrintStdOutAndErr(service, replica, result);
_logger.LogInformation("Stopped container {ContainerName} with ID {ContainerId} exited with {ExitCode}", replica, shortContainerId, result.ExitCode);
result = ProcessUtil.Run("docker", $"rm {containerId}", throwOnError: false, cancellationToken: timeoutCts.Token);
PrintStdOutAndErr(service, replica, result);
_logger.LogInformation("Removed container {ContainerName} with ID {ContainerId} exited with {ExitCode}", replica, shortContainerId, result.ExitCode);
service.Replicas.TryRemove(replica, out _);
};
if (serviceDescription.Bindings.Count > 0)
{
// Each replica is assigned a list of internal ports, one mapped to each external
// port
for (int i = 0; i < serviceDescription.Replicas; i++)
{
var ports = new List <(int, int, string)>();
foreach (var binding in serviceDescription.Bindings)
{
if (binding.Port == null)
{
continue;
}
ports.Add((service.PortMap[binding.Port.Value][i], binding.Port.Value, binding.Protocol));
}
dockerInfo.Threads[i] = new Thread(() => RunDockerContainer(ports));
}
}
else
{
for (int i = 0; i < service.Description.Replicas; i++)
{
dockerInfo.Threads[i] = new Thread(() => RunDockerContainer(Enumerable.Empty <(int, int, string)>()));
}
}
for (int i = 0; i < service.Description.Replicas; i++)
{
dockerInfo.Threads[i].Start();
}
service.Items[typeof(DockerInformation)] = dockerInfo;
return(Task.CompletedTask);
}
private async Task StartContainerAsync(Application application, Service service, DockerRunInfo docker, string?dockerNetwork)
{
var serviceDescription = service.Description;
var environmentArguments = "";
var volumes = "";
var workingDirectory = docker.WorkingDirectory != null ? $"-w {docker.WorkingDirectory}" : "";
var hostname = "host.docker.internal";
var dockerImage = docker.Image ?? service.Description.Name;
if (docker.DockerFile != null)
{
var dockerBuildResult = await ProcessUtil.RunAsync(
$"docker",
$"build \"{docker.DockerFileContext?.FullName}\" -t {dockerImage} -f \"{docker.DockerFile}\"",
docker.WorkingDirectory,
throwOnError : false);
if (dockerBuildResult.ExitCode != 0)
{
_logger.LogInformation("Running docker command with exception info {ExceptionStdOut} {ExceptionStdErr}", dockerBuildResult.StandardOutput, dockerBuildResult.StandardError);
throw new CommandException("'docker build' failed.");
}
}
if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
{
// See: https://github.com/docker/for-linux/issues/264
//
// host.docker.internal is making it's way into linux docker but doesn't work yet
// instead we use the machine IP
var addresses = await Dns.GetHostAddressesAsync(Dns.GetHostName());
hostname = addresses[0].ToString();
}
var dockerInfo = new DockerInformation(new Task[service.Description.Replicas]);
async Task RunDockerContainer(IEnumerable <(int ExternalPort, int Port, int?ContainerPort, string?Protocol)> ports)
{
var hasPorts = ports.Any();
var replica = service.Description.Name.ToLower() + "_" + Guid.NewGuid().ToString().Substring(0, 10).ToLower();
var status = new DockerStatus(service, replica);
service.Replicas[replica] = status;
service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Added, status));
var environment = new Dictionary <string, string>();
var portString = "";
if (hasPorts)
{
status.Ports = ports.Select(p => p.Port);
// These are the ports that the application should use for binding
// 1. Tell the docker container what port to bind to
portString = docker.Private ? "" : string.Join(" ", ports.Select(p => $"-p {p.Port}:{p.ContainerPort ?? p.Port}"));
if (docker.IsAspNet)
{
// 2. Configure ASP.NET Core to bind to those same ports
environment["ASPNETCORE_URLS"] = string.Join(";", ports.Select(p => $"{p.Protocol ?? "http"}://*:{p.ContainerPort ?? p.Port}"));
// Set the HTTPS port for the redirect middleware
foreach (var p in ports)
{
if (string.Equals(p.Protocol, "https", StringComparison.OrdinalIgnoreCase))
{
// We need to set the redirect URL to the exposed port so the redirect works cleanly
environment["HTTPS_PORT"] = p.ExternalPort.ToString();
}
}
}
// 3. For non-ASP.NET Core apps, pass the same information in the PORT env variable as a semicolon separated list.
environment["PORT"] = string.Join(";", ports.Select(p => $"{p.ContainerPort ?? p.Port}"));
// This the port for the container proxy (containerport:externalport)
environment["PROXY_PORT"] = string.Join(";", ports.Select(p => $"{p.ContainerPort ?? p.Port}:{p.ExternalPort}"));
}
// See: https://github.com/docker/for-linux/issues/264
//
// The way we do proxying here doesn't really work for multi-container scenarios on linux
// without some more setup.
application.PopulateEnvironment(service, (key, value) => environment[key] = value, hostname);
environment["APP_INSTANCE"] = replica;
environment["CONTAINER_HOST"] = hostname;
status.Environment = environment;
foreach (var pair in environment)
{
environmentArguments += $"-e \"{pair.Key}={pair.Value}\" ";
}
foreach (var volumeMapping in docker.VolumeMappings)
{
if (volumeMapping.Source != null)
{
var sourcePath = Path.GetFullPath(Path.Combine(application.ContextDirectory, volumeMapping.Source));
volumes += $"-v {sourcePath}:{volumeMapping.Target} ";
}
else if (volumeMapping.Name != null)
{
volumes += $"-v {volumeMapping.Name}:{volumeMapping.Target} ";
}
}
var command = $"run -d {workingDirectory} {volumes} {environmentArguments} {portString} --name {replica} --restart=unless-stopped {dockerImage} {docker.Args ?? ""}";
_logger.LogInformation("Running image {Image} for {Replica}", docker.Image, replica);
service.Logs.OnNext($"[{replica}]: docker {command}");
status.DockerCommand = command;
status.DockerNetwork = dockerNetwork;
WriteReplicaToStore(replica);
var result = await ProcessUtil.RunAsync(
"docker",
command,
throwOnError : false,
cancellationToken : dockerInfo.StoppingTokenSource.Token,
outputDataReceived : data => service.Logs.OnNext($"[{replica}]: {data}"),
errorDataReceived : data => service.Logs.OnNext($"[{replica}]: {data}"));
if (result.ExitCode != 0)
{
_logger.LogError("docker run failed for {ServiceName} with exit code {ExitCode}:" + result.StandardError, service.Description.Name, result.ExitCode);
service.Replicas.TryRemove(replica, out _);
service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Removed, status));
PrintStdOutAndErr(service, replica, result);
return;
}
var containerId = (string?)result.StandardOutput.Trim();
// There's a race condition that sometimes makes us miss the output
// so keep trying to get the container id
while (string.IsNullOrEmpty(containerId))
{
// Try to get the ID of the container
result = await ProcessUtil.RunAsync("docker", $"ps --no-trunc -f name={replica} --format " + "{{.ID}}");
containerId = result.ExitCode == 0 ? result.StandardOutput.Trim() : null;
}
var shortContainerId = containerId.Substring(0, Math.Min(12, containerId.Length));
status.ContainerId = shortContainerId;
_logger.LogInformation("Running container {ContainerName} with ID {ContainerId}", replica, shortContainerId);
if (!string.IsNullOrEmpty(dockerNetwork))
{
status.DockerNetworkAlias = docker.NetworkAlias ?? serviceDescription.Name;
var networkCommand = $"network connect {dockerNetwork} {replica} --alias {status.DockerNetworkAlias}";
service.Logs.OnNext($"[{replica}]: docker {networkCommand}");
_logger.LogInformation("Running docker command {Command}", networkCommand);
result = await ProcessUtil.RunAsync("docker", networkCommand);
PrintStdOutAndErr(service, replica, result);
}
service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Started, status));
_logger.LogInformation("Collecting docker logs for {ContainerName}.", replica);
var backOff = TimeSpan.FromSeconds(5);
while (!dockerInfo.StoppingTokenSource.Token.IsCancellationRequested)
{
var logsRes = await ProcessUtil.RunAsync("docker", $"logs -f {containerId}",
outputDataReceived : data => service.Logs.OnNext($"[{replica}]: {data}"),
errorDataReceived : data => service.Logs.OnNext($"[{replica}]: {data}"),
throwOnError : false,
cancellationToken : dockerInfo.StoppingTokenSource.Token);
if (logsRes.ExitCode != 0)
{
break;
}
if (!dockerInfo.StoppingTokenSource.IsCancellationRequested)
{
try
{
// Avoid spamming logs if restarts are happening
await Task.Delay(backOff, dockerInfo.StoppingTokenSource.Token);
}
catch (OperationCanceledException)
{
break;
}
}
backOff *= 2;
}
_logger.LogInformation("docker logs collection for {ContainerName} complete with exit code {ExitCode}", replica, result.ExitCode);
// Docker has a tendency to get stuck so we're going to timeout this shutdown process
var timeoutCts = new CancellationTokenSource(DockerStopTimeout);
_logger.LogInformation("Stopping container {ContainerName} with ID {ContainerId}", replica, shortContainerId);
result = await ProcessUtil.RunAsync("docker", $"stop {containerId}", throwOnError : false, cancellationToken : timeoutCts.Token);
if (timeoutCts.IsCancellationRequested)
{
_logger.LogWarning($"Failed to stop container after {DockerStopTimeout.Seconds} seconds, container will most likely be running.", replica, shortContainerId);
}
PrintStdOutAndErr(service, replica, result);
service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Stopped, status));
_logger.LogInformation("Stopped container {ContainerName} with ID {ContainerId} exited with {ExitCode}", replica, shortContainerId, result.ExitCode);
result = await ProcessUtil.RunAsync("docker", $"rm {containerId}", throwOnError : false, cancellationToken : timeoutCts.Token);
if (timeoutCts.IsCancellationRequested)
{
_logger.LogWarning($"Failed to remove container after {DockerStopTimeout.Seconds} seconds, container will most likely still exist.", replica, shortContainerId);
}
PrintStdOutAndErr(service, replica, result);
_logger.LogInformation("Removed container {ContainerName} with ID {ContainerId} exited with {ExitCode}", replica, shortContainerId, result.ExitCode);
service.Replicas.TryRemove(replica, out _);
service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Removed, status));
};
if (serviceDescription.Bindings.Count > 0)
{
// Each replica is assigned a list of internal ports, one mapped to each external
// port
for (var i = 0; i < serviceDescription.Replicas; i++)
{
var ports = new List <(int, int, int?, string?)>();
foreach (var binding in serviceDescription.Bindings)
{
if (binding.Port == null)
{
continue;
}
ports.Add((binding.Port.Value, binding.ReplicaPorts[i], binding.ContainerPort, binding.Protocol));
}
dockerInfo.Tasks[i] = RunDockerContainer(ports);
}
}
else
{
for (var i = 0; i < service.Description.Replicas; i++)
{
dockerInfo.Tasks[i] = RunDockerContainer(Enumerable.Empty <(int, int, int?, string?)>());
}
}
service.Items[typeof(DockerInformation)] = dockerInfo;
}
