private Task StartContainerAsync(Application application, Service service)
{
if (service.Description.DockerImage == null)
{
return(Task.CompletedTask);
}
if (!_dockerInstalled.Value)
{
_logger.LogError("Unable to start docker container for service {ServiceName}, Docker is not installed.", service.Description.Name);
service.Logs.OnNext($"Unable to start docker container for service {service.Description.Name}, Docker is not installed.");
return(Task.CompletedTask);
}
var serviceDescription = service.Description;
var environmentArguments = "";
var dockerInfo = new DockerInformation()
{
Threads = new Thread[service.Description.Replicas.Value]
};
void RunDockerContainer(IEnumerable <(int Port, int BindingPort, string Protocol)> ports)
{
var hasPorts = ports.Any();
var replica = service.Description.Name.ToLower() + "_" + Guid.NewGuid().ToString().Substring(0, 10).ToLower();
var status = new DockerStatus();
service.Replicas[replica] = status;
var environment = new Dictionary <string, string>
{
// Default to development environment
["DOTNET_ENVIRONMENT"] = "Development",
// Remove the color codes from the console output
["DOTNET_LOGGING__CONSOLE__DISABLECOLORS"] = "true"
};
var portString = "";
if (hasPorts)
{
status.Ports = ports.Select(p => p.Port);
portString = string.Join(" ", ports.Select(p => $"-p {p.Port}:{p.Port}"));
// These ports should also be passed in not assuming ASP.NET Core
environment["ASPNETCORE_URLS"] = string.Join(";", ports.Select(p => $"{p.Protocol ?? "http"}://*:{p.Port}"));
foreach (var p in ports)
{
environment[$"{p.Protocol?.ToUpper() ?? "HTTP"}_PORT"] = p.BindingPort.ToString();
}
}
application.PopulateEnvironment(service, (key, value) => environment[key] = value, "host.docker.internal");
environment["APP_INSTANCE"] = replica;
foreach (var pair in environment)
{
environmentArguments += $"-e {pair.Key}={pair.Value} ";
}
var command = $"run -d {environmentArguments} {portString} --name {replica} --restart=unless-stopped {service.Description.DockerImage} {service.Description.Args ?? ""}";
_logger.LogInformation("Running docker command {Command}", command);
service.Logs.OnNext($"[{replica}]: {command}");
status.DockerCommand = command;
var result = ProcessUtil.Run("docker", command, throwOnError: false, cancellationToken: dockerInfo.StoppingTokenSource.Token);
if (result.ExitCode != 0)
{
_logger.LogError("docker run failed for {ServiceName} with exit code {ExitCode}:" + result.StandardError, service.Description.Name, result.ExitCode);
service.Replicas.TryRemove(replica, out _);
PrintStdOutAndErr(service, replica, result);
return;
}
var containerId = result.StandardOutput.Trim();
// There's a race condition that sometimes makes us miss the output
// so keep trying to get the container id
while (string.IsNullOrEmpty(containerId))
{
// Try to get the ID of the container
result = ProcessUtil.Run("docker", $"ps --no-trunc -f name={replica} --format " + "{{.ID}}");
containerId = result.ExitCode == 0 ? result.StandardOutput.Trim() : null;
}
var shortContainerId = containerId.Substring(0, Math.Min(12, containerId.Length));
status.ContainerId = shortContainerId;
_logger.LogInformation("Running container {ContainerName} with ID {ContainerId}", replica, shortContainerId);
_logger.LogInformation("Collecting docker logs for {ContainerName}.", replica);
ProcessUtil.Run("docker", $"logs -f {containerId}",
outputDataReceived: data => service.Logs.OnNext($"[{replica}]: {data}"),
onStart: pid =>
{
status.DockerLogsPid = pid;
},
throwOnError: false,
cancellationToken: dockerInfo.StoppingTokenSource.Token);
_logger.LogInformation("docker logs collection for {ContainerName} complete with exit code {ExitCode}", replica, result.ExitCode);
// Docker has a tendency to hang so we're going to timeout this shutdown process
var timeoutCts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
_logger.LogInformation("Stopping container {ContainerName} with ID {ContainerId}", replica, shortContainerId);
result = ProcessUtil.Run("docker", $"stop {containerId}", throwOnError: false, cancellationToken: timeoutCts.Token);
PrintStdOutAndErr(service, replica, result);
_logger.LogInformation("Stopped container {ContainerName} with ID {ContainerId} exited with {ExitCode}", replica, shortContainerId, result.ExitCode);
result = ProcessUtil.Run("docker", $"rm {containerId}", throwOnError: false, cancellationToken: timeoutCts.Token);
PrintStdOutAndErr(service, replica, result);
_logger.LogInformation("Removed container {ContainerName} with ID {ContainerId} exited with {ExitCode}", replica, shortContainerId, result.ExitCode);
service.Replicas.TryRemove(replica, out _);
};
if (serviceDescription.Bindings.Count > 0)
{
// Each replica is assigned a list of internal ports, one mapped to each external
// port
for (int i = 0; i < serviceDescription.Replicas; i++)
{
var ports = new List <(int, int, string)>();
foreach (var binding in serviceDescription.Bindings)
{
if (binding.Port == null)
{
continue;
}
ports.Add((service.PortMap[binding.Port.Value][i], binding.Port.Value, binding.Protocol));
}
dockerInfo.Threads[i] = new Thread(() => RunDockerContainer(ports));
}
}
else
{
for (int i = 0; i < service.Description.Replicas; i++)
{
dockerInfo.Threads[i] = new Thread(() => RunDockerContainer(Enumerable.Empty <(int, int, string)>()));
}
}
for (int i = 0; i < service.Description.Replicas; i++)
{
dockerInfo.Threads[i].Start();
}
service.Items[typeof(DockerInformation)] = dockerInfo;
return(Task.CompletedTask);
}
private Task LaunchService(Application application, Service service)
{
var serviceDescription = service.Description;
if (serviceDescription.DockerImage != null)
{
return(Docker.RunAsync(_logger, service));
}
var serviceName = serviceDescription.Name;
var path = "";
var workingDirectory = "";
var args = service.Description.Args ?? "";
if (serviceDescription.Project != null)
{
var fullProjectPath = Path.GetFullPath(Path.Combine(application.ContextDirectory, serviceDescription.Project));
path = GetExePath(fullProjectPath);
workingDirectory = Path.GetDirectoryName(fullProjectPath);
service.Status["projectFilePath"] = fullProjectPath;
}
else
{
path = Path.GetFullPath(Path.Combine(application.ContextDirectory, serviceDescription.Executable));
workingDirectory = serviceDescription.WorkingDirectory != null?
Path.GetFullPath(Path.Combine(application.ContextDirectory, serviceDescription.WorkingDirectory)) :
Path.GetDirectoryName(path);
// If this is a dll then use dotnet to run it
if (Path.GetExtension(path) == ".dll")
{
args = $"\"{path}\" {args}".Trim();
path = "dotnet";
}
}
service.Status["executablePath"] = path;
service.Status["workingDirectory"] = workingDirectory;
service.Status["args"] = args;
service.Status["debugMode"] = _debugMode;
var processInfo = new ProcessInfo
{
Threads = new Thread[service.Description.Replicas.Value]
};
void RunApplication(IEnumerable <(int Port, string Protocol)> ports)
{
var hasPorts = ports.Any();
var restarts = 0;
var environment = new Dictionary <string, string>
{
// Default to development environment
["DOTNET_ENVIRONMENT"] = "Development"
};
application.PopulateEnvironment(service, (k, v) => environment[k] = v);
if (_debugMode)
{
environment["DOTNET_STARTUP_HOOKS"] = typeof(Hosting.Runtime.HostingRuntimeHelpers).Assembly.Location;
}
if (hasPorts)
{
// These ports should also be passed in not assuming ASP.NET Core
environment["ASPNETCORE_URLS"] = string.Join(";", ports.Select(p => $"{p.Protocol ?? "http"}://localhost:{p.Port}"));
}
while (!processInfo.StoppedTokenSource.IsCancellationRequested)
{
var replica = serviceName + "_" + Guid.NewGuid().ToString().Substring(0, 10).ToLower();
var status = service.Replicas[replica] = new ServiceReplica();
// This isn't your host name
environment["APP_INSTANCE"] = replica;
status["exitCode"] = null;
status["pid"] = null;
status["env"] = environment;
if (hasPorts)
{
status["ports"] = ports;
}
service.Status["restarts"] = restarts;
_logger.LogInformation("Launching service {ServiceName} from {ExePath} {args}", replica, path, args);
var metricsTokenSource = CancellationTokenSource.CreateLinkedTokenSource(processInfo.StoppedTokenSource.Token);
var metricsThread = new Thread(state => CollectMetrics((int)state, replica, status, metricsTokenSource.Token));
try
{
var result = ProcessUtil.Run(path, args,
environmentVariables: environment,
workingDirectory: workingDirectory,
outputDataReceived: data =>
{
if (data == null)
{
return;
}
service.Logs.Add("[" + replica + "]: " + data);
},
onStart: pid =>
{
if (hasPorts)
{
_logger.LogInformation("{ServiceName} running on process id {PID} bound to {Address}", replica, pid, string.Join(", ", ports.Select(p => $"{p.Protocol ?? "http"}://localhost:{p.Port}")));
}
else
{
_logger.LogInformation("{ServiceName} running on process id {PID}", replica, pid);
}
status["pid"] = pid;
metricsThread.Start(pid);
},
throwOnError: false,
cancellationToken: processInfo.StoppedTokenSource.Token);
status["exitCode"] = result.ExitCode;
if (status["pid"] != null)
{
metricsTokenSource.Cancel();
metricsThread.Join();
}
}
catch (Exception ex)
{
_logger.LogError(0, ex, "Failed to launch process for service {ServiceName}", replica);
Thread.Sleep(5000);
}
restarts++;
if (status["exitCode"] != null)
{
_logger.LogInformation("{ServiceName} process exited with exit code {ExitCode}", replica, status["exitCode"]);
}
// Remove the replica from the set
service.Replicas.TryRemove(replica, out _);
}
}
if (serviceDescription.Bindings.Count > 0)
{
// Each replica is assigned a list of internal ports, one mapped to each external
// port
for (int i = 0; i < serviceDescription.Replicas; i++)
{
var ports = new List <(int, string)>();
foreach (var binding in serviceDescription.Bindings)
{
if (binding.Port == null)
{
continue;
}
ports.Add((service.PortMap[binding.Port.Value][i], binding.Protocol));
}
processInfo.Threads[i] = new Thread(() => RunApplication(ports));
}
}
else
{
for (int i = 0; i < service.Description.Replicas; i++)
{
processInfo.Threads[i] = new Thread(() => RunApplication(Enumerable.Empty <(int, string)>()));
}
}
for (int i = 0; i < service.Description.Replicas; i++)
{
processInfo.Threads[i].Start();
}
service.Items[typeof(ProcessInfo)] = processInfo;
return(Task.CompletedTask);
}
private async Task LaunchService(Application application, Service service)
{
var serviceDescription = service.Description;
if (serviceDescription.DockerImage != null)
{
return;
}
var serviceName = serviceDescription.Name;
var path = "";
var workingDirectory = "";
var args = service.Description.Args ?? "";
if (serviceDescription.Project != null)
{
var expandedProject = Environment.ExpandEnvironmentVariables(serviceDescription.Project);
var fullProjectPath = Path.GetFullPath(Path.Combine(application.ContextDirectory, expandedProject));
path = GetExePath(fullProjectPath);
workingDirectory = Path.GetDirectoryName(fullProjectPath);
service.Status.ProjectFilePath = fullProjectPath;
}
else
{
var expandedExecutable = Environment.ExpandEnvironmentVariables(serviceDescription.Executable);
path = Path.GetFullPath(Path.Combine(application.ContextDirectory, expandedExecutable));
workingDirectory = serviceDescription.WorkingDirectory != null?
Path.GetFullPath(Path.Combine(application.ContextDirectory, Environment.ExpandEnvironmentVariables(serviceDescription.WorkingDirectory))) :
Path.GetDirectoryName(path);
}
// If this is a dll then use dotnet to run it
if (Path.GetExtension(path) == ".dll")
{
args = $"\"{path}\" {args}".Trim();
path = "dotnet";
}
service.Status.ExecutablePath = path;
service.Status.WorkingDirectory = workingDirectory;
service.Status.Args = args;
var processInfo = new ProcessInfo
{
Tasks = new Task[service.Description.Replicas.Value]
};
if (service.Status.ProjectFilePath != null && service.Description.Build.GetValueOrDefault() && _buildProjects)
{
// Sometimes building can fail because of file locking (like files being open in VS)
_logger.LogInformation("Building project {ProjectFile}", service.Status.ProjectFilePath);
service.Logs.OnNext($"dotnet build \"{service.Status.ProjectFilePath}\" /nologo");
var buildResult = await ProcessUtil.RunAsync("dotnet", $"build \"{service.Status.ProjectFilePath}\" /nologo",
outputDataReceived : data => service.Logs.OnNext(data),
throwOnError : false);
if (buildResult.ExitCode != 0)
{
_logger.LogInformation("Building {ProjectFile} failed with exit code {ExitCode}: " + buildResult.StandardOutput + buildResult.StandardError, service.Status.ProjectFilePath, buildResult.ExitCode);
return;
}
}
async Task RunApplicationAsync(IEnumerable <(int Port, int BindingPort, string Protocol)> ports)
{
var hasPorts = ports.Any();
var environment = new Dictionary <string, string>
{
// Default to development environment
["DOTNET_ENVIRONMENT"] = "Development"
};
application.PopulateEnvironment(service, (k, v) => environment[k] = v);
if (_debugMode)
{
environment["DOTNET_STARTUP_HOOKS"] = typeof(Hosting.Runtime.HostingRuntimeHelpers).Assembly.Location;
}
if (hasPorts)
{
// These ports should also be passed in not assuming ASP.NET Core
environment["ASPNETCORE_URLS"] = string.Join(";", ports.Select(p => $"{p.Protocol ?? "http"}://localhost:{p.Port}"));
foreach (var p in ports)
{
environment[$"{p.Protocol?.ToUpper() ?? "HTTP"}_PORT"] = p.BindingPort.ToString();
}
}
while (!processInfo.StoppedTokenSource.IsCancellationRequested)
{
var replica = serviceName + "_" + Guid.NewGuid().ToString().Substring(0, 10).ToLower();
var status = new ProcessStatus(service, replica);
service.Replicas[replica] = status;
service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Added, status));
// This isn't your host name
environment["APP_INSTANCE"] = replica;
status.ExitCode = null;
status.Pid = null;
status.Environment = environment;
if (hasPorts)
{
status.Ports = ports.Select(p => p.Port);
}
_logger.LogInformation("Launching service {ServiceName}: {ExePath} {args}", replica, path, args);
try
{
service.Logs.OnNext($"[{replica}]:{path} {args}");
var result = await ProcessUtil.RunAsync(path, args,
environmentVariables : environment,
workingDirectory : workingDirectory,
outputDataReceived : data => service.Logs.OnNext($"[{replica}]: {data}"),
onStart : pid =>
{
if (hasPorts)
{
_logger.LogInformation("{ServiceName} running on process id {PID} bound to {Address}", replica, pid, string.Join(", ", ports.Select(p => $"{p.Protocol ?? "http"}://localhost:{p.Port}")));
}
else
{
_logger.LogInformation("{ServiceName} running on process id {PID}", replica, pid);
}
status.Pid = pid;
service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Started, status));
},
throwOnError : false,
cancellationToken : processInfo.StoppedTokenSource.Token);
status.ExitCode = result.ExitCode;
if (status.Pid != null)
{
service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Stopped, status));
}
}
catch (Exception ex)
{
_logger.LogError(0, ex, "Failed to launch process for service {ServiceName}", replica);
Thread.Sleep(5000);
}
service.Restarts++;
if (status.ExitCode != null)
{
_logger.LogInformation("{ServiceName} process exited with exit code {ExitCode}", replica, status.ExitCode);
}
// Remove the replica from the set
service.Replicas.TryRemove(replica, out _);
service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Removed, status));
}
}
if (serviceDescription.Bindings.Count > 0)
{
// Each replica is assigned a list of internal ports, one mapped to each external
// port
for (int i = 0; i < serviceDescription.Replicas; i++)
{
var ports = new List <(int, int, string)>();
foreach (var binding in serviceDescription.Bindings)
{
if (binding.Port == null)
{
continue;
}
ports.Add((service.PortMap[binding.Port.Value][i], binding.Port.Value, binding.Protocol));
}
processInfo.Tasks[i] = RunApplicationAsync(ports);
}
}
else
{
for (int i = 0; i < service.Description.Replicas; i++)
{
processInfo.Tasks[i] = RunApplicationAsync(Enumerable.Empty <(int, int, string)>());
}
}
service.Items[typeof(ProcessInfo)] = processInfo;
}
private Task LaunchService(Application application, Service service)
{
var serviceDescription = service.Description;
if (serviceDescription.DockerImage != null)
{
return(Docker.RunAsync(_logger, service));
}
var serviceName = serviceDescription.Name;
var path = "";
var workingDirectory = "";
var args = service.Description.Args ?? "";
var applicationName = "";
if (serviceDescription.Project != null)
{
var fullProjectPath = Path.GetFullPath(Path.Combine(application.ContextDirectory, serviceDescription.Project));
path = GetExePath(fullProjectPath);
workingDirectory = Path.GetDirectoryName(fullProjectPath);
// TODO: Requires msbuild
applicationName = Path.GetFileNameWithoutExtension(fullProjectPath);
service.Status.ProjectFilePath = fullProjectPath;
}
else
{
applicationName = Path.GetFileNameWithoutExtension(serviceDescription.Executable);
path = Path.GetFullPath(Path.Combine(application.ContextDirectory, serviceDescription.Executable));
workingDirectory = serviceDescription.WorkingDirectory != null?
Path.GetFullPath(Path.Combine(application.ContextDirectory, serviceDescription.WorkingDirectory)) :
Path.GetDirectoryName(path);
}
// If this is a dll then use dotnet to run it
if (Path.GetExtension(path) == ".dll")
{
applicationName = Path.GetFileNameWithoutExtension(path);
args = $"\"{path}\" {args}".Trim();
path = "dotnet";
}
service.Status.ExecutablePath = path;
service.Status.WorkingDirectory = workingDirectory;
service.Status.Args = args;
var processInfo = new ProcessInfo
{
Threads = new Thread[service.Description.Replicas.Value]
};
if (service.Status.ProjectFilePath != null && service.Description.Build && _buildProjects)
{
_logger.LogInformation("Building project {ProjectFile}", service.Status.ProjectFilePath);
service.Logs.OnNext("======================BUILDING====================");
var buildResult = ProcessUtil.Run("dotnet", $"build \"{service.Status.ProjectFilePath}\" /nologo",
outputDataReceived: data => service.Logs.OnNext(data),
throwOnError: false);
service.Logs.OnNext("");
if (buildResult.ExitCode != 0)
{
_logger.LogInformation("Building {ProjectFile} failed with exit code {ExitCode}: " + buildResult.StandardError, service.Status.ProjectFilePath, buildResult.ExitCode);
return(Task.CompletedTask);
}
}
void RunApplication(IEnumerable <(int Port, int BindingPort, string Protocol)> ports)
{
var hasPorts = ports.Any();
var environment = new Dictionary <string, string>
{
// Default to development environment
["DOTNET_ENVIRONMENT"] = "Development"
};
application.PopulateEnvironment(service, (k, v) => environment[k] = v);
if (_debugMode)
{
environment["DOTNET_STARTUP_HOOKS"] = typeof(Hosting.Runtime.HostingRuntimeHelpers).Assembly.Location;
}
if (hasPorts)
{
// These ports should also be passed in not assuming ASP.NET Core
environment["ASPNETCORE_URLS"] = string.Join(";", ports.Select(p => $"{p.Protocol ?? "http"}://localhost:{p.Port}"));
foreach (var p in ports)
{
environment[$"{p.Protocol?.ToUpper() ?? "HTTP"}_PORT"] = p.BindingPort.ToString();
}
}
while (!processInfo.StoppedTokenSource.IsCancellationRequested)
{
var replica = serviceName + "_" + Guid.NewGuid().ToString().Substring(0, 10).ToLower();
var status = new ProcessStatus();
service.Replicas[replica] = status;
// This isn't your host name
environment["APP_INSTANCE"] = replica;
status.ExitCode = null;
status.Pid = null;
status.Environment = environment;
if (hasPorts)
{
status.Ports = ports.Select(p => p.Port);
}
_logger.LogInformation("Launching service {ServiceName}: {ExePath} {args}", replica, path, args);
var metricsTokenSource = CancellationTokenSource.CreateLinkedTokenSource(processInfo.StoppedTokenSource.Token);
// This is the thread that will collect diagnostics from the running process
// - Logs - I'll collect structured logs from Microsoft.Extensions.Logging
// - Metrics - It'll collect EventCounters
// - Distribued Traces - It'll create spans
var diagnosticsThread = new Thread(state =>
{
_diagnosticsCollector.ProcessEvents(
applicationName,
service.Description.Name,
(int)state,
replica,
status,
metricsTokenSource.Token);
});
try
{
var result = ProcessUtil.Run(path, args,
environmentVariables: environment,
workingDirectory: workingDirectory,
outputDataReceived: data => service.Logs.OnNext("[" + replica + "]: " + data),
onStart: pid =>
{
if (hasPorts)
{
_logger.LogInformation("{ServiceName} running on process id {PID} bound to {Address}", replica, pid, string.Join(", ", ports.Select(p => $"{p.Protocol ?? "http"}://localhost:{p.Port}")));
}
else
{
_logger.LogInformation("{ServiceName} running on process id {PID}", replica, pid);
}
status.Pid = pid;
diagnosticsThread.Start(pid);
},
throwOnError: false,
cancellationToken: processInfo.StoppedTokenSource.Token);
status.ExitCode = result.ExitCode;
if (status.Pid != null)
{
metricsTokenSource.Cancel();
diagnosticsThread.Join();
}
}
catch (Exception ex)
{
_logger.LogError(0, ex, "Failed to launch process for service {ServiceName}", replica);
Thread.Sleep(5000);
}
service.Restarts++;
if (status.ExitCode != null)
{
_logger.LogInformation("{ServiceName} process exited with exit code {ExitCode}", replica, status.ExitCode);
}
// Remove the replica from the set
service.Replicas.TryRemove(replica, out _);
}
}
if (serviceDescription.Bindings.Count > 0)
{
// Each replica is assigned a list of internal ports, one mapped to each external
// port
for (int i = 0; i < serviceDescription.Replicas; i++)
{
var ports = new List <(int, int, string)>();
foreach (var binding in serviceDescription.Bindings)
{
if (binding.Port == null)
{
continue;
}
ports.Add((service.PortMap[binding.Port.Value][i], binding.Port.Value, binding.Protocol));
}
processInfo.Threads[i] = new Thread(() => RunApplication(ports));
}
}
else
{
for (int i = 0; i < service.Description.Replicas; i++)
{
processInfo.Threads[i] = new Thread(() => RunApplication(Enumerable.Empty <(int, int, string)>()));
}
}
for (int i = 0; i < service.Description.Replicas; i++)
{
processInfo.Threads[i].Start();
}
service.Items[typeof(ProcessInfo)] = processInfo;
return(Task.CompletedTask);
}
