public override async Task ExecuteAsync(OutputContext output, ApplicationBuilder application, IngressBuilder ingress)
{
// This code assumes that in the future we might support other ingress types besides nginx.
//
// Right now we only know some hardcoded details about ingress-nginx that we use for both
// validation and generation of manifests.
//
// For instance we don't support k8s 1.18.X IngressClass resources because that version
// isn't broadly available yet.
var ingressClass = "nginx";
if (!IngressClasses.Add(ingressClass))
{
output.WriteDebugLine($"Already validated ingress class '{ingressClass}'.");
return;
}
if (await KubectlDetector.GetKubernetesServerVersion(output) == null)
{
throw new CommandException($"Cannot validate ingress because kubectl is not installed.");
}
if (!await KubectlDetector.IsKubectlConnectedToClusterAsync(output))
{
throw new CommandException($"Cannot validate ingress because kubectl is not connected to a cluster.");
}
output.WriteDebugLine($"Validating ingress class '{ingressClass}'.");
var config = KubernetesClientConfiguration.BuildDefaultConfig();
// If namespace is null, set it to default
config.Namespace ??= "default";
var kubernetes = new Kubernetes(config);
// Looking for a deployment using a standard label.
// Note: using a deployment instead of a service - minikube doesn't create a service for the controller.
try
{
var result = await kubernetes.ListDeploymentForAllNamespacesWithHttpMessagesAsync(
labelSelector : "app.kubernetes.io/name in (ingress-nginx, nginx-ingress-controller)");
if (result.Body.Items.Count > 0)
{
foreach (var service in result.Body.Items)
{
output.WriteInfoLine($"Found existing ingress controller '{service.Metadata.Name}' in namespace '{service.Metadata.NamespaceProperty}'.");
}
return;
}
}
catch (HttpOperationException ex) when(ex.Response.StatusCode == HttpStatusCode.NotFound)
{
// The kubernetes client uses exceptions for 404s.
}
catch (Exception ex)
{
output.WriteDebugLine("Failed to query secret.");
output.WriteDebugLine(ex.ToString());
throw new CommandException("Unable connect to kubernetes.", ex);
}
if (Force)
{
output.WriteDebugLine("Skipping because force was specified.");
return;
}
if (!Interactive)
{
throw new CommandException(
$"An ingress was specified for the application, but the 'ingress-nginx' controller could not be found. " +
$"Rerun the command with --interactive to deploy a controller for development, or with --force to skip validation. Alternatively " +
$"see our documentation on ingress: https://aka.ms/tye/ingress");
}
output.WriteAlwaysLine(
"Tye can deploy the ingress-nginx controller for you. This will be a basic deployment suitable for " +
"experimentation and development. Your production needs, or requirements may differ depending on your Kubernetes distribution. " +
"See: https://aka.ms/tye/ingress for documentation.");
if (!output.Confirm($"Deploy ingress-nginx"))
{
// user skipped deployment of ingress, continue with deployment.
return;
}
// We want to be able to detect minikube because the process for enabling nginx-ingress is different there,
// it's shipped as an addon.
//
// see: https://github.com/telepresenceio/telepresence/blob/4364fd83d5926bef46babd704e7bd6c82a75dbd6/telepresence/startup.py#L220
if (config.CurrentContext == "minikube")
{
output.WriteDebugLine($"Running 'minikube addons enable ingress'");
output.WriteCommandLine("minikube", "addon enable ingress");
var capture = output.Capture();
var exitCode = await ProcessUtil.ExecuteAsync(
$"minikube",
$"addons enable ingress",
System.Environment.CurrentDirectory,
stdOut : capture.StdOut,
stdErr : capture.StdErr);
output.WriteDebugLine($"Done running 'minikube addons enable ingress' exit code: {exitCode}");
if (exitCode != 0)
{
throw new CommandException("'minikube addons enable ingress' failed.");
}
output.WriteInfoLine($"Deployed ingress-nginx.");
}
else
{
// If we get here then we should deploy the ingress controller.
// The first time we apply the ingress controller, the validating webhook will not have started.
// This causes an error to be returned from the process. As this always happens, we are going to
// not check the error returned and assume the kubectl command worked. This is double checked in
// the future as well when we try to create the ingress resource.
output.WriteDebugLine($"Running 'kubectl apply'");
output.WriteCommandLine("kubectl", $"apply -f \"https://aka.ms/tye/ingress/deploy\"");
var capture = output.Capture();
var exitCode = await ProcessUtil.ExecuteAsync(
$"kubectl",
$"apply -f \"https://aka.ms/tye/ingress/deploy\"",
System.Environment.CurrentDirectory);
output.WriteDebugLine($"Done running 'kubectl apply' exit code: {exitCode}");
output.WriteInfoLine($"Waiting for ingress-nginx controller to start.");
// We need to then wait for the webhooks that are created by ingress-nginx to start. Deploying an ingress immediately
// after creating the controller will fail if the webhook isn't ready.
//
// Internal error occurred: failed calling webhook "validate.nginx.ingress.kubernetes.io":
// Post https://ingress-nginx-controller-admission.ingress-nginx.svc:443/networking.k8s.io/v1/ingresses?timeout=30s:
// dial tcp 10.0.31.130:443: connect: connection refused
//
// Unfortunately this is the likely case for us.
try
{
output.WriteDebugLine("Watching for ingress-nginx controller readiness...");
var response = await kubernetes.ListNamespacedPodWithHttpMessagesAsync(
namespaceParameter : "ingress-nginx",
labelSelector : "app.kubernetes.io/component=controller,app.kubernetes.io/name=ingress-nginx",
watch : true);
var tcs = new TaskCompletionSource <object?>();
using var watcher = response.Watch <V1Pod, V1PodList>(
onEvent: (@event, pod) =>
{
// Wait for the readiness-check to pass.
if (pod.Status.Conditions.All(c => string.Equals(c.Status, bool.TrueString, StringComparison.OrdinalIgnoreCase)))
{
tcs.TrySetResult(null); // Success!
output.WriteDebugLine($"Pod '{pod.Metadata.Name}' is ready.");
}
},
onError: ex =>
{
tcs.TrySetException(ex);
output.WriteDebugLine("Watch operation failed.");
},
onClosed: () =>
{
// YOLO?
tcs.TrySetResult(null);
output.WriteDebugLine("Watch operation completed.");
});
await tcs.Task;
}
catch (Exception ex)
{
output.WriteDebugLine("Failed to ingress-nginx pods.");
output.WriteDebugLine(ex.ToString());
throw new CommandException("Failed to query ingress-nginx pods.", ex);
}
output.WriteInfoLine($"Deployed ingress-nginx.");
}
}
public static KubernetesIngressOutput CreateIngress(
OutputContext output,
ApplicationBuilder application,
IngressBuilder ingress)
{
var root = new YamlMappingNode();
root.Add("kind", "Ingress");
root.Add("apiVersion", "extensions/v1beta1");
var metadata = new YamlMappingNode();
root.Add("metadata", metadata);
metadata.Add("name", ingress.Name);
if (!string.IsNullOrEmpty(application.Namespace))
{
metadata.Add("namespace", application.Namespace);
}
var annotations = new YamlMappingNode();
metadata.Add("annotations", annotations);
annotations.Add("kubernetes.io/ingress.class", new YamlScalarNode("nginx")
{
Style = ScalarStyle.SingleQuoted,
});
annotations.Add("nginx.ingress.kubernetes.io/rewrite-target", new YamlScalarNode("/$2")
{
Style = ScalarStyle.SingleQuoted,
});
var labels = new YamlMappingNode();
metadata.Add("labels", labels);
labels.Add("app.kubernetes.io/part-of", new YamlScalarNode(application.Name)
{
Style = ScalarStyle.SingleQuoted,
});
var spec = new YamlMappingNode();
root.Add("spec", spec);
if (ingress.Rules.Count > 0)
{
var rules = new YamlSequenceNode();
spec.Add("rules", rules);
// k8s ingress is grouped by host first, then grouped by path
foreach (var hostgroup in ingress.Rules.GroupBy(r => r.Host))
{
var rule = new YamlMappingNode();
rules.Add(rule);
if (!string.IsNullOrEmpty(hostgroup.Key))
{
rule.Add("host", hostgroup.Key);
}
var http = new YamlMappingNode();
rule.Add("http", http);
var paths = new YamlSequenceNode();
http.Add("paths", paths);
foreach (var ingressRule in hostgroup)
{
var path = new YamlMappingNode();
paths.Add(path);
var backend = new YamlMappingNode();
path.Add("backend", backend);
backend.Add("serviceName", ingressRule.Service);
var service = application.Services.FirstOrDefault(s => s.Name == ingressRule.Service);
if (service is null)
{
throw new InvalidOperationException($"Could not resolve service '{ingressRule.Service}'.");
}
var binding = service.Bindings.FirstOrDefault(b => b.Name is null || b.Name == "http");
if (binding is null)
{
throw new InvalidOperationException($"Could not resolve an http binding for service '{service.Name}'.");
}
backend.Add("servicePort", (binding.Port ?? 80).ToString(CultureInfo.InvariantCulture));
// Tye implements path matching similar to this example:
// https://kubernetes.github.io/ingress-nginx/examples/rewrite/
//
// Therefore our rewrite-target is set to $2 - we want to make sure we have
// two capture groups.
if (string.IsNullOrEmpty(ingressRule.Path) || ingressRule.Path == "/")
{
path.Add("path", "/()(.*)"); // () is an empty capture group.
}
else
{
var regex = $"{ingressRule.Path.TrimEnd('/')}(/|$)(.*)";
path.Add("path", regex);
}
}
}
}
return(new KubernetesIngressOutput(ingress.Name, new YamlDocument(root)));
}
public static async Task <ApplicationBuilder> CreateAsync(OutputContext output, FileInfo source)
{
if (source is null)
{
throw new ArgumentNullException(nameof(source));
}
var config = ConfigFactory.FromFile(source);
ValidateConfigApplication(config);
var builder = new ApplicationBuilder(source, config.Name ?? source.Directory.Name.ToLowerInvariant());
if (!string.IsNullOrEmpty(config.Registry))
{
builder.Registry = new ContainerRegistry(config.Registry);
}
foreach (var configService in config.Services)
{
ServiceBuilder service;
if (!string.IsNullOrEmpty(configService.Project))
{
var expandedProject = Environment.ExpandEnvironmentVariables(configService.Project);
var projectFile = new FileInfo(Path.Combine(builder.Source.DirectoryName, expandedProject));
var project = new ProjectServiceBuilder(configService.Name, projectFile);
service = project;
project.Build = configService.Build ?? true;
project.Args = configService.Args;
project.Replicas = configService.Replicas ?? 1;
await ProjectReader.ReadProjectDetailsAsync(output, project);
// We don't apply more container defaults here because we might need
// to prompty for the registry name.
project.ContainerInfo = new ContainerInfo()
{
UseMultiphaseDockerfile = false,
};
}
else if (!string.IsNullOrEmpty(configService.Image))
{
var container = new ContainerServiceBuilder(configService.Name, configService.Image)
{
Args = configService.Args,
Replicas = configService.Replicas ?? 1
};
service = container;
}
else if (!string.IsNullOrEmpty(configService.Executable))
{
var expandedExecutable = Environment.ExpandEnvironmentVariables(configService.Executable);
var workingDirectory = "";
// Special handling of .dlls as executables (it will be executed as dotnet {dll})
if (Path.GetExtension(expandedExecutable) == ".dll")
{
expandedExecutable = Path.GetFullPath(Path.Combine(builder.Source.Directory.FullName, expandedExecutable));
workingDirectory = Path.GetDirectoryName(expandedExecutable) !;
}
var executable = new ExecutableServiceBuilder(configService.Name, expandedExecutable)
{
Args = configService.Args,
WorkingDirectory = configService.WorkingDirectory != null?
Path.GetFullPath(Path.Combine(builder.Source.Directory.FullName, Environment.ExpandEnvironmentVariables(configService.WorkingDirectory))) :
workingDirectory,
Replicas = configService.Replicas ?? 1
};
service = executable;
}
else if (configService.External)
{
var external = new ExternalServiceBuilder(configService.Name);
service = external;
}
else
{
throw new CommandException("Unable to determine service type.");
}
builder.Services.Add(service);
foreach (var configBinding in configService.Bindings)
{
var binding = new BindingBuilder()
{
Name = configBinding.Name,
AutoAssignPort = configBinding.AutoAssignPort,
ConnectionString = configBinding.ConnectionString,
Host = configBinding.Host,
ContainerPort = configBinding.ContainerPort,
Port = configBinding.Port,
Protocol = configBinding.Protocol,
};
// Assume HTTP for projects only (containers may be different)
if (binding.ConnectionString == null && configService.Project != null)
{
binding.Protocol ??= "http";
}
service.Bindings.Add(binding);
}
foreach (var configEnvVar in configService.Configuration)
{
var envVar = new EnvironmentVariable(configEnvVar.Name, configEnvVar.Value);
if (service is ProjectServiceBuilder project)
{
project.EnvironmentVariables.Add(envVar);
}
else if (service is ContainerServiceBuilder container)
{
container.EnvironmentVariables.Add(envVar);
}
else if (service is ExecutableServiceBuilder executable)
{
executable.EnvironmentVariables.Add(envVar);
}
else if (service is ExternalServiceBuilder)
{
throw new CommandException("External services do not support environment variables.");
}
else
{
throw new CommandException("Unable to determine service type.");
}
}
foreach (var configVolume in configService.Volumes)
{
var volume = new VolumeBuilder(configVolume.Source, configVolume.Target);
if (service is ProjectServiceBuilder project)
{
project.Volumes.Add(volume);
}
else if (service is ContainerServiceBuilder container)
{
container.Volumes.Add(volume);
}
else if (service is ExecutableServiceBuilder executable)
{
throw new CommandException("Executable services do not support volumes.");
}
else if (service is ExternalServiceBuilder)
{
throw new CommandException("External services do not support volumes.");
}
else
{
throw new CommandException("Unable to determine service type.");
}
}
}
foreach (var configIngress in config.Ingress)
{
var ingress = new IngressBuilder(configIngress.Name);
ingress.Replicas = configIngress.Replicas ?? 1;
builder.Ingress.Add(ingress);
foreach (var configBinding in configIngress.Bindings)
{
var binding = new IngressBindingBuilder()
{
AutoAssignPort = configBinding.AutoAssignPort,
Name = configBinding.Name,
Port = configBinding.Port,
Protocol = configBinding.Protocol ?? "http",
};
ingress.Bindings.Add(binding);
}
foreach (var configRule in configIngress.Rules)
{
var rule = new IngressRuleBuilder()
{
Host = configRule.Host,
Path = configRule.Path,
Service = configRule.Service,
};
ingress.Rules.Add(rule);
}
}
return(builder);
}
public override Task ExecuteAsync(OutputContext output, ApplicationBuilder application, IngressBuilder ingress)
{
ingress.Outputs.Add(KubernetesManifestGenerator.CreateIngress(output, application, ingress));
return(Task.CompletedTask);
}
public abstract Task ExecuteAsync(OutputContext output, ApplicationBuilder application, IngressBuilder ingres);
public override async Task ExecuteAsync(OutputContext output, ApplicationBuilder application, IngressBuilder ingress)
{
ingress.Outputs.Add(await KubernetesManifestGenerator.CreateIngress(output, application, ingress));
}
