public async Task <IList <string> > ListClustersAsync()
{
List <string> clusters = new List <string>();
var conf = await KubernetesClientConfiguration.LoadKubeConfigAsync();
foreach (var cluster in conf.Clusters)
{
clusters.Add(cluster.Name);
}
return(clusters);
}
private async void LoadKubeConfigAsync()
{
try
{
Config = await KubernetesClientConfiguration.LoadKubeConfigAsync();
Contexts = Config.Contexts.Select(c => c.Name).ToArray();
CurrentContext = Config.CurrentContext ?? Contexts.FirstOrDefault();
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
throw;
}
}
public override async Task ExecuteAsync(OutputContext output, ApplicationBuilder application, ServiceBuilder service)
{
var bindings = service.Outputs.OfType <ComputedBindings>().FirstOrDefault();
if (bindings is null)
{
return;
}
foreach (var binding in bindings.Bindings)
{
if (binding is SecretInputBinding secretInputBinding)
{
if (!Secrets.Add(secretInputBinding.Name))
{
output.WriteDebugLine($"Already validated secret '{secretInputBinding.Name}'.");
continue;
}
output.WriteDebugLine($"Validating secret '{secretInputBinding.Name}'.");
var config = KubernetesClientConfiguration.BuildDefaultConfig();
// Workaround for https://github.com/kubernetes-client/csharp/issues/372
var store = await KubernetesClientConfiguration.LoadKubeConfigAsync();
var context = store.Contexts.Where(c => c.Name == config.CurrentContext).FirstOrDefault();
config.Namespace ??= context?.ContextDetails?.Namespace;
var kubernetes = new Kubernetes(config);
try
{
var result = await kubernetes.ReadNamespacedSecretWithHttpMessagesAsync(secretInputBinding.Name, config.Namespace ?? "default");
output.WriteInfoLine($"Found existing secret '{secretInputBinding.Name}'.");
continue;
}
catch (HttpOperationException ex) when(ex.Response.StatusCode == HttpStatusCode.NotFound)
{
// The kubernetes client uses exceptions for 404s.
}
catch (Exception ex)
{
output.WriteDebugLine("Failed to query secret.");
output.WriteDebugLine(ex.ToString());
throw new CommandException("Unable connect to kubernetes.", ex);
}
if (Force)
{
output.WriteDebugLine("Skipping because force was specified.");
continue;
}
if (!Interactive)
{
throw new CommandException(
$"The secret '{secretInputBinding.Name}' used for service '{secretInputBinding.Service.Name}' is missing from the deployment environment. " +
$"Rerun the command with --interactive to specify the value interactively, or with --force to skip validation. Alternatively " +
$"use the following command to manually create the secret." + System.Environment.NewLine +
$"kubectl create secret generic {secretInputBinding.Name} --from-literal=connectionstring=<value>");
}
// If we get here then we should create the secret.
var text = output.Prompt($"Enter the connection string to use for service '{secretInputBinding.Service.Name}'", allowEmpty: true);
if (string.IsNullOrWhiteSpace(text))
{
output.WriteAlways($"Skipping creation of secret for '{secretInputBinding.Service.Name}'. This may prevent creation of pods until secrets are created.");
output.WriteAlways($"Manually create a secret with:");
output.WriteAlways($"kubectl create secret generic {secretInputBinding.Name} --from-literal=connectionstring=<value>");
continue;
}
var secret = new V1Secret(type: "Opaque", stringData: new Dictionary <string, string>()
{
{ "connectionstring", text },
});
secret.Metadata = new V1ObjectMeta();
secret.Metadata.Name = secretInputBinding.Name;
output.WriteDebugLine($"Creating secret '{secret.Metadata.Name}'.");
try
{
await kubernetes.CreateNamespacedSecretWithHttpMessagesAsync(secret, config.Namespace ?? "default");
output.WriteInfoLine($"Created secret '{secret.Metadata.Name}'.");
}
catch (Exception ex)
{
output.WriteDebugLine("Failed to create secret.");
output.WriteDebugLine(ex.ToString());
throw new CommandException("Failed to create secret.", ex);
}
}
}
var yaml = service.Outputs.OfType <IYamlManifestOutput>().ToArray();
if (yaml.Length == 0)
{
output.WriteDebugLine($"No yaml manifests found for service '{service.Name}'. Skipping.");
return;
}
using var tempFile = TempFile.Create();
output.WriteDebugLine($"Writing output to '{tempFile.FilePath}'.");
{
await using var stream = File.OpenWrite(tempFile.FilePath);
await using var writer = new StreamWriter(stream, Encoding.UTF8, bufferSize: -1, leaveOpen: true);
var yamlStream = new YamlStream(yaml.Select(y => y.Yaml));
yamlStream.Save(writer, assignAnchors: false);
}
// kubectl apply logic is implemented in the client in older versions of k8s. The capability
// to get the same behavior in the server isn't present in every version that's relevant.
//
// https://kubernetes.io/docs/reference/using-api/api-concepts/#server-side-apply
//
output.WriteDebugLine("Running 'kubectl apply'.");
output.WriteCommandLine("kubectl", $"apply -f \"{tempFile.FilePath}\"");
var capture = output.Capture();
var exitCode = await Process.ExecuteAsync(
$"kubectl",
$"apply -f \"{tempFile.FilePath}\"",
System.Environment.CurrentDirectory,
stdOut : capture.StdOut,
stdErr : capture.StdErr);
output.WriteDebugLine($"Done running 'kubectl apply' exit code: {exitCode}");
if (exitCode != 0)
{
throw new CommandException("'kubectl apply' failed.");
}
output.WriteInfoLine($"Deployed service '{service.Name}'.");
}
public override async Task ExecuteAsync(OutputContext output, ApplicationBuilder application, ServiceBuilder service)
{
var bindings = service.Outputs.OfType <ComputedBindings>().FirstOrDefault();
if (bindings is null)
{
return;
}
foreach (var binding in bindings.Bindings)
{
if (binding is SecretInputBinding secretInputBinding)
{
if (!Secrets.Add(secretInputBinding.Name))
{
output.WriteDebugLine($"Already validated secret '{secretInputBinding.Name}'.");
continue;
}
output.WriteDebugLine($"Validating secret '{secretInputBinding.Name}'.");
var config = KubernetesClientConfiguration.BuildDefaultConfig();
// Workaround for https://github.com/kubernetes-client/csharp/issues/372
var store = await KubernetesClientConfiguration.LoadKubeConfigAsync();
var context = store.Contexts.Where(c => c.Name == config.CurrentContext).FirstOrDefault();
// Use namespace of application, or current context, or 'default'
config.Namespace = application.Namespace;
config.Namespace ??= context?.ContextDetails?.Namespace ?? "default";
var kubernetes = new Kubernetes(config);
try
{
var result = await kubernetes.ReadNamespacedSecretWithHttpMessagesAsync(secretInputBinding.Name, config.Namespace);
output.WriteInfoLine($"Found existing secret '{secretInputBinding.Name}'.");
continue;
}
catch (HttpOperationException ex) when(ex.Response.StatusCode == HttpStatusCode.NotFound)
{
// The kubernetes client uses exceptions for 404s.
}
catch (Exception ex)
{
output.WriteDebugLine("Failed to query secret.");
output.WriteDebugLine(ex.ToString());
throw new CommandException("Unable connect to kubernetes.", ex);
}
if (Force)
{
output.WriteDebugLine("Skipping because force was specified.");
continue;
}
if (!Interactive && secretInputBinding is SecretConnectionStringInputBinding)
{
throw new CommandException(
$"The secret '{secretInputBinding.Name}' used for service '{secretInputBinding.Service.Name}' is missing from the deployment environment. " +
$"Rerun the command with --interactive to specify the value interactively, or with --force to skip validation. Alternatively " +
$"use the following command to manually create the secret." + System.Environment.NewLine +
$"kubectl create secret generic {secretInputBinding.Name} --namespace {config.Namespace} --from-literal=connectionstring=<value>");
}
if (!Interactive && secretInputBinding is SecretUrlInputBinding)
{
throw new CommandException(
$"The secret '{secretInputBinding.Name}' used for service '{secretInputBinding.Service.Name}' is missing from the deployment environment. " +
$"Rerun the command with --interactive to specify the value interactively, or with --force to skip validation. Alternatively " +
$"use the following command to manually create the secret." + System.Environment.NewLine +
$"kubectl create secret generic {secretInputBinding.Name} --namespace {config.Namespace} --from-literal=protocol=<value> --from-literal=host=<value> --from-literal=port=<value>");
}
V1Secret secret;
if (secretInputBinding is SecretConnectionStringInputBinding)
{
// If we get here then we should create the secret.
var text = output.Prompt($"Enter the connection string to use for service '{secretInputBinding.Service.Name}'", allowEmpty: true);
if (string.IsNullOrWhiteSpace(text))
{
output.WriteAlwaysLine($"Skipping creation of secret for '{secretInputBinding.Service.Name}'. This may prevent creation of pods until secrets are created.");
output.WriteAlwaysLine($"Manually create a secret with:");
output.WriteAlwaysLine($"kubectl create secret generic {secretInputBinding.Name} --namespace {config.Namespace} --from-literal=connectionstring=<value>");
continue;
}
secret = new V1Secret(type: "Opaque", stringData: new Dictionary <string, string>()
{
{ "connectionstring", text },
});
}
else if (secretInputBinding is SecretUrlInputBinding)
{
// If we get here then we should create the secret.
string text;
Uri? uri = null;
while (true)
{
text = output.Prompt($"Enter the URI to use for service '{secretInputBinding.Service.Name}'", allowEmpty: true);
if (string.IsNullOrEmpty(text))
{
break; // skip
}
else if (Uri.TryCreate(text, UriKind.Absolute, out uri))
{
break; // success
}
output.WriteAlwaysLine($"Invalid URI: '{text}'");
}
if (string.IsNullOrWhiteSpace(text))
{
output.WriteAlwaysLine($"Skipping creation of secret for '{secretInputBinding.Service.Name}'. This may prevent creation of pods until secrets are created.");
output.WriteAlwaysLine($"Manually create a secret with:");
output.WriteAlwaysLine($"kubectl create secret generic {secretInputBinding.Name} --namespace {config.Namespace} --from-literal=protocol=<value> --from-literal=host=<value> --from-literal=port=<value>");
continue;
}
secret = new V1Secret(type: "Opaque", stringData: new Dictionary <string, string>()
{
{ "protocol", uri !.Scheme },
public static async Task ExecuteUndeployAsync(OutputContext output, ConfigApplication application, bool interactive, bool whatIf)
{
var config = KubernetesClientConfiguration.BuildDefaultConfig();
// Workaround for https://github.com/kubernetes-client/csharp/issues/372
var store = await KubernetesClientConfiguration.LoadKubeConfigAsync();
var context = store.Contexts.Where(c => c.Name == config.CurrentContext).FirstOrDefault();
config.Namespace ??= context?.ContextDetails?.Namespace;
var kubernetes = new Kubernetes(config);
// Due to some limitations in the k8s SDK we currently have a hardcoded list of resource
// types that we handle deletes for. If we start adding extensibility for the *kinds* of
// k8s resources we create, or the ability to deploy additional files along with the
// resources we understand then we should revisit this.
//
// Basically the challenges are:
//
// - kubectl api-resources --all (and similar) are implemented client-side (n+1 problem)
// - the C# k8s SDK doesn't have an untyped api for operations on arbitrary resources, the
// closest thing is the custom resource APIs
// - Legacy resources without an api group don't follow the same URL scheme as more modern
// ones, and thus cannot be addressed using the custom resource APIs.
//
// So solving 'undeploy' generically would involve doing a bunch of work to query things
// generically, including going outside of what's provided by the SDK.
//
// - querying api-resources
// - querying api-groups
// - handcrafing requests to list for each resource
// - handcrafting requests to delete each resource
var resources = new List <Resource>();
try
{
output.WriteDebugLine("Querying services");
var response = await kubernetes.ListNamespacedServiceWithHttpMessagesAsync(
config.Namespace,
labelSelector : $"app.kubernetes.io/part-of={application.Name}");
foreach (var resource in response.Body.Items)
{
resource.Kind = V1Service.KubeKind;
}
resources.AddRange(response.Body.Items.Select(item => new Resource(item, item.Metadata, DeleteService)));
output.WriteDebugLine($"Found {response.Body.Items.Count} matching services");
}
catch (Exception ex)
{
output.WriteDebugLine("Failed to query services.");
output.WriteDebugLine(ex.ToString());
throw new CommandException("Unable connect to kubernetes.", ex);
}
try
{
output.WriteDebugLine("Querying deployments");
var response = await kubernetes.ListNamespacedDeploymentWithHttpMessagesAsync(
config.Namespace,
labelSelector : $"app.kubernetes.io/part-of={application.Name}");
foreach (var resource in response.Body.Items)
{
resource.Kind = V1Deployment.KubeKind;
}
resources.AddRange(response.Body.Items.Select(item => new Resource(item, item.Metadata, DeleteDeployment)));
output.WriteDebugLine($"Found {response.Body.Items.Count} matching deployments");
}
catch (Exception ex)
{
output.WriteDebugLine("Failed to query deployments.");
output.WriteDebugLine(ex.ToString());
throw new CommandException("Unable connect to kubernetes.", ex);
}
try
{
output.WriteDebugLine("Querying secrets");
var response = await kubernetes.ListNamespacedSecretWithHttpMessagesAsync(
config.Namespace,
labelSelector : $"app.kubernetes.io/part-of={application.Name}");
foreach (var resource in response.Body.Items)
{
resource.Kind = V1Secret.KubeKind;
}
resources.AddRange(response.Body.Items.Select(item => new Resource(item, item.Metadata, DeleteSecret)));
output.WriteDebugLine($"Found {response.Body.Items.Count} matching secrets");
}
catch (Exception ex)
{
output.WriteDebugLine("Failed to query secrets.");
output.WriteDebugLine(ex.ToString());
throw new CommandException("Unable connect to kubernetes.", ex);
}
output.WriteInfoLine($"Found {resources.Count} resource(s).");
var exceptions = new List <(Resource resource, HttpOperationException exception)>();
foreach (var resource in resources)
{
var operation = Operations.Delete;
if (interactive && !output.Confirm($"Delete {resource.Obj.Kind} '{resource.Metadata.Name}'?"))
{
operation = Operations.None;
}
if (whatIf && operation == Operations.Delete)
{
operation = Operations.Explain;
}
if (operation == Operations.None)
{
output.WriteAlwaysLine($"Skipping '{resource.Obj.Kind}' '{resource.Metadata.Name}' ...");
}
else if (operation == Operations.Explain)
{
output.WriteAlwaysLine($"whatif: Deleting '{resource.Obj.Kind}' '{resource.Metadata.Name}' ...");
}
else if (operation == Operations.Delete)
{
output.WriteAlwaysLine($"Deleting '{resource.Obj.Kind}' '{resource.Metadata.Name}' ...");
try
{
var response = await resource.Deleter(resource.Metadata.Name);
output.WriteDebugLine($"Successfully deleted resource: '{resource.Obj.Kind}' '{resource.Metadata.Name}'.");
}
catch (HttpOperationException ex)
{
output.WriteDebugLine($"Failed to delete resource: '{resource.Obj.Kind}' '{resource.Metadata.Name}'.");
output.WriteDebugLine(ex.ToString());
exceptions.Add((resource, ex));
}
}
}
if (exceptions.Count > 0)
{
throw new CommandException(
$"Failed to delete some resources: " + Environment.NewLine + Environment.NewLine +
string.Join(Environment.NewLine, exceptions.Select(e => $"\t'{e.resource.Obj.Kind}' '{e.resource.Metadata.Name}': {e.exception.Body}.")));
}
Task <Rest.HttpOperationResponse <V1Status> > DeleteService(string name)
{
return(kubernetes !.DeleteNamespacedServiceWithHttpMessagesAsync(name, config !.Namespace));
}
Task <Rest.HttpOperationResponse <V1Status> > DeleteDeployment(string name)
{
return(kubernetes !.DeleteNamespacedDeploymentWithHttpMessagesAsync(name, config !.Namespace));
}
Task <Rest.HttpOperationResponse <V1Status> > DeleteSecret(string name)
{
return(kubernetes !.DeleteNamespacedSecretWithHttpMessagesAsync(name, config !.Namespace));
}
}
public virtual async Task <K8SConfiguration> Get()
{
return(await KubernetesClientConfiguration.LoadKubeConfigAsync());
}
public override async Task ExecuteAsync(OutputContext output, ApplicationBuilder application, IngressBuilder ingress)
{
// This code assumes that in the future we might support other ingress types besides nginx.
//
// Right now we only know some hardcoded details about ingress-nginx that we use for both
// validation and generation of manifests.
//
// For instance we don't support k8s 1.18.X IngressClass resources because that version
// isn't broadly available yet.
var ingressClass = "nginx";
if (!IngressClasses.Add(ingressClass))
{
output.WriteDebugLine($"Already validated ingress class '{ingressClass}'.");
return;
}
if (!await KubectlDetector.Instance.IsKubectlInstalled.Value)
{
throw new CommandException($"Cannot validate ingress because kubectl is not installed.");
}
if (!await KubectlDetector.Instance.IsKubectlConnectedToCluster.Value)
{
throw new CommandException($"Cannot validate ingress because kubectl is not connected to a cluster.");
}
output.WriteDebugLine($"Validating ingress class '{ingressClass}'.");
var config = KubernetesClientConfiguration.BuildDefaultConfig();
// Workaround for https://github.com/kubernetes-client/csharp/issues/372
var store = await KubernetesClientConfiguration.LoadKubeConfigAsync();
var context = store.Contexts.Where(c => c.Name == config.CurrentContext).FirstOrDefault();
// Use namespace of application, or current context, or 'default'
config.Namespace = application.Namespace;
config.Namespace ??= context?.ContextDetails?.Namespace ?? "default";
var kubernetes = new Kubernetes(config);
// Looking for a deployment using a standard label.
// Note: using a deployment instead of a service - minikube doesn't create a service for the controller.
try
{
var result = await kubernetes.ListDeploymentForAllNamespacesWithHttpMessagesAsync(
labelSelector : "app.kubernetes.io/name in (ingress-nginx, nginx-ingress-controller)");
if (result.Body.Items.Count > 0)
{
foreach (var service in result.Body.Items)
{
output.WriteInfoLine($"Found existing ingress controller '{service.Metadata.Name}' in namespace '{service.Metadata.NamespaceProperty}'.");
}
return;
}
}
catch (HttpOperationException ex) when(ex.Response.StatusCode == HttpStatusCode.NotFound)
{
// The kubernetes client uses exceptions for 404s.
}
catch (Exception ex)
{
output.WriteDebugLine("Failed to query secret.");
output.WriteDebugLine(ex.ToString());
throw new CommandException("Unable connect to kubernetes.", ex);
}
if (Force)
{
output.WriteDebugLine("Skipping because force was specified.");
return;
}
if (!Interactive)
{
throw new CommandException(
$"An ingress was specified for the application, but the 'ingress-nginx' controller could not be found. " +
$"Rerun the command with --interactive to deploy a controller for development, or with --force to skip validation. Alternatively " +
$"see our documentation on ingress: https://aka.ms/tye/ingress");
}
output.WriteAlwaysLine(
"Tye can deploy the ingress-nginx controller for you. This will be a basic deployment suitable for " +
"experimentation and development. Your production needs, or requirments may differ depending on your Kubernetes distribution. " +
"See: https://aka.ms/tye/ingress for documentation.");
if (!output.Confirm($"Deploy ingress-nginx"))
{
// user skipped deployment of ingress, continue with deployment.
return;
}
// We want to be able to detect minikube because the process for enabling nginx-ingress is different there,
// it's shipped as an addon.
//
// see: https://github.com/telepresenceio/telepresence/blob/4364fd83d5926bef46babd704e7bd6c82a75dbd6/telepresence/startup.py#L220
if (config.CurrentContext == "minikube")
{
output.WriteDebugLine($"Running 'minikube addons enable ingress'");
output.WriteCommandLine("minikube", "addon enable ingress");
var capture = output.Capture();
var exitCode = await Process.ExecuteAsync(
$"minikube",
$"addons enable ingress",
System.Environment.CurrentDirectory,
stdOut : capture.StdOut,
stdErr : capture.StdErr);
output.WriteDebugLine($"Done running 'minikube addons enable ingress' exit code: {exitCode}");
if (exitCode != 0)
{
throw new CommandException("'minikube addons enable ingress' failed.");
}
output.WriteInfoLine($"Deployed ingress-nginx.");
}
else
{
// If we get here then we should deploy the ingress controller.
output.WriteDebugLine($"Running 'kubectl apply'");
output.WriteCommandLine("kubectl", $"apply -f \"https://aka.ms/tye/ingress/deploy\"");
var capture = output.Capture();
var exitCode = await Process.ExecuteAsync(
$"kubectl",
$"apply -f \"https://aka.ms/tye/ingress/deploy\"",
System.Environment.CurrentDirectory,
stdOut : capture.StdOut,
stdErr : capture.StdErr);
output.WriteDebugLine($"Done running 'kubectl apply' exit code: {exitCode}");
if (exitCode != 0)
{
throw new CommandException("'kubectl apply' failed.");
}
output.WriteInfoLine($"Waiting for ingress-nginx controller to start.");
// We need to then wait for the webhooks that are created by ingress-nginx to start. Deploying an ingress immediately
// after creating the controller will fail if the webhook isn't ready.
//
// Internal error occurred: failed calling webhook "validate.nginx.ingress.kubernetes.io":
// Post https://ingress-nginx-controller-admission.ingress-nginx.svc:443/extensions/v1beta1/ingresses?timeout=30s:
// dial tcp 10.0.31.130:443: connect: connection refused
//
// Unfortunately this is the likely case for us.
try
{
output.WriteDebugLine("Watching for ingress-nginx controller readiness...");
var response = await kubernetes.ListNamespacedPodWithHttpMessagesAsync(
namespaceParameter : "ingress-nginx",
labelSelector : "app.kubernetes.io/component=controller,app.kubernetes.io/name=ingress-nginx",
watch : true);
var tcs = new TaskCompletionSource <object?>();
using var watcher = response.Watch <V1Pod, V1PodList>(
onEvent: (@event, pod) =>
{
// Wait for the readiness-check to pass.
if (pod.Status.Conditions.All(c => string.Equals(c.Status, bool.TrueString, StringComparison.OrdinalIgnoreCase)))
{
tcs.TrySetResult(null); // Success!
output.WriteDebugLine($"Pod '{pod.Metadata.Name}' is ready.");
}
},
onError: ex =>
{
tcs.TrySetException(ex);
output.WriteDebugLine("Watch operation failed.");
},
onClosed: () =>
{
// YOLO?
tcs.TrySetResult(null);
output.WriteDebugLine("Watch operation completed.");
});
await tcs.Task;
}
catch (Exception ex)
{
output.WriteDebugLine("Failed to ingress-nginx pods.");
output.WriteDebugLine(ex.ToString());
throw new CommandException("Failed to query ingress-nginx pods.", ex);
}
output.WriteInfoLine($"Deployed ingress-nginx.");
}
}
public async Task <IEnumerable <Context> > Get()
{
return((await KubernetesClientConfiguration.LoadKubeConfigAsync()
.ConfigureAwait(false))
.Contexts.Select(context => new Context(context.Name)));
}
