/// <summary>
/// Performs low-level initialization of a cluster.
/// </summary>
/// <param name="controller">The setup controller.</param>
/// <param name="upgradeLinux">Optionally upgrade the node's Linux distribution (defaults to <c>false</c>).</param>
/// <param name="patchLinux">Optionally apply any available Linux security patches (defaults to <c>true</c>).</param>
public void BaseInitialize(ISetupController controller, bool upgradeLinux = false, bool patchLinux = true)
{
Covenant.Requires <ArgumentException>(controller != null, nameof(controller));
var hostingEnvironment = controller.Get <HostingEnvironment>(KubeSetupProperty.HostingEnvironment);
// Wait for boot/connect.
controller.LogProgress(this, verb: "login", message: $"[{KubeConst.SysAdminUser}]");
WaitForBoot();
VerifyNodeOS(controller);
BaseDisableSwap(controller);
BaseInstallToolScripts(controller);
BaseConfigureDebianFrontend(controller);
UpdateRootCertificates();
BaseInstallPackages(controller);
BaseConfigureApt(controller);
BaseConfigureBashEnvironment(controller);
BaseConfigureDnsIPv4Preference(controller);
BaseRemoveSnap(controller);
BaseRemovePackages(controller);
if (patchLinux)
{
BasePatchLinux(controller);
}
BaseCreateKubeFolders(controller);
if (upgradeLinux)
{
BaseUpgradeLinuxDistribution(controller);
}
}
/// <summary>
/// Installs hypervisor guest integration services.
/// </summary>
/// <param name="controller">The setup controller.</param>
public void BaseInstallGuestIntegrationServices(ISetupController controller)
{
Covenant.Requires <ArgumentException>(controller != null, nameof(controller));
var hostingEnvironment = controller.Get <HostingEnvironment>(KubeSetupProperty.HostingEnvironment);
// This currently applies only to on-premise hypervisors.
if (!KubeHelper.IsOnPremiseHypervisorEnvironment(hostingEnvironment))
{
return;
}
InvokeIdempotent("base/guest-integration",
() =>
{
controller.LogProgress(this, verb: "setup", message: "guest integration services");
var guestServicesScript =
$@"#!/bin/bash
set -euo pipefail
cat <<EOF >> /etc/initramfs-tools/modules
hv_vmbus
hv_storvsc
hv_blkvsc
hv_netvsc
EOF
{KubeNodeFolder.Bin}/safe-apt-get install -yq linux-virtual linux-cloud-tools-virtual linux-tools-virtual
update-initramfs -u
";
SudoCommand(CommandBundle.FromScript(guestServicesScript), RunOptions.Defaults | RunOptions.FaultOnError);
});
}
/// <summary>
/// Disables <b>cloud-init</b>.
/// </summary>
/// <param name="controller">The setup controller.</param>
public void BaseDisableCloudInit(ISetupController controller)
{
Covenant.Requires <ArgumentException>(controller != null, nameof(controller));
var hostingEnvironment = controller.Get <HostingEnvironment>(KubeSetupProperty.HostingEnvironment);
// Do this only for non-cloud environments.
if (KubeHelper.IsCloudEnvironment(hostingEnvironment))
{
return;
}
InvokeIdempotent("base/cloud-init",
() =>
{
controller.LogProgress(this, verb: "disable", message: "cloud-init");
var disableCloudInitScript =
$@"
set -euo pipefail
mkdir -p /etc/cloud
touch /etc/cloud/cloud-init.disabled
";
SudoCommand(CommandBundle.FromScript(disableCloudInitScript), RunOptions.Defaults | RunOptions.FaultOnError);
});
}
/// <summary>
/// Returns the <see cref="IKubernetes"/> client persisted in the controller passed.
/// </summary>
/// <param name="controller">The setup controller.</param>
/// <returns>The <see cref="Kubernetes"/> client.</returns>
/// <exception cref="InvalidOperationException">
/// Thrown when there is no persisted Kubernetes client, indicating that <see cref="ConnectCluster(ISetupController)"/>
/// has not been called yet.
/// </exception>
public static IKubernetes GetK8sClient(ISetupController controller)
{
Covenant.Requires <ArgumentNullException>(controller != null, nameof(controller));
try
{
return(controller.Get <IKubernetes>(KubeSetupProperty.K8sClient));
}
catch (Exception e)
{
throw new InvalidOperationException($"Cannot retrieve the Kubernetes client because the cluster hasn't been connected via [{nameof(ConnectCluster)}()].", e);
}
}
/// <summary>
/// Disables DHCP.
/// </summary>
/// <param name="controller">The setup controller.</param>
public void BaseDisableDhcp(ISetupController controller)
{
Covenant.Requires <ArgumentException>(controller != null, nameof(controller));
var hostingEnvironment = controller.Get <HostingEnvironment>(KubeSetupProperty.HostingEnvironment);
InvokeIdempotent("base/dhcp",
() =>
{
controller.LogProgress(this, verb: "disable", message: "dhcp");
var initNetPlanScript =
$@"
set -euo pipefail
rm -rf /etc/netplan/*
cat <<EOF > /etc/netplan/no-dhcp.yaml
# This file is used to disable the network when a new VM is created
# from a template is booted. The [neon-init] service handles network
# provisioning in conjunction with the cluster prepare step.
#
# Cluster prepare inserts a virtual DVD disc with a script that
# handles the network configuration which [neon-init] will
# execute.
network:
version: 2
renderer: networkd
ethernets:
eth0:
dhcp4: no
EOF
";
SudoCommand(CommandBundle.FromScript(initNetPlanScript), RunOptions.Defaults | RunOptions.FaultOnError);
});
}
/// <summary>
/// Constructor.
/// </summary>
/// <param name="controller">The setup controller.</param>
internal SetupClusterStatus(ISetupController controller)
{
Covenant.Requires <ArgumentNullException>(controller != null, nameof(controller));
this.isClone = false;
this.controller = controller;
this.cluster = controller.Get <ClusterProxy>(KubeSetupProperty.ClusterProxy);
this.GlobalStatus = controller.GlobalStatus;
this.globalStatus = this.GlobalStatus;
// Initialize the cluster node/host status instances.
this.Nodes = new List <SetupNodeStatus>();
foreach (var node in cluster.Nodes)
{
Nodes.Add(new SetupNodeStatus(node, node.NodeDefinition));
}
this.Hosts = new List <SetupNodeStatus>();
foreach (var host in cluster.Hosts)
{
Hosts.Add(new SetupNodeStatus(host, new object()));
}
// Initialize the setup steps.
this.Steps = new List <SetupStepStatus>();
foreach (var step in controller.GetStepStatus().Where(step => !step.IsQuiet))
{
Steps.Add(step);
}
this.CurrentStep = Steps.SingleOrDefault(step => step.Number == controller.CurrentStepNumber);
}
/// <summary>
/// Performs common node configuration.
/// </summary>
/// <param name="controller">The setup controller.</param>
/// <param name="clusterManifest">The cluster manifest.</param>
public void SetupNode(ISetupController controller, ClusterManifest clusterManifest)
{
Covenant.Requires <ArgumentNullException>(controller != null, nameof(controller));
Covenant.Requires <ArgumentNullException>(clusterManifest != null, nameof(clusterManifest));
var nodeDefinition = NeonHelper.CastTo <NodeDefinition>(Metadata);
var clusterDefinition = Cluster.Definition;
var hostingManager = controller.Get <IHostingManager>(KubeSetupProperty.HostingManager);
InvokeIdempotent("setup/node",
() =>
{
PrepareNode(controller);
ConfigureEnvironmentVariables(controller);
SetupPackageProxy(controller);
UpdateHostname(controller);
NodeInitialize(controller);
NodeInstallCriO(controller, clusterManifest);
NodeInstallIPVS(controller);
NodeInstallPodman(controller);
NodeInstallKubernetes(controller);
SetupKublet(controller);
});
}
/// <summary>
/// <para>
/// Connects to a Kubernetes cluster if it already exists. This sets the <see cref="KubeSetupProperty.K8sClient"/>
/// property in the setup controller state when Kubernetes is running and a connection has not already
/// been established.
/// </para>
/// <note>
/// The <see cref="KubeSetupProperty.K8sClient"/> will not be set when Kubernetes has not been started, so
/// <see cref="ObjectDictionary.Get{TValue}(string)"/> calls for this property will fail when the
/// cluster has not been connected yet, which will be useful for debugging setup steps that require
/// a connection but this hasn't happened yet.
/// </note>
/// </summary>
/// <param name="controller">The setup controller.</param>
public static void ConnectCluster(ISetupController controller)
{
Covenant.Requires <ArgumentNullException>(controller != null, nameof(controller));
if (controller.ContainsKey(KubeSetupProperty.K8sClient))
{
return; // Already connected
}
var cluster = controller.Get <ClusterProxy>(KubeSetupProperty.ClusterProxy);
var configFile = GetCurrentKubeConfigPath();
if (!string.IsNullOrEmpty(configFile) && File.Exists(configFile))
{
// We're using a generated wrapper class to handle transient retries rather than
// modifying the built-in base retry policy. We're really just trying to handle
// the transients that happen during setup when the API server is unavailable for
// some reaon (like it's being restarted).
var k8s = new KubernetesWithRetry(KubernetesClientConfiguration.BuildConfigFromConfigFile(configFile, currentContext: cluster.KubeContext.Name));
k8s.RetryPolicy =
new ExponentialRetryPolicy(
transientDetector:
exception =>
{
var exceptionType = exception.GetType();
// Exceptions like this happen when a API server connection can't be established
// because the server isn't running or ready.
if (exceptionType == typeof(HttpRequestException) && exception.InnerException != null && exception.InnerException.GetType() == typeof(SocketException))
{
return(true);
}
var httpOperationException = exception as HttpOperationException;
if (httpOperationException != null)
{
var statusCode = httpOperationException.Response.StatusCode;
switch (statusCode)
{
case HttpStatusCode.GatewayTimeout:
case HttpStatusCode.InternalServerError:
case HttpStatusCode.RequestTimeout:
case HttpStatusCode.ServiceUnavailable:
case (HttpStatusCode)423: // Locked
case (HttpStatusCode)429: // Too many requests
return(true);
}
}
// This might be another variant of the check just above. This looks like an SSL negotiation problem.
if (exceptionType == typeof(HttpRequestException) && exception.InnerException != null && exception.InnerException.GetType() == typeof(IOException))
{
return(true);
}
return(false);
},
maxAttempts: int.MaxValue,
initialRetryInterval: TimeSpan.FromSeconds(1),
maxRetryInterval: TimeSpan.FromSeconds(5),
timeout: TimeSpan.FromMinutes(5));
controller.Add(KubeSetupProperty.K8sClient, k8s);
}
}
/// <summary>
/// Installs a prepositioned Helm chart from a control-plane node.
/// </summary>
/// <param name="controller">The setup controller.</param>
/// <param name="chartName">
/// <para>
/// The name of the Helm chart.
/// </para>
/// <note>
/// Helm does not allow dashes <b>(-)</b> in chart names but to avoid problems
/// with copy/pasting, we will automatically convert any dashes to underscores
/// before installing the chart. This is also nice because this means that the
/// chart name passed can be the same as the release name in the calling code.
/// </note>
/// </param>
/// <param name="releaseName">Optionally specifies the component release name.</param>
/// <param name="namespace">Optionally specifies the namespace where Kubernetes namespace where the Helm chart should be installed. This defaults to <b>default</b></param>
/// <param name="prioritySpec">
/// <para>
/// Optionally specifies the Helm variable and priority class for any pods deployed by the chart.
/// This needs to be specified as: <b>PRIORITYCLASSNAME</b> or <b>VALUENAME=PRIORITYCLASSNAME</b>,
/// where <b>VALUENAME</b> optionally specifies the name of the Helm value and <b>PRIORITYCLASSNAME</b>
/// is one of the priority class names defined by <see cref="PriorityClass"/>.
/// </para>
/// <note>
/// The priority class will saved as the <b>priorityClassName</b> Helm value when no value
/// name is specified.
/// </note>
/// </param>
/// <param name="values">Optionally specifies Helm chart values.</param>
/// <param name="progressMessage">Optionally specifies progress message. This defaults to <paramref name="releaseName"/>.</param>
/// <returns>The tracking <see cref="Task"/>.</returns>
/// <exception cref="KeyNotFoundException">Thrown if the priority class specified by <paramref name="prioritySpec"/> is not defined by <see cref="PriorityClass"/>.</exception>
/// <remarks>
/// neonKUBE images prepositions the Helm chart files embedded as resources in the <b>Resources/Helm</b>
/// project folder to cluster node images as the <b>/lib/neonkube/helm/charts.zip</b> archive. This
/// method unzips that file to the same folder (if it hasn't been unzipped already) and then installs
/// the helm chart (if it hasn't already been installed).
/// </remarks>
public async Task InstallHelmChartAsync(
ISetupController controller,
string chartName,
string releaseName = null,
string @namespace = "default",
string prioritySpec = null,
Dictionary <string, object> values = null,
string progressMessage = null)
{
await SyncContext.Clear;
Covenant.Requires <ArgumentNullException>(controller != null, nameof(controller));
Covenant.Requires <ArgumentNullException>(!string.IsNullOrEmpty(chartName), nameof(chartName));
chartName = chartName.Replace('-', '_');
if (string.IsNullOrEmpty(releaseName))
{
releaseName = chartName.Replace("_", "-");
}
// Extract the Helm chart value name and priority class name from [priorityClass]
// when passed.
string priorityClassVariable = null;
string priorityClassName = null;
if (!string.IsNullOrEmpty(prioritySpec))
{
var equalPos = prioritySpec.IndexOf('=');
if (equalPos == -1)
{
priorityClassVariable = "priorityClassName";
priorityClassName = prioritySpec;
}
else
{
priorityClassVariable = prioritySpec.Substring(0, equalPos).Trim();
priorityClassName = prioritySpec.Substring(equalPos + 1).Trim();
if (string.IsNullOrEmpty(priorityClassVariable) || string.IsNullOrEmpty(priorityClassName))
{
throw new FormatException($"[{prioritySpec}] is not valid. This must be formatted like: NAME=PRIORITYCLASSNAME");
}
}
PriorityClass.EnsureKnown(priorityClassName);
}
// Unzip the Helm chart archive if we haven't done so already.
InvokeIdempotent("setup/helm-unzip",
() =>
{
controller.LogProgress(this, verb: "unzip", message: "helm charts");
var zipPath = LinuxPath.Combine(KubeNodeFolder.Helm, "charts.zip");
SudoCommand($"unzip -o {zipPath} -d {KubeNodeFolder.Helm} || true");
SudoCommand($"rm -f {zipPath}");
});
// Install the chart when we haven't already done so.
InvokeIdempotent($"setup/helm-install-{releaseName}",
() =>
{
controller.LogProgress(this, verb: "install", message: progressMessage ?? releaseName);
var valueOverrides = new StringBuilder();
if (!string.IsNullOrEmpty(priorityClassVariable))
{
valueOverrides.AppendWithSeparator($"--set {priorityClassVariable}={priorityClassName}");
}
if (values != null)
{
foreach (var value in values)
{
if (value.Value == null)
{
valueOverrides.AppendWithSeparator($"--set {value.Key}=null");
continue;
}
var valueType = value.Value.GetType();
switch (value.Value)
{
case string s:
valueOverrides.AppendWithSeparator($"--set-string {value.Key}=\"{value.Value}\"");
break;
case Boolean b:
valueOverrides.AppendWithSeparator($"--set {value.Key}=\"{value.Value.ToString().ToLower()}\"");
break;
default:
valueOverrides.AppendWithSeparator($"--set {value.Key}={value.Value}");
break;
}
}
}
var helmChartScript = new StringBuilder();
helmChartScript.AppendLineLinux(
$@"
set -euo pipefail
cd {KubeNodeFolder.Helm}
");
if (controller.Get <bool>(KubeSetupProperty.MaintainerMode))
{
helmChartScript.AppendLineLinux(
$@"
if `helm list --namespace {@namespace} | awk '{{print $1}}' | grep -q ""^{releaseName}$""`; then
helm uninstall {releaseName} --namespace {@namespace}
fi
");
}
helmChartScript.AppendLineLinux(
$@"
helm install {releaseName} --namespace {@namespace} -f {chartName}/values.yaml {valueOverrides} ./{chartName}
START=`date +%s`
DEPLOY_END=$((START+15))
set +e
until [ `helm status {releaseName} --namespace {@namespace} | grep ""STATUS: deployed"" | wc -l` -eq 1 ];
do
if [ $((`date +%s`)) -gt $DEPLOY_END ]; then
helm uninstall {releaseName} --namespace {@namespace} || true
exit 1
fi
sleep 1
done
");
var scriptString = helmChartScript.ToString();
SudoCommand(CommandBundle.FromScript(helmChartScript), RunOptions.FaultOnError).EnsureSuccess();
});
await Task.CompletedTask;
}
/// <summary>
/// Configures cluster package manager caching.
/// </summary>
/// <param name="controller">The setup controller.</param>
public void SetupPackageProxy(ISetupController controller)
{
Covenant.Requires <ArgumentNullException>(controller != null, nameof(controller));
var nodeDefinition = NeonHelper.CastTo <NodeDefinition>(Metadata);
var clusterDefinition = Cluster.Definition;
var hostingManager = controller.Get <IHostingManager>(KubeSetupProperty.HostingManager);
InvokeIdempotent("setup/package-caching",
() =>
{
controller.LogProgress(this, verb: "configure", message: "apt package proxy");
// Configure the [apt-cacher-ng] pckage proxy service on control-plane nodes.
if (NodeDefinition.Role == NodeRole.ControlPlane)
{
var proxyServiceScript =
$@"
set -eou pipefail # Enable full failure detection
{KubeNodeFolder.Bin}/safe-apt-get update
{KubeNodeFolder.Bin}/safe-apt-get install -yq apt-cacher-ng
# Configure the cache to pass-thru SSL requests
# and then restart.
echo ""PassThroughPattern:^.*:443$"" >> /etc/apt-cacher-ng/acng.conf
systemctl restart apt-cacher-ng
set -eo pipefail # Revert back to partial failure detection
# Give the proxy service a chance to start.
sleep 5
";
SudoCommand(CommandBundle.FromScript(proxyServiceScript), RunOptions.FaultOnError);
}
var sbPackageProxies = new StringBuilder();
if (clusterDefinition.PackageProxy != null)
{
foreach (var proxyEndpoint in clusterDefinition.PackageProxy.Split(' ', StringSplitOptions.RemoveEmptyEntries))
{
sbPackageProxies.AppendWithSeparator(proxyEndpoint);
}
}
// Configure the package manager to use the first control-plane as the proxy by default,
// failing over to the other control-plane nodes (in order) when necessary.
var proxySelectorScript =
$@"
# Configure APT proxy selection.
echo {sbPackageProxies} > {KubeNodeFolder.Config}/package-proxy
cat <<EOF > /usr/local/bin/get-package-proxy
#!/bin/bash
#------------------------------------------------------------------------------
# FILE: get-package-proxy
# CONTRIBUTOR: Generated by [neon-cli] during cluster setup.
#
# This script determine which (if any) configured APT proxy caches are running
# and returns its endpoint or ""DIRECT"" if none of the proxies are available and
# the distribution's mirror should be accessed directly. This uses the
# [{KubeNodeFolder.Config}/package-proxy] file to obtain the list of proxies.
#
# This is called when the following is specified in the APT configuration,
# as we do further below:
#
# Acquire::http::Proxy-Auto-Detect ""/usr/local/bin/get-package-proxy"";
#
# See this link for more information:
#
# https://trent.utfs.org/wiki/Apt-get#Failover_Proxy
NEON_PACKAGE_PROXY=$(cat {KubeNodeFolder.Config}/package-proxy)
if [ ""\${{NEON_PACKAGE_PROXY}}"" == """" ] ; then
echo DIRECT
exit 0
fi
for proxy in ${{NEON_PACKAGE_PROXY}}; do
if nc -w1 -z \${{proxy/:/ }}; then
echo http://\${{proxy}}/
exit 0
fi
done
echo DIRECT
exit 0
EOF
chmod 775 /usr/local/bin/get-package-proxy
cat <<EOF > /etc/apt/apt.conf
//-----------------------------------------------------------------------------
// FILE: /etc/apt/apt.conf
// CONTRIBUTOR: Generated by during neonKUBE cluster setup.
//
// This file configures APT on the local machine to proxy requests through the
// [apt-cacher-ng] instance(s) at the configured. This uses the [/usr/local/bin/get-package-proxy]
// script to select a working PROXY if there are more than one, or to go directly to the package
// mirror if none of the proxies are available.
//
// Presumably, this cache is running on the local network which can dramatically
// reduce external network traffic to the APT mirrors and improve cluster setup
// and update performance.
Acquire::http::Proxy-Auto-Detect ""/usr/local/bin/get-package-proxy"";
EOF
";
SudoCommand(CommandBundle.FromScript(proxySelectorScript), RunOptions.FaultOnError);
});
}