/// <summary>
/// Disables <b>cloud-init</b>.
/// </summary>
/// <param name="controller">The setup controller.</param>
public void BaseDisableCloudInit(ISetupController controller)
{
Covenant.Requires <ArgumentException>(controller != null, nameof(controller));
var hostingEnvironment = controller.Get <HostingEnvironment>(KubeSetupProperty.HostingEnvironment);
// Do this only for non-cloud environments.
if (KubeHelper.IsCloudEnvironment(hostingEnvironment))
{
return;
}
InvokeIdempotent("base/cloud-init",
() =>
{
controller.LogProgress(this, verb: "disable", message: "cloud-init");
var disableCloudInitScript =
$@"
set -euo pipefail
mkdir -p /etc/cloud
touch /etc/cloud/cloud-init.disabled
";
SudoCommand(CommandBundle.FromScript(disableCloudInitScript), RunOptions.Defaults | RunOptions.FaultOnError);
});
}
/// <summary>
/// Removes unneeded packages.
/// </summary>
/// <param name="controller">The setup controller.</param>
public void BaseRemovePackages(ISetupController controller)
{
Covenant.Requires <ArgumentException>(controller != null, nameof(controller));
InvokeIdempotent("base/remove-packages",
() =>
{
controller.LogProgress(this, verb: "remove", message: "unneeded packages");
// Purge unneeded packages.
var removePackagesScript =
$@"
set -euo pipefail
{KubeNodeFolder.Bin}/safe-apt-get purge -y \
git git-man \
iso-codes \
locales \
manpages man-db \
python3-twisted \
snapd \
vim vim-runtime vim-tiny
{KubeNodeFolder.Bin}/safe-apt-get autoremove -y
";
SudoCommand(CommandBundle.FromScript(removePackagesScript), RunOptions.Defaults | RunOptions.FaultOnError);
});
}
/// <summary>
/// Installs hypervisor guest integration services.
/// </summary>
/// <param name="controller">The setup controller.</param>
public void BaseInstallGuestIntegrationServices(ISetupController controller)
{
Covenant.Requires <ArgumentException>(controller != null, nameof(controller));
var hostingEnvironment = controller.Get <HostingEnvironment>(KubeSetupProperty.HostingEnvironment);
// This currently applies only to on-premise hypervisors.
if (!KubeHelper.IsOnPremiseHypervisorEnvironment(hostingEnvironment))
{
return;
}
InvokeIdempotent("base/guest-integration",
() =>
{
controller.LogProgress(this, verb: "setup", message: "guest integration services");
var guestServicesScript =
$@"#!/bin/bash
set -euo pipefail
cat <<EOF >> /etc/initramfs-tools/modules
hv_vmbus
hv_storvsc
hv_blkvsc
hv_netvsc
EOF
{KubeNodeFolder.Bin}/safe-apt-get install -yq linux-virtual linux-cloud-tools-virtual linux-tools-virtual
update-initramfs -u
";
SudoCommand(CommandBundle.FromScript(guestServicesScript), RunOptions.Defaults | RunOptions.FaultOnError);
});
}
/// <summary>
/// Creates a bundle that simply uploads and runs a (<c>string</c>) script.
/// </summary>
/// <param name="script">The script text.</param>
/// <returns>The <see cref="CommandBundle"/>.</returns>
public static CommandBundle FromScript(string script)
{
var bundle = new CommandBundle("./script.sh");
bundle.AddFile("script.sh", script, isExecutable: true);
return(bundle);
}
/// <summary>
/// Constructs a configuration step that executes a command under <b>sudo</b>
/// on a specific cluster node.
/// </summary>
/// <param name="nodeName">The node name.</param>
/// <param name="command">The Linux command.</param>
/// <param name="args">The command arguments.</param>
/// <remarks>
/// <note>
/// You can add <see cref="CommandBundle.ArgBreak"/> as one of the arguments. This is
/// a meta argument that indicates that the following non-command line option
/// is not to be considered to be the value for the previous command line option.
/// This is a formatting hint for <see cref="ToBash(string)"/> and will
/// not be included in the command itself.
/// </note>
/// </remarks>
private CommandStep(string nodeName, string command, params object[] args)
{
Covenant.Requires <ArgumentNullException>(!string.IsNullOrEmpty(nodeName), nameof(nodeName));
Covenant.Requires <ArgumentNullException>(!string.IsNullOrEmpty(command), nameof(command));
Covenant.Requires <ArgumentNullException>(args != null, nameof(args));
this.nodeName = nodeName;
this.commandBundle = new CommandBundle(command, args);
}
/// <summary>
/// Configures the Debian frontend terminal to non-interactive.
/// </summary>
/// <param name="controller">The setup controller.</param>
public void BaseConfigureBashEnvironment(ISetupController controller)
{
Covenant.Requires <ArgumentException>(controller != null, nameof(controller));
InvokeIdempotent("base/bash-environment",
() =>
{
controller.LogProgress(this, verb: "configure", message: "environment variables");
var script =
@"
set -euo pipefail
echo '. /etc/environment' > /etc/profile.d/env.sh
";
SudoCommand(CommandBundle.FromScript(script), RunOptions.Defaults | RunOptions.FaultOnError);
});
}
/// <inheritdoc/>
public override string ToString()
{
var sb = new StringBuilder();
sb.Append($"{commandBundle.Command}");
foreach (var arg in CommandBundle.NormalizeArgs(commandBundle.Args))
{
sb.AppendWithSeparator(arg);
}
if (commandBundle.Count > 0)
{
sb.Append($" [files={commandBundle.Count}]");
}
return(sb.ToString());
}
/// <summary>
/// Ubuntu defaults DNS to prefer IPv6 lookups over IPv4 which can cause
/// performance problems. This method reconfigures DNS to favor IPv4.
/// </summary>
/// <param name="controller">The setup controller.</param>
public void BaseConfigureDnsIPv4Preference(ISetupController controller)
{
Covenant.Requires <ArgumentException>(controller != null, nameof(controller));
InvokeIdempotent("base/dns-ipv4",
() =>
{
controller.LogProgress(this, verb: "configure", message: "dns ipv4 preference");
var script =
@"
set -euo pipefail
#------------------------------------------------------------------------------
# We need to modify how [getaddressinfo] handles DNS lookups
# so that IPv4 lookups are preferred over IPv6. Ubuntu prefers
# IPv6 lookups by default. This can cause performance problems
# because in most situations right now, the server would be doing
# 2 DNS queries, one for AAAA (IPv6) which will nearly always
# fail (at least until IPv6 is more prevalent) and then querying
# for the for A (IPv4) record.
#
# This can also cause issues when the server is behind a NAT.
# I ran into a situation where [apt-get update] started failing
# because one of the archives had an IPv6 address in addition to
# an IPv4. Here's a note about this issue:
#
# http://ubuntuforums.org/showthread.php?t=2282646
#
# We're going to uncomment the line below in [gai.conf] and
# change it to the following line to prefer IPv4.
#
# #precedence ::ffff:0:0/96 10
# precedence ::ffff:0:0/96 100
#
# Note that this does not completely prevent the resolver from
# returning IPv6 addresses. You'll need to prvent this on an
# application by application basis, like using the [curl -4] option.
sed -i 's!^#precedence ::ffff:0:0/96 10$!precedence ::ffff:0:0/96 100!g' /etc/gai.conf
";
SudoCommand(CommandBundle.FromScript(script), RunOptions.Defaults | RunOptions.FaultOnError);
});
}
/// <summary>
/// Removes any installed snaps as well as the entire snap infrastructure.
/// </summary>
/// <param name="controller">The setup controller.</param>
public void BaseRemoveSnap(ISetupController controller)
{
Covenant.Requires <ArgumentException>(controller != null, nameof(controller));
InvokeIdempotent("base/remove-snap",
() =>
{
controller.LogProgress(this, verb: "remove", message: "snap");
var script =
@"
set -euo pipefail
safe-apt-get purge snapd -yq
rm -rf ~/snap
rm -rf /var/cache/snapd
rm -rf /snap
";
SudoCommand(CommandBundle.FromScript(script), RunOptions.Defaults | RunOptions.FaultOnError);
});
}
/// <summary>
/// Create the node folders required by neoneKUBE.
/// </summary>
/// <param name="controller">The setup controller.</param>
public void BaseCreateKubeFolders(ISetupController controller)
{
Covenant.Requires <ArgumentException>(controller != null, nameof(controller));
InvokeIdempotent("base/folders",
() =>
{
controller.LogProgress(this, verb: "create", message: "node folders");
var folderScript =
$@"
set -euo pipefail
mkdir -p {KubeNodeFolder.Bin}
chmod 750 {KubeNodeFolder.Bin}
mkdir -p {KubeNodeFolder.Config}
chmod 750 {KubeNodeFolder.Config}
mkdir -p {KubeNodeFolder.Setup}
chmod 750 {KubeNodeFolder.Setup}
mkdir -p {KubeNodeFolder.Helm}
chmod 750 {KubeNodeFolder.Helm}
mkdir -p {KubeNodeFolder.State}
chmod 750 {KubeNodeFolder.State}
mkdir -p {KubeNodeFolder.State}/setup
chmod 750 {KubeNodeFolder.State}/setup
mkdir -p {KubeNodeFolder.NeonRun}
chmod 740 {KubeNodeFolder.NeonRun}
";
SudoCommand(CommandBundle.FromScript(folderScript), RunOptions.Defaults | RunOptions.FaultOnError);
});
}
/// <summary>
/// Disables DHCP.
/// </summary>
/// <param name="controller">The setup controller.</param>
public void BaseDisableDhcp(ISetupController controller)
{
Covenant.Requires <ArgumentException>(controller != null, nameof(controller));
var hostingEnvironment = controller.Get <HostingEnvironment>(KubeSetupProperty.HostingEnvironment);
InvokeIdempotent("base/dhcp",
() =>
{
controller.LogProgress(this, verb: "disable", message: "dhcp");
var initNetPlanScript =
$@"
set -euo pipefail
rm -rf /etc/netplan/*
cat <<EOF > /etc/netplan/no-dhcp.yaml
# This file is used to disable the network when a new VM is created
# from a template is booted. The [neon-init] service handles network
# provisioning in conjunction with the cluster prepare step.
#
# Cluster prepare inserts a virtual DVD disc with a script that
# handles the network configuration which [neon-init] will
# execute.
network:
version: 2
renderer: networkd
ethernets:
eth0:
dhcp4: no
EOF
";
SudoCommand(CommandBundle.FromScript(initNetPlanScript), RunOptions.Defaults | RunOptions.FaultOnError);
});
}
/// <summary>
/// Configures cluster package manager caching.
/// </summary>
/// <param name="controller">The setup controller.</param>
public void SetupPackageProxy(ISetupController controller)
{
Covenant.Requires <ArgumentNullException>(controller != null, nameof(controller));
var nodeDefinition = NeonHelper.CastTo <NodeDefinition>(Metadata);
var clusterDefinition = Cluster.Definition;
var hostingManager = controller.Get <IHostingManager>(KubeSetupProperty.HostingManager);
InvokeIdempotent("setup/package-caching",
() =>
{
controller.LogProgress(this, verb: "configure", message: "apt package proxy");
// Configure the [apt-cacher-ng] pckage proxy service on control-plane nodes.
if (NodeDefinition.Role == NodeRole.ControlPlane)
{
var proxyServiceScript =
$@"
set -eou pipefail # Enable full failure detection
{KubeNodeFolder.Bin}/safe-apt-get update
{KubeNodeFolder.Bin}/safe-apt-get install -yq apt-cacher-ng
# Configure the cache to pass-thru SSL requests
# and then restart.
echo ""PassThroughPattern:^.*:443$"" >> /etc/apt-cacher-ng/acng.conf
systemctl restart apt-cacher-ng
set -eo pipefail # Revert back to partial failure detection
# Give the proxy service a chance to start.
sleep 5
";
SudoCommand(CommandBundle.FromScript(proxyServiceScript), RunOptions.FaultOnError);
}
var sbPackageProxies = new StringBuilder();
if (clusterDefinition.PackageProxy != null)
{
foreach (var proxyEndpoint in clusterDefinition.PackageProxy.Split(' ', StringSplitOptions.RemoveEmptyEntries))
{
sbPackageProxies.AppendWithSeparator(proxyEndpoint);
}
}
// Configure the package manager to use the first control-plane as the proxy by default,
// failing over to the other control-plane nodes (in order) when necessary.
var proxySelectorScript =
$@"
# Configure APT proxy selection.
echo {sbPackageProxies} > {KubeNodeFolder.Config}/package-proxy
cat <<EOF > /usr/local/bin/get-package-proxy
#!/bin/bash
#------------------------------------------------------------------------------
# FILE: get-package-proxy
# CONTRIBUTOR: Generated by [neon-cli] during cluster setup.
#
# This script determine which (if any) configured APT proxy caches are running
# and returns its endpoint or ""DIRECT"" if none of the proxies are available and
# the distribution's mirror should be accessed directly. This uses the
# [{KubeNodeFolder.Config}/package-proxy] file to obtain the list of proxies.
#
# This is called when the following is specified in the APT configuration,
# as we do further below:
#
# Acquire::http::Proxy-Auto-Detect ""/usr/local/bin/get-package-proxy"";
#
# See this link for more information:
#
# https://trent.utfs.org/wiki/Apt-get#Failover_Proxy
NEON_PACKAGE_PROXY=$(cat {KubeNodeFolder.Config}/package-proxy)
if [ ""\${{NEON_PACKAGE_PROXY}}"" == """" ] ; then
echo DIRECT
exit 0
fi
for proxy in ${{NEON_PACKAGE_PROXY}}; do
if nc -w1 -z \${{proxy/:/ }}; then
echo http://\${{proxy}}/
exit 0
fi
done
echo DIRECT
exit 0
EOF
chmod 775 /usr/local/bin/get-package-proxy
cat <<EOF > /etc/apt/apt.conf
//-----------------------------------------------------------------------------
// FILE: /etc/apt/apt.conf
// CONTRIBUTOR: Generated by during neonKUBE cluster setup.
//
// This file configures APT on the local machine to proxy requests through the
// [apt-cacher-ng] instance(s) at the configured. This uses the [/usr/local/bin/get-package-proxy]
// script to select a working PROXY if there are more than one, or to go directly to the package
// mirror if none of the proxies are available.
//
// Presumably, this cache is running on the local network which can dramatically
// reduce external network traffic to the APT mirrors and improve cluster setup
// and update performance.
Acquire::http::Proxy-Auto-Detect ""/usr/local/bin/get-package-proxy"";
EOF
";
SudoCommand(CommandBundle.FromScript(proxySelectorScript), RunOptions.FaultOnError);
});
}
/// <summary>
/// Installs the <b>neon-init</b> service which is a poor man's cloud-init like
/// service we use to configure the network and credentials for VMs hosted in non-cloud
/// hypervisors.
/// </summary>
/// <param name="controller">The setup controller.</param>
/// <remarks>
/// <para>
/// Install and configure the [neon-init] service. This is a simple script
/// that is configured to run as a oneshot systemd service before networking is
/// started. This is currently used to configure the node's static IP address
/// configuration on first boot, so we don't need to rely on DHCP (which may not
/// be available in some environments).
/// </para>
/// <para>
/// [neon-init] is intended to run the first time a node is booted after
/// being created from a template. It checks to see if a special ISO with a
/// configuration script named [neon-init.sh] is inserted into the VMs DVD
/// drive and when present, the script will be executed and the [/etc/neon-init/ready]
/// file will be created to indicate that the service no longer needs to do this for
/// subsequent reboots.
/// </para>
/// <note>
/// The script won't create the [/etc/neon-init] when the script ISO doesn't exist
/// for debugging purposes.
/// </note>
/// </remarks>
public void BaseInstallNeonInit(ISetupController controller)
{
Covenant.Requires <ArgumentException>(controller != null, nameof(controller));
InvokeIdempotent("base/neon-init",
() =>
{
controller.LogProgress(this, verb: "setup", message: "neon-init.service");
var neonNodePrepScript =
$@"# Ensure that the neon binary folder exists.
mkdir -p {KubeNodeFolder.Bin}
# Create the systemd unit file.
cat <<EOF > /etc/systemd/system/neon-init.service
[Unit]
Description=neonKUBE one-time node preparation service
After=systemd-networkd.service
[Service]
Type=oneshot
ExecStart={KubeNodeFolder.Bin}/neon-init
RemainAfterExit=false
StandardOutput=journal+console
[Install]
WantedBy=multi-user.target
EOF
# Create the service script.
cat <<EOF > {KubeNodeFolder.Bin}/neon-init
#!/bin/bash
#------------------------------------------------------------------------------
# FILE: neon-init
# CONTRIBUTOR: Jeff Lill
# COPYRIGHT: Copyright (c) 2005-2022 by neonFORGE LLC. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the ""License"");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an ""AS IS"" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# This script is run early during node boot before the netork is configured
# as a poor man's way for neonKUBE cluster setup to configure the network
# without requiring DHCP. Here's how this works:
#
# 1. neonKUBE cluster setup creates a node VM from a template.
#
# 2. Setup creates a temporary ISO (DVD) image with a script named
# [neon-init.sh] on it and uploads this to the Hyper-V
# or XenServer host machine.
#
# 3. Setup inserts the VFD into the VM's DVD drive and starts the VM.
#
# 4. The VM boots, eventually running this script (via the
# [neon-init] service).
#
# 5. This script checks whether a DVD is present, mounts
# it and checks it for the [neon-init.sh] script.
#
# 6. If the DVD and script file are present, this service will
# execute the script via Bash, peforming any required custom setup.
# Then this script creates the [/etc/neon-init] file which
# prevents the service from doing anything during subsequent node
# reboots.
#
# 7. The service just exits if the DVD and/or script file are
# not present. This shouldn't happen in production but is useful
# for script debugging.
#
# NOTE: Ubuntu 22.04 seems to have removed the [/dev/dvd] device but
# [/dev/cdrom] works, su we're swiching to that.
# Run the prep script only once.
if [ -f /etc/neon-init/ready ] ; then
echo ""INFO: Machine is already ready.""
exit 0
fi
# Create a mount point for the DVD.
if ! mkdir -p /media/neon-init ; then
echo ""ERROR: Cannot create DVD mount point.""
exit 1
fi
# Wait up to 120 seconds for for the DVD to be discovered. It can
# take some time for this to happen.
mounted=$false
for i in {{1..24}}; do
# Sleep for 5 seconds. We're doing this first to give Linux
# a chance to discover the DVD and then this will act as a
# retry interval. 24 iterations at 5 seconds each is 120 seconds.
sleep 5
# Try mounting the DVD.
if mount /dev/cdrom /media/neon-init ; then
mounted=$true
break
fi
echo ""WARNING: No DVD is present (yet).""
done
if ! $mounted; then
echo ""WARNING: No DVD is present: exiting""
exit 1
fi
# Check for the [neon-init.sh] script and execute it.
if [ ! -f /media/neon-init/neon-init.sh ] ; then
echo ""WARNING: No [neon-init.sh] script is present on the DVD: exiting""
rm -rf /media/neon-init
exit 0
fi
# The script file is present so execute it. Note that we're
# passing the path where the DVD is mounted as a parameter.
echo ""INFO: Running [neon-init.sh]""
bash /media/neon-init/neon-init.sh /media/neon-init
# Unmount the DVD and cleanup.
echo ""INFO: Cleanup""
umount /media/neon-init
rm -rf /media/neon-init
# Disable [neon-init] so it does nothing the next time it's launched.
mkdir -p /etc/neon-init
touch /etc/neon-init/ready
EOF
chmod 744 {KubeNodeFolder.Bin}/neon-init
# Configure [neon-init] to start at boot.
systemctl enable neon-init
systemctl daemon-reload
# ---------------------------------------------------
";
SudoCommand(CommandBundle.FromScript(neonNodePrepScript), RunOptions.Defaults | RunOptions.FaultOnError);
});
}
/// <summary>
/// Configures the APT package manager.
/// </summary>
/// <param name="controller">The setup controller.</param>
/// <param name="packageManagerRetries">Optionally specifies the packager manager retries (defaults to <b>5</b>).</param>
/// <param name="allowPackageManagerIPv6">Optionally prevent the package manager from using IPv6 (defaults to <c>false</c>.</param>
public void BaseConfigureApt(ISetupController controller, int packageManagerRetries = 5, bool allowPackageManagerIPv6 = false)
{
Covenant.Requires <ArgumentException>(controller != null, nameof(controller));
InvokeIdempotent("base/apt",
() =>
{
controller.LogProgress(this, verb: "configure", message: "package manager");
if (!allowPackageManagerIPv6)
{
// Restrict the [apt] package manager to using IPv4 to communicate
// with the package mirrors, since IPv6 doesn't work sometimes.
UploadText("/etc/apt/apt.conf.d/99-force-ipv4-transport", "Acquire::ForceIPv4 \"true\";");
SudoCommand("chmod 644 /etc/apt/apt.conf.d/99-force-ipv4-transport", RunOptions.Defaults | RunOptions.FaultOnError);
}
// Configure [apt] to retry.
UploadText("/etc/apt/apt.conf.d/99-retries", $"APT::Acquire::Retries \"{packageManagerRetries}\";");
SudoCommand("chmod 644 /etc/apt/apt.conf.d/99-retries", RunOptions.Defaults | RunOptions.FaultOnError);
// We're going to disable apt updating services so we can control when this happens.
var disableAptServices =
@"
set -euo pipefail
#------------------------------------------------------------------------------
# Disable the [apt-timer] and [apt-daily] services. We're doing this
# for two reasons:
#
# 1. These services interfere with with [apt-get] usage during
# cluster setup and is also likely to interfere with end-user
# configuration activities as well.
#
# 2. Automatic updates for production and even test clusters is
# just not a great idea. You just don't want a random update
# applied in the middle of the night which might cause trouble.
#
# We're going to implement our own cluster updating machanism
# that will be smart enough to update the nodes such that the
# impact on cluster workloads will be limited.
systemctl stop apt-daily.timer
systemctl mask apt-daily.timer
systemctl stop apt-daily.service
systemctl mask apt-daily.service
# It may be possible for the auto updater to already be running so we'll
# wait here for it to release any lock files it holds.
while fuser /var/lib/dpkg/lock >/dev/null 2>&1; do
sleep 1
done
#------------------------------------------------------------------------------
# Update the APT config to disable updates there as well.
cat > /etc/apt/apt.conf.d/20auto-upgrades <<EOF
APT::Periodic::Update-Package-Lists ""0"";
APT::Periodic::Download-Upgradeable-Packages ""0"";
APT::Periodic::AutocleanInterval ""0"";
APT::Periodic::Unattended-Upgrade ""1"";
EOF
";
SudoCommand(CommandBundle.FromScript(disableAptServices), RunOptions.Defaults | RunOptions.FaultOnError);
});
}
/// <summary>
/// Configures the <b>kublet</b> service.
/// </summary>
/// <param name="controller">The setup controller.</param>
/// <remarks>
/// <note>
/// Kubelet is installed in <see cref="NodeSshProxy{TMetadata}.NodeInstallKubernetes"/> when configuring
/// the node image and is then configured for the cluster here.
/// </note>
/// </remarks>
public void SetupKublet(ISetupController controller)
{
Covenant.Requires <ArgumentNullException>(controller != null, nameof(controller));
InvokeIdempotent("setup/kublet",
() =>
{
controller.LogProgress(this, verb: "setup", message: "kublet");
// Configure the image GC thesholds. We'll stick with the defaults of 80/85% of the
// OS disk for most nodes and customize this at 93/95% for clusters with OS disks
// less than 64GB. We want higher thesholds for smaller disks to leave more space
// for user images and local volumes, especially for the neonDESKTOP built-in cluster.
//
// We're going to use this command to retrieve the node's disk information:
//
// fdisk --list -o Device,Size,Type | grep ^/dev
//
// which will produce output like:
//
// /dev/sda1 1M BIOS boot
// /dev/sda2 128G Linux filesystem
//
// We're going to look for the line with "Linux filesystem" and and then extract and
// parse the size in the second column to decide which GC thresholds to use.
//
// $note(jefflill):
//
// This assumes that there's only one Linux filesystem for each cluster node which is
// currently the case for all neonKUBE clusters. The cStor disks are managed by OpenEBS
// and will not be reported as a file system. I'll add an assert to verify this to
// make this easier diagnose in the future if we decide to allow multiple file systems.
//
// $todo(jefflill):
//
// We're hardcoding this now based on the current node disk size but eventually it
// might make sense to add settings to the cluster definition so user can override
// this, perhaps customizing specfic nodes.
var imageLowGcThreshold = 80;
var imageHighGcThreshold = 85;
var diskSize = 0L;
var result = SudoCommand(CommandBundle.FromScript("fdisk --list -o Device,Size,Type | grep ^/dev")).EnsureSuccess();
using (var reader = new StringReader(result.OutputText))
{
var filesystemCount = 0;
foreach (var line in reader.Lines())
{
if (!line.EndsWith("Linux filesystem") && !line.EndsWith("Linux"))
{
continue;
}
filesystemCount++;
var fields = line.Split(' ', StringSplitOptions.RemoveEmptyEntries);
var sizeField = fields[1];
var sizeUnit = sizeField.Last();
var rawSize = decimal.Parse(sizeField.Substring(0, sizeField.Length - 1));
switch (sizeUnit)
{
case 'G':
diskSize = (long)(rawSize * ByteUnits.GibiBytes);
break;
case 'T':
diskSize = (long)(rawSize * ByteUnits.TebiBytes);
break;
default:
Covenant.Assert(false, $"Expecting partition size unit to be [G] or [T], not [{sizeUnit}].");
break;
}
}
Covenant.Assert(filesystemCount == 1, $"Expected exactly [1] Linux file system but are seeing [{filesystemCount}].");
}
if (diskSize < 64 * ByteUnits.GibiBytes)
{
imageLowGcThreshold = 93;
imageHighGcThreshold = 95;
}
var script =
$@"
set -euo pipefail
echo KUBELET_EXTRA_ARGS=--feature-gates=\""AllAlpha=false\"" --cgroup-driver=systemd --container-runtime-endpoint='unix:///var/run/crio/crio.sock' --runtime-request-timeout=5m --resolv-conf=/run/systemd/resolve/resolv.conf --image-gc-low-threshold={imageLowGcThreshold} --image-gc-high-threshold={imageHighGcThreshold} > /etc/default/kubelet
systemctl daemon-reload
systemctl restart kubelet
systemctl enable kubelet
";
SudoCommand(CommandBundle.FromScript(script), RunOptions.FaultOnError);
});
}
/// <summary>
/// Installs a prepositioned Helm chart from a control-plane node.
/// </summary>
/// <param name="controller">The setup controller.</param>
/// <param name="chartName">
/// <para>
/// The name of the Helm chart.
/// </para>
/// <note>
/// Helm does not allow dashes <b>(-)</b> in chart names but to avoid problems
/// with copy/pasting, we will automatically convert any dashes to underscores
/// before installing the chart. This is also nice because this means that the
/// chart name passed can be the same as the release name in the calling code.
/// </note>
/// </param>
/// <param name="releaseName">Optionally specifies the component release name.</param>
/// <param name="namespace">Optionally specifies the namespace where Kubernetes namespace where the Helm chart should be installed. This defaults to <b>default</b></param>
/// <param name="prioritySpec">
/// <para>
/// Optionally specifies the Helm variable and priority class for any pods deployed by the chart.
/// This needs to be specified as: <b>PRIORITYCLASSNAME</b> or <b>VALUENAME=PRIORITYCLASSNAME</b>,
/// where <b>VALUENAME</b> optionally specifies the name of the Helm value and <b>PRIORITYCLASSNAME</b>
/// is one of the priority class names defined by <see cref="PriorityClass"/>.
/// </para>
/// <note>
/// The priority class will saved as the <b>priorityClassName</b> Helm value when no value
/// name is specified.
/// </note>
/// </param>
/// <param name="values">Optionally specifies Helm chart values.</param>
/// <param name="progressMessage">Optionally specifies progress message. This defaults to <paramref name="releaseName"/>.</param>
/// <returns>The tracking <see cref="Task"/>.</returns>
/// <exception cref="KeyNotFoundException">Thrown if the priority class specified by <paramref name="prioritySpec"/> is not defined by <see cref="PriorityClass"/>.</exception>
/// <remarks>
/// neonKUBE images prepositions the Helm chart files embedded as resources in the <b>Resources/Helm</b>
/// project folder to cluster node images as the <b>/lib/neonkube/helm/charts.zip</b> archive. This
/// method unzips that file to the same folder (if it hasn't been unzipped already) and then installs
/// the helm chart (if it hasn't already been installed).
/// </remarks>
public async Task InstallHelmChartAsync(
ISetupController controller,
string chartName,
string releaseName = null,
string @namespace = "default",
string prioritySpec = null,
Dictionary <string, object> values = null,
string progressMessage = null)
{
await SyncContext.Clear;
Covenant.Requires <ArgumentNullException>(controller != null, nameof(controller));
Covenant.Requires <ArgumentNullException>(!string.IsNullOrEmpty(chartName), nameof(chartName));
chartName = chartName.Replace('-', '_');
if (string.IsNullOrEmpty(releaseName))
{
releaseName = chartName.Replace("_", "-");
}
// Extract the Helm chart value name and priority class name from [priorityClass]
// when passed.
string priorityClassVariable = null;
string priorityClassName = null;
if (!string.IsNullOrEmpty(prioritySpec))
{
var equalPos = prioritySpec.IndexOf('=');
if (equalPos == -1)
{
priorityClassVariable = "priorityClassName";
priorityClassName = prioritySpec;
}
else
{
priorityClassVariable = prioritySpec.Substring(0, equalPos).Trim();
priorityClassName = prioritySpec.Substring(equalPos + 1).Trim();
if (string.IsNullOrEmpty(priorityClassVariable) || string.IsNullOrEmpty(priorityClassName))
{
throw new FormatException($"[{prioritySpec}] is not valid. This must be formatted like: NAME=PRIORITYCLASSNAME");
}
}
PriorityClass.EnsureKnown(priorityClassName);
}
// Unzip the Helm chart archive if we haven't done so already.
InvokeIdempotent("setup/helm-unzip",
() =>
{
controller.LogProgress(this, verb: "unzip", message: "helm charts");
var zipPath = LinuxPath.Combine(KubeNodeFolder.Helm, "charts.zip");
SudoCommand($"unzip -o {zipPath} -d {KubeNodeFolder.Helm} || true");
SudoCommand($"rm -f {zipPath}");
});
// Install the chart when we haven't already done so.
InvokeIdempotent($"setup/helm-install-{releaseName}",
() =>
{
controller.LogProgress(this, verb: "install", message: progressMessage ?? releaseName);
var valueOverrides = new StringBuilder();
if (!string.IsNullOrEmpty(priorityClassVariable))
{
valueOverrides.AppendWithSeparator($"--set {priorityClassVariable}={priorityClassName}");
}
if (values != null)
{
foreach (var value in values)
{
if (value.Value == null)
{
valueOverrides.AppendWithSeparator($"--set {value.Key}=null");
continue;
}
var valueType = value.Value.GetType();
switch (value.Value)
{
case string s:
valueOverrides.AppendWithSeparator($"--set-string {value.Key}=\"{value.Value}\"");
break;
case Boolean b:
valueOverrides.AppendWithSeparator($"--set {value.Key}=\"{value.Value.ToString().ToLower()}\"");
break;
default:
valueOverrides.AppendWithSeparator($"--set {value.Key}={value.Value}");
break;
}
}
}
var helmChartScript = new StringBuilder();
helmChartScript.AppendLineLinux(
$@"
set -euo pipefail
cd {KubeNodeFolder.Helm}
");
if (controller.Get <bool>(KubeSetupProperty.MaintainerMode))
{
helmChartScript.AppendLineLinux(
$@"
if `helm list --namespace {@namespace} | awk '{{print $1}}' | grep -q ""^{releaseName}$""`; then
helm uninstall {releaseName} --namespace {@namespace}
fi
");
}
helmChartScript.AppendLineLinux(
$@"
helm install {releaseName} --namespace {@namespace} -f {chartName}/values.yaml {valueOverrides} ./{chartName}
START=`date +%s`
DEPLOY_END=$((START+15))
set +e
until [ `helm status {releaseName} --namespace {@namespace} | grep ""STATUS: deployed"" | wc -l` -eq 1 ];
do
if [ $((`date +%s`)) -gt $DEPLOY_END ]; then
helm uninstall {releaseName} --namespace {@namespace} || true
exit 1
fi
sleep 1
done
");
var scriptString = helmChartScript.ToString();
SudoCommand(CommandBundle.FromScript(helmChartScript), RunOptions.FaultOnError).EnsureSuccess();
});
await Task.CompletedTask;
}
/// <summary>
/// Configures NTP and also installs some tool scripts for managing this.
/// </summary>
/// <param name="controller">The setup controller.</param>
public void SetupConfigureNtp(ISetupController controller)
{
Covenant.Requires <ArgumentNullException>(controller != null, nameof(controller));
InvokeIdempotent("setup/ntp",
() =>
{
controller.LogProgress(this, verb: "configure", message: "ntp");
var clusterDefinition = this.Cluster.Definition;
var nodeDefinition = this.NodeDefinition;
var script =
$@"
mkdir -p /var/log/ntp
cat <<EOF > /etc/ntp.conf
# For more information about this file, see the man pages
# ntp.conf(5), ntp_acc(5), ntp_auth(5), ntp_clock(5), ntp_misc(5), ntp_mon(5).
# Specifies that NTP will implicitly trust the time source,
# even if it is way off from the current system time. This
# can happen when:
#
# 1. The BIOS time is set incorrectly
# 2. The machine is a VM that woke up from a long sleep
# 3. The time source is messed up
# 4. Somebody is screwing with us by sending bad time responses
#
# Normally, NTP is configured to panic when it sees a difference
# of 1000 seconds or more between the source and the system time
# and NTP will log an error and terminate.
#
# NTP is configured to start with the [-g] option, which will
# ignore 1000+ second differences when the service starts. This
# handles issues #1 and #2 above.
#
# The setting below disables the 1000 second check and also allows
# NTP to modify the system clock in steps of 1 second (normally,
# this is much smaller).
# \$todo(jefflill):
#
# This was the default setting when NTP was installed but I'm not
# entirely sure that this is what we'd want for production. The
# [man ntp.conf] page specifically recommends not messing with
# these defaults.
tinker panic 0 dispersion 1.000
# Specifies the file used to track the frequency offset of The
# local clock oscillator.
driftfile /var/lib/ntp/ntp.drift
# Configure a log file.
logfile /var/log/ntp/ntp.log
# Permit time synchronization with our time source, but do not
# permit the source to query or modify the service on this system.
restrict default kod nomodify notrap nopeer noquery
restrict -6 default kod nomodify notrap nopeer noquery
# Permit all access over the loopback interface. This could
# be tightened as well, but to do so would effect some of
# the administrative functions.
restrict 127.0.0.1
restrict -6 ::1
# Hosts on local networks are less restricted.
restrict 10.0.0.0 mask 255.0.0.0 nomodify notrap
restrict 169.254.0.0 mask 255.255.0.0 nomodify notrap
restrict 172.16.0.0 mask 255.255.0.0 nomodify notrap
restrict 192.168.0.0 mask 255.255.0.0 nomodify notrap
# Use local system time when the time sources are not reachable.
server 127.127.1.0 # local clock
fudge 127.127.1.0 stratum 10
# Specify the time sources.
EOF
# Append the time sources to the configuration file.
#
# We're going to prefer the first source. This will generally
# be the first control-plane node. The nice thing about this
# the worker nodes will be using the same time source on
# the local network, so the clocks should be very close to
# being closely synchronized.
sources=({GetNtpSources()})
echo ""*** TIME SOURCES = ${{sources}}"" 1>&2
preferred_set=false
for source in ""${{sources[@]}}"";
do
if ! ${{preferred_set}} ; then
echo ""server $source burst iburst prefer"" >> /etc/ntp.conf
preferred_set=true
else
echo ""server $source burst iburst"" >> /etc/ntp.conf
fi
done
# Generate the [{KubeNodeFolder.Bin}/update-time] script.
cat <<EOF > {KubeNodeFolder.Bin}/update-time
#!/bin/bash
#------------------------------------------------------------------------------
# This script stops the NTP time service and forces an immediate update. This
# is called from the NTP init.d script (as modified below) to ensure that we
# obtain the current time on boot (even if the system clock is way out of sync).
# This may also be invoked manually.
#
# Usage:
#
# update-time [--norestart]
#
# --norestart - Don't restart NTP
. $<load-cluster-conf-quiet>
restart=true
for arg in \$@
do
if [ ""\${{arg}}"" = ""--norestart"" ] ; then
restart=false
fi
done
if \${{restart}} ; then
service ntp stop
fi
ntpdate ${{sources[@]}}
if \${{restart}} ; then
service ntp start
fi
EOF
chmod 700 {KubeNodeFolder.Bin}/update-time
# Edit the NTP [/etc/init.d/ntp] script to initialize the hardware clock and
# call [update-time] before starting NTP.
cat <<""EOF"" > /etc/init.d/ntp
#!/bin/sh
### BEGIN INIT INFO
# Provides: ntp
# Required-Start: $network $remote_fs $syslog
# Required-Stop: $network $remote_fs $syslog
# Default-Start: 2 3 4 5
# Default-Stop: 1
# Short-Description: Start NTP daemon
### END INIT INFO
PATH=/sbin:/bin:/usr/sbin:/usr/bin
. /lib/lsb/init-functions
DAEMON=/usr/sbin/ntpd
PIDFILE=/var/run/ntpd.pid
test -x $DAEMON || exit 5
if [ -r /etc/default/ntp ]; then
. /etc/default/ntp
fi
if [ -e /var/lib/ntp/ntp.conf.dhcp ]; then
NTPD_OPTS=""$NTPD_OPTS -c /var/lib/ntp/ntp.conf.dhcp""
fi
LOCKFILE=/var/lock/ntpdate
lock_ntpdate() {{
if [ -x /usr/bin/lockfile-create ]; then
lockfile-create $LOCKFILE
lockfile-touch $LOCKFILE &
LOCKTOUCHPID=""$!""
fi
}}
unlock_ntpdate() {{
if [ -x /usr/bin/lockfile-create ] ; then
kill $LOCKTOUCHPID
lockfile-remove $LOCKFILE
fi
}}
RUNASUSER=ntp
UGID=$(getent passwd $RUNASUSER | cut -f 3,4 -d:) || true
if test ""$(uname -s)"" = ""Linux""; then
NTPD_OPTS=""$NTPD_OPTS -u $UGID""
fi
case $1 in
start)
log_daemon_msg ""Starting NTP server"" ""ntpd""
if [ -z ""$UGID"" ]; then
log_failure_msg ""user \""$RUNASUSER\"" does not exist""
exit 1
fi
#------------------------------
# This is the modification.
# This bit of voodoo disables Hyper-V time synchronization with The
# host server. We don't want this because the cluster is doing its
# own time management and time sync will fight us. This is described
# here:
#
# https://social.msdn.microsoft.com/Forums/en-US/8c0a1026-0b02-405a-848e-628e68229eaf/i-have-a-lot-of-time-has-been-changed-in-the-journal-of-my-linux-boxes?forum=WAVirtualMachinesforWindows
log_daemon_msg ""Start: Disabling Hyper-V time synchronization"" ""ntpd""
echo 2dd1ce17-079e-403c-b352-a1921ee207ee > /sys/bus/vmbus/drivers/hv_util/unbind
log_daemon_msg ""Finished: Disabling Hyper-V time synchronization"" ""ntpd""
log_daemon_msg ""Start: Updating current time"" ""ntpd""
{KubeNodeFolder.Bin}/update-time --norestart
log_daemon_msg ""Finished: Updating current time"" ""ntpd""
#------------------------------
lock_ntpdate
start-stop-daemon --start --quiet --oknodo --pidfile $PIDFILE --startas $DAEMON -- -p $PIDFILE $NTPD_OPTS
status=$?
unlock_ntpdate
log_end_msg $status
;;
stop)
log_daemon_msg ""Stopping NTP server"" ""ntpd""
start-stop-daemon --stop --quiet --oknodo --pidfile $PIDFILE
log_end_msg $?
rm -f $PIDFILE
;;
restart|force-reload)
$0 stop && sleep 2 && $0 start
;;
try-restart)
if $0 status >/dev/null; then
$0 restart
else
exit 0
fi
;;
reload)
exit 3
;;
status)
status_of_proc $DAEMON ""NTP server""
;;
*)
echo ""Usage: $0 {{start|stop|restart|try-restart|force-reload|status}}""
exit 2
;;
esac
EOF
# Restart NTP to ensure that we've picked up the current time.
service ntp restart
";
SudoCommand(CommandBundle.FromScript(script), RunOptions.Defaults | RunOptions.FaultOnError);
});
}
