/// <summary>
/// Initializes a near virgin server with the basic capabilities required
/// for a cluster host node.
/// </summary>
/// <param name="node">The target cluster node.</param>
/// <param name="clusterDefinition">The cluster definition.</param>
/// <param name="kubeSetupInfo">Kubernetes setup details.</param>
/// <param name="shutdown">Optionally shuts down the node.</param>
public static void PrepareNode(SshProxy <NodeDefinition> node, ClusterDefinition clusterDefinition, KubeSetupInfo kubeSetupInfo, bool shutdown = false)
{
Covenant.Requires <ArgumentNullException>(node != null);
Covenant.Requires <ArgumentNullException>(clusterDefinition != null);
Covenant.Requires <ArgumentNullException>(kubeSetupInfo != null);
if (node.FileExists($"{KubeHostFolders.State}/setup/prepared"))
{
return; // Already prepared
}
//-----------------------------------------------------------------
// Ensure that the cluster host folders exist.
node.CreateHostFolders();
//-----------------------------------------------------------------
// Package manager configuration.
if (!clusterDefinition.NodeOptions.AllowPackageManagerIPv6)
{
// Restrict the [apt] package manager to using IPv4 to communicate
// with the package mirrors, since IPv6 often doesn't work.
node.UploadText("/etc/apt/apt.conf.d/99-force-ipv4-transport", "Acquire::ForceIPv4 \"true\";");
node.SudoCommand("chmod 644 /etc/apt/apt.conf.d/99-force-ipv4-transport");
}
// Configure [apt] to retry.
node.UploadText("/etc/apt/apt.conf.d/99-retries", $"APT::Acquire::Retries \"{clusterDefinition.NodeOptions.PackageManagerRetries}\";");
node.SudoCommand("chmod 644 /etc/apt/apt.conf.d/99-retries");
//-----------------------------------------------------------------
// Other configuration.
ConfigureOpenSSH(node, TimeSpan.Zero);
node.UploadConfigFiles(clusterDefinition, kubeSetupInfo);
node.UploadResources(clusterDefinition, kubeSetupInfo);
if (clusterDefinition != null)
{
ConfigureEnvironmentVariables(node, clusterDefinition);
}
node.SudoCommand("safe-apt-get update");
node.InvokeIdempotentAction("setup/prep-node",
() =>
{
node.Status = "preparing";
node.SudoCommand("setup-prep.sh");
node.Reboot(wait: true);
});
// We need to upload the cluster configuration and initialize drives attached
// to the node. We're going to assume that these are not already initialized.
// $todo(jeff.lill):
//
// We may need an option that allows an operator to pre-build a hardware
// based drive array or something. I'm going to defer this to later and
// concentrate on commodity hardware and cloud deployments for now.
CommonSteps.ConfigureEnvironmentVariables(node, clusterDefinition);
node.Status = "setup: disk";
node.SudoCommand("setup-disk.sh");
// Clear any DHCP leases to be super sure that cloned node
// VMs will obtain fresh IP addresses.
node.Status = "clear: DHCP leases";
node.SudoCommand("rm -f /var/lib/dhcp/*");
// Indicate that the node has been fully prepared.
node.SudoCommand($"touch {KubeHostFolders.State}/setup/prepared");
// Shutdown the node if requested.
if (shutdown)
{
node.Status = "shutdown";
node.SudoCommand("shutdown 0", RunOptions.Defaults | RunOptions.Shutdown);
}
}
/// <summary>
/// Configures OpenVPN on a manager node.
/// </summary>
/// <param name="manager">The manager.</param>
private void ConfigManagerVpn(SshProxy <NodeDefinition> manager)
{
// Upload the setup and configuration files.
//
// NOTE:
//
// These steps are redundant and will be repeated during the
// common node configuration, but we need some of the scripts
// here, before that happens.
manager.CreateHiveHostFolders();
manager.UploadConfigFiles(hive.Definition);
manager.UploadResources(hive.Definition);
// Install OpenVPN.
manager.Status = "vpn install";
manager.SudoCommand("safe-apt-get update");
manager.SudoCommand("safe-apt-get install -yq openvpn");
// Configure OpenVPN.
var nodesSubnet = NetworkCidr.Parse(hive.Definition.Network.NodesSubnet);
var vpnSubnet = NetworkCidr.Parse(manager.Metadata.VpnPoolSubnet);
var duplicateCN = hive.Definition.Vpn.AllowSharedCredentials ? "duplicate-cn" : ";duplicate-cn";
var vpnServerAddress = NetHelper.UintToAddress(NetHelper.AddressToUint(vpnSubnet.Address) + 1);
var serverConf =
$@"#------------------------------------------------------------------------------
# OpenVPN config file customized for the [{manager.Name}] neonHIVE manager node.
# OpenVPN listening port.
port {NetworkPorts.OpenVPN}
# Enable TCP and/or UDP transports.
proto tcp
;proto udp
# Set packet tunneling mode.
dev tun
# SSL/TLS root certificate (ca), certificate
# (cert), and private key (key). Each client
# and the server must have their own cert and
# key file. The server and all clients will
# use the same ca file.
#
# See the [easy-rsa] directory for a series
# of scripts for generating RSA certificates
# and private keys. Remember to use
# a unique Common Name for the server
# and each of the client certificates.
#
# Any X509 key management system can be used.
# OpenVPN can also use a PKCS #12 formatted key file
# (see [pkcs12] directive in man page).
ca ca.crt
cert server.crt
key server.key # This file should be kept secret
# Diffie hellman parameters (2048-bit) generated via:
#
# openssl dhparam -out dhparam.pem 2048
#
dh dhparam.pem
# The currently recommended topology.
topology subnet
# Configure server mode and supply a VPN subnet
# for OpenVPN to draw client addresses from.
# The server will take {vpnServerAddress} for itself,
# the rest will be made available to clients.
# Each client will be able to reach the server
# on {vpnServerAddress}. Comment this line out if you are
# ethernet bridging. See the man page for more info.
server {vpnSubnet.Address} {vpnSubnet.Mask}
# Maintain a record of client virtual IP address
# associations in this file. If OpenVPN goes down or
# is restarted, reconnecting clients can be assigned
# the same virtual IP address from the pool that was
# previously assigned.
;ifconfig-pool-persist ipp.txt
# Push routes to the client to allow it
# to reach other private subnets behind
# the server. Remember that these
# private subnets will also need
# to know to route the OpenVPN client
# address pool ({vpnSubnet.Address})
# back to this specific OpenVPN server.
push ""route {nodesSubnet.Address} {nodesSubnet.Mask}""
# Uncomment this directive if multiple clients
# might connect with the same certificate/key
# files or common names. This is recommended
# only for testing purposes. For production use,
# each client should have its own certificate/key
# pair.
{duplicateCN}
# The keepalive directive causes ping-like
# messages to be sent back and forth over
# the link so that each side knows when
# the other side has gone down.
# Ping every 10 seconds, assume that remote
# peer is down if no ping received during
# a 120 second time period.
keepalive 10 120
# For extra security beyond that provided
# by SSL/TLS, create an [HMAC firewall]
# to help block DoS attacks and UDP port flooding.
#
# Generate with:
# openvpn --genkey --secret ta.key
#
# The server and each client must have
# a copy of this key.
# The second parameter should be '0'
# on the server and '1' on the clients.
tls-auth ta.key 0 # This file is secret
# Select a cryptographic cipher.
# This config item must be copied to
# the client config file as well.
cipher AES-256-CBC
# Enable compression on the VPN link.
# Don't enable this unless it is also
# enabled in the client config file.
#
# We're not enabling this due to the
# VORACLE security vulnerablity:
#
# https://community.openvpn.net/openvpn/wiki/VORACLE
#
# The maximum number of concurrently connected
# clients we want to allow.
max-clients {VpnOptions.ServerAddressCount - 2}
# This macro sets the TCP_NODELAY socket flag on
# the server as well as pushes it to connecting
# clients. The TCP_NODELAY flag disables the Nagle
# algorithm on TCP sockets causing packets to be
# transmitted immediately with low latency, rather
# than waiting a short period of time in order to
# aggregate several packets into a larger containing
# packet. In VPN applications over TCP, TCP_NODELAY
# is generally a good latency optimization.
tcp-nodelay
# It's a good idea to reduce the OpenVPN
# daemon's privileges after initialization.
#
# You can uncomment this out on
# non-Windows systems.
;user nobody
;group nobody
# The persist options will try to avoid
# accessing certain resources on restart
# that may no longer be accessible because
# of the privilege downgrade.
persist-key
persist-tun
# Output a short status file showing
# current connections, truncated
# and rewritten every minute.
status openvpn-status.log
# By default, log messages will go to the syslog (ork
# on Windows, if running as a service, they will go to
# the [\Program Files\OpenVPN\log] directory).
# Use log or log-append to override this default.
# [log] will truncate the log file on OpenVPN startup,
# while [log-append] will append to it. Use one
# or the other (but not both).
log /var/log/openvpn.log
;log-append openvpn.log
# Set the appropriate level of log
# file verbosity.
#
# 0 is silent, except for fatal errors
# 4 is reasonable for general usage
# 5 and 6 can help to debug connection problems
# 9 is extremely verbose
verb 4
# Silence repeating messages. At most 20
# sequential messages of the same message
# category will be output to the log.
;mute 20
";
manager.Status = "vpn config";
manager.SudoCommand("mkdir -p /etc/openvpn");
manager.UploadText("/etc/openvpn/server.conf", serverConf);
manager.UploadText("/etc/openvpn/ca.crt", vpnCaFiles.GetCert("ca"));
manager.UploadText("/etc/openvpn/server.crt", vpnCaFiles.GetCert("server"));
manager.UploadText("/etc/openvpn/server.key", vpnCaFiles.GetKey("server"));
manager.SudoCommand("chmod 600", "/etc/openvpn/server.key"); // This is a secret!
manager.UploadText("/etc/openvpn/ta.key", vpnCaFiles.GetTaKey());
manager.SudoCommand("chmod 600", "/etc/openvpn/ta.key"); // This is a secret too!
manager.UploadText("/etc/openvpn/dhparam.pem", vpnCaFiles.GetDHParam());
// Initialize the [root] user's credentials.
vpnCredentials =
new VpnCredentials()
{
CaCert = vpnCaFiles.GetCert("ca"),
UserCert = vpnCaFiles.GetCert(HiveConst.RootUser),
UserKey = vpnCaFiles.GetKey(HiveConst.RootUser),
TaKey = vpnCaFiles.GetTaKey(),
CaZipKey = VpnCaFiles.GenerateKey(),
CaZip = vpnCaFiles.ToZipBytes()
};
// Upload the initial (empty) Certificate Revocation List (CRL) file and then
// upload a OpenVPN systemd unit drop-in so that it will recognize revoked certificates.
manager.UploadText("/etc/openvpn/crl.pem", vpnCaFiles.GetFile("crl.pem"));
manager.SudoCommand("chmod 664", "/etc/openvpn/crl.pem"); // OpenVPN needs to be able to read this after having its privileges downgraded.
var openVpnUnit =
@"[Unit]
Description=OpenVPN connection to %i
PartOf=openvpn.service
ReloadPropagatedFrom=openvpn.service
Before=systemd-user-sessions.service
Documentation=man:openvpn(8)
Documentation=https://community.openvpn.net/openvpn/wiki/Openvpn23ManPage
Documentation=https://community.openvpn.net/openvpn/wiki/HOWTO
[Service]
PrivateTmp=true
KillMode=mixed
Type=forking
ExecStart=/usr/sbin/openvpn --daemon ovpn-%i --status /run/openvpn/%i.status 10 --cd /etc/openvpn --script-security 2 --config /etc/openvpn/%i.conf --writepid /run/openvpn/%i.pid --crl-verify /etc/openvpn/crl.pem
PIDFile=/run/openvpn/%i.pid
ExecReload=/bin/kill -HUP $MAINPID
WorkingDirectory=/etc/openvpn
ProtectSystem=yes
CapabilityBoundingSet=CAP_IPC_LOCK CAP_NET_ADMIN CAP_NET_BIND_SERVICE CAP_NET_RAW CAP_SETGID CAP_SETUID CAP_SYS_CHROOT CAP_DAC_READ_SEARCH CAP_AUDIT_WRITE
LimitNPROC=10
DeviceAllow=/dev/null rw
DeviceAllow=/dev/net/tun rw
[Install]
WantedBy=multi-user.target
";
manager.UploadText("/etc/systemd/system/[email protected]", openVpnUnit);
manager.SudoCommand("chmod 644 /etc/systemd/system/[email protected]");
// Do a daemon-reload so systemd will be aware of the new drop-in.
manager.SudoCommand("systemctl disable openvpn");
manager.SudoCommand("systemctl daemon-reload");
// Enable and restart OpenVPN.
manager.SudoCommand("systemctl enable openvpn");
manager.SudoCommand("systemctl restart openvpn");
//-----------------------------------------------------------------
// SPECIAL NOTE:
//
// I figured out that I need this lovely bit of code after banging my head on the desk for
// 12 freaking days. The problem was getting OpenVPN to work in Windows Azure (this will
// also probably impact other cloud environments).
//
// Azure implements VNETs as layer 3 overlays. This means that the host network interfaces
// are not actually on an ethernet segment and the VPN default gateway is actually handling
// all of the ARP packets, routing between the VNET subnets, load balancers, and the Internet.
// This is problematic for OpenVPN traffic because the VPN client IP address space is not
// part of the VNET which means the VNET gateway is not able to route packets from hive
// hosts back to the manager's OpenVPN client addresses by default.
//
// The solution is to configure the managers with secondary NIC cards in a different subnet
// and provision special Azure user-defined routes that direct VPN return packets to the
// correct manager.
//
// I figured this part out the second day. The problem was though that it simply didn't work.
// From an external VPN client, I would try to ping a worker node through OpenVPN running on
// a manager. I'd see the ping traffic:
//
// 1. manager/tun0: request
// 2. manager/eth1: request
// 3. worker/eth0: request
// 4. worker/eth0: reply
// 5. manager/eth0: reply
// 6: NOTHING! EXPECTED: manager/tun0: reply
//
// So the problem was that I could see the ICMP ping request hit the various interfaces
// on the manager and be received by the worker. I'd then see the worker send the reply,
// and be routed via the user-defined Azure rult back to the manager. The problem was
// that the packet was simply dropped there. It never made it back to tun0 so OpenVPN
// could forward it back to the client.
//
// After days and days of trying to learn about Linux routing, iptables and policy rules,
// I finally ran across this posting for the second time:
//
// https://unix.stackexchange.com/questions/21093/output-traffic-on-different-interfaces-based-on-destination-port
//
// This was the key. I ran across this a few days ago and didn't read it closely enough.
// It made more sense after learning more about this stuff.
//
// Linux has a built-in IP address spoofing filter enabled by default. This filter has the
// kernel discard any packets whose source address doesn't match the IP address/route implied
// by the remote interface that transmitted the packet. This is exactly what's happening
// when Azure forwards the VPN return packets via the user-defined route. I'd see return
// packets hit eth0 on the manager, be processed by the low-level RAW and MANGLE iptables
// and then they'd disappear.
//
// The solution is simply to disable the spoofing filter. I'm going to go ahead and do this
// for all interfaces which should be fine for hives hosted in cloud environments, because the
// VNET/Load Balancer/Security Groups will be used to lock things down. Local hives will
// need to be manually placed behind a suitable router/firewall as well.
//
// For robustness, I'm going to deploy this as a service daemon that polls the filter state
// for each interface every 5 seconds, and disables any enabled filters. This will ensure
// that the filters will always be disabled, even as interfaces are bought up and down.
var disableSpoofUnit =
$@"[Unit]
Description=Disable Network Anti-Spoofing Filters
Documentation=
After=
Requires=
Before=
[Service]
Type=simple
ExecStart={HiveHostFolders.Bin}/disable-spoof-filters.sh
[Install]
WantedBy=multi-user.target
";
var disableSpoofScript =
@"#!/bin/bash
#------------------------------------------------------------------------------
# This script is a deployed as a service to ensure that the Linux anti-spoofing
# filters are disabled for the network interfaces on manager nodes hosting
# OpenVPN. This is required to allow VPN return traffic from other nodes to
# routed back to tun0 and ultimately, connected VPN clients.
#
# Note that it appears that we need to disable the filter for all interfaces
# for this to actually work.
while :
do
flush=false
for f in /proc/sys/net/ipv4/conf/*/rp_filter
do
filter_enabled=$(cat $f)
if [ ""$filter_enabled"" == ""1"" ] ; then
echo 0 > $f
flush=true
fi
done
if [ ""$flush"" == ""true"" ] ; then
echo 1 > /proc/sys/net/ipv4/route/flush
fi
sleep 5
done";
manager.UploadText("/lib/systemd/system/disable-spoof-filters.service", disableSpoofUnit);
manager.SudoCommand("chmod 644 /lib/systemd/system/disable-spoof-filters.service");
manager.UploadText($"{HiveHostFolders.Bin}/disable-spoof-filters.sh", disableSpoofScript);
manager.SudoCommand($"chmod 770 {HiveHostFolders.Bin}/disable-spoof-filters.sh");
manager.SudoCommand("systemctl enable disable-spoof-filters");
manager.SudoCommand("systemctl restart disable-spoof-filters");
}