/// <summary>
/// Configures the global environment variables that describe the configuration
/// of the server within the cluster.
/// </summary>
/// <param name="node">The server to be updated.</param>
/// <param name="clusterDefinition">The cluster definition.</param>
public static void ConfigureEnvironmentVariables(SshProxy <NodeDefinition> node, ClusterDefinition clusterDefinition)
{
node.Status = "environment variables";
// We're going to append the new variables to the existing Linux [/etc/environment] file.
var sb = new StringBuilder();
// Append all of the existing environment variables except for those
// whose names start with "NEON_" to make the operation idempotent.
//
// Note that we're going to special case PATH to add any Neon
// related directories.
using (var currentEnvironmentStream = new MemoryStream())
{
node.Download("/etc/environment", currentEnvironmentStream);
currentEnvironmentStream.Position = 0;
using (var reader = new StreamReader(currentEnvironmentStream))
{
foreach (var line in reader.Lines())
{
if (line.StartsWith("PATH="))
{
if (!line.Contains(KubeHostFolders.Bin))
{
sb.AppendLine(line + $":/snap/bin:{KubeHostFolders.Bin}");
}
else
{
sb.AppendLine(line);
}
}
else if (!line.StartsWith("NEON_"))
{
sb.AppendLine(line);
}
}
}
}
// Add the global cluster related environment variables.
sb.AppendLine($"NEON_CLUSTER_PROVISIONER={clusterDefinition.Provisioner}");
sb.AppendLine($"NEON_CLUSTER={clusterDefinition.Name}");
sb.AppendLine($"NEON_DATACENTER={clusterDefinition.Datacenter.ToLowerInvariant()}");
sb.AppendLine($"NEON_ENVIRONMENT={clusterDefinition.Environment.ToString().ToLowerInvariant()}");
var sbPackageProxies = new StringBuilder();
foreach (var proxyEndpoint in clusterDefinition.PackageProxy.Split(new char[] { ' ' }, StringSplitOptions.RemoveEmptyEntries))
{
sbPackageProxies.AppendWithSeparator(proxyEndpoint);
}
sb.AppendLine($"NEON_PACKAGE_PROXY={sbPackageProxies}");
if (clusterDefinition.Hosting != null)
{
sb.AppendLine($"NEON_HOSTING={clusterDefinition.Hosting.Environment.ToMemberString().ToLowerInvariant()}");
}
sb.AppendLine($"NEON_NODE_NAME={node.Name}");
if (node.Metadata != null)
{
sb.AppendLine($"NEON_NODE_ROLE={node.Metadata.Role}");
sb.AppendLine($"NEON_NODE_IP={node.Metadata.PrivateAddress}");
sb.AppendLine($"NEON_NODE_HDD={node.Metadata.Labels.StorageHDD.ToString().ToLowerInvariant()}");
}
sb.AppendLine($"NEON_ARCHIVE_FOLDER={KubeHostFolders.Archive(KubeConst.SysAdminUser)}");
sb.AppendLine($"NEON_BIN_FOLDER={KubeHostFolders.Bin}");
sb.AppendLine($"NEON_CONFIG_FOLDER={KubeHostFolders.Config}");
sb.AppendLine($"NEON_EXEC_FOLDER={KubeHostFolders.Exec(KubeConst.SysAdminUser)}");
sb.AppendLine($"NEON_SETUP_FOLDER={KubeHostFolders.Setup}");
sb.AppendLine($"NEON_STATE_FOLDER={KubeHostFolders.State}");
sb.AppendLine($"NEON_TMPFS_FOLDER={KubeHostFolders.Tmpfs}");
// Kubernetes related variables for masters.
if (node.Metadata.IsMaster)
{
sb.AppendLine($"KUBECONFIG=/etc/kubernetes/admin.conf");
}
// Upload the new environment to the server.
node.UploadText("/etc/environment", sb, tabStop: 4);
}
/// <summary>
/// Sets cluster definition related variables for a <see cref="PreprocessReader"/>.
/// </summary>
/// <param name="preprocessReader">The reader.</param>
/// <param name="clusterDefinition">The cluster definition.</param>
/// <param name="kubeSetupInfo">The Kubernetes setup details.</param>
/// <param name="nodeDefinition">The target node definition.</param>
private static void SetClusterVariables(PreprocessReader preprocessReader, ClusterDefinition clusterDefinition, KubeSetupInfo kubeSetupInfo, NodeDefinition nodeDefinition)
{
Covenant.Requires <ArgumentNullException>(preprocessReader != null, nameof(preprocessReader));
Covenant.Requires <ArgumentNullException>(clusterDefinition != null, nameof(clusterDefinition));
Covenant.Requires <ArgumentNullException>(kubeSetupInfo != null, nameof(kubeSetupInfo));
// Generate the master node variables in sorted order. The variable
// names will be formatted as:
//
// NEON_MASTER_#
//
// where [#] is the zero-based index of the node. This is compatible
// with the [getmaster] function included the script.
//
// Each variable defines an associative array with [name] and [address]
// properties.
//
// Then generate the NEON_MASTER_NAMES and NEON_MASTER_ADDRESSES arrays.
//
// NOTE: We need to use Linux-style line endings.
var sbMasters = new StringBuilder();
var sbMasterNamesArray = new StringBuilder();
var sbMasterAddressesArray = new StringBuilder();
var sbPeerMasterAddressesArray = new StringBuilder();
var sbMasterNodesSummary = new StringBuilder();
var index = 0;
var masterNameWidth = 0;
sbMasterNamesArray.Append("(");
sbMasterAddressesArray.Append("(");
sbPeerMasterAddressesArray.Append("(");
foreach (var master in clusterDefinition.SortedMasters)
{
sbMasters.Append($"declare -x -A NEON_MASTER_{index}\n");
sbMasters.Append($"NEON_MASTER_{index}=( [\"name\"]=\"{master.Name}\" [\"address\"]=\"{master.PrivateAddress}\" )\n");
sbMasters.Append("\n");
index++;
sbMasterNamesArray.Append($" \"{master.Name}\"");
sbMasterAddressesArray.Append($" \"{master.PrivateAddress}\"");
if (master != nodeDefinition)
{
sbPeerMasterAddressesArray.Append($" \"{master.PrivateAddress}\"");
}
masterNameWidth = Math.Max(master.Name.Length, masterNameWidth);
}
sbMasterNamesArray.Append(" )");
sbMasterAddressesArray.Append(" )");
sbPeerMasterAddressesArray.Append(" )");
foreach (var master in clusterDefinition.SortedMasters)
{
var nameField = master.Name;
if (nameField.Length < masterNameWidth)
{
nameField += new string(' ', masterNameWidth - nameField.Length);
}
// The blanks below are just enough so that the "=" sign lines up
// with the summary output from [cluster.conf.sh].
if (sbMasterNodesSummary.Length == 0)
{
sbMasterNodesSummary.Append($" echo \"NEON_MASTER_NODES = {nameField}: {master.PrivateAddress}\" 1>&2\n");
}
else
{
sbMasterNodesSummary.Append($" echo \" {nameField}: {master.PrivateAddress}\" 1>&2\n");
}
}
foreach (var master in clusterDefinition.SortedMasters)
{
sbMasters.Append($"declare -x -A NEON_MASTER_{index}\n");
sbMasters.Append($"NEON_MASTER_{index}=( [\"name\"]=\"{master.Name}\" [\"address\"]=\"{master.PrivateAddress}\" )\n");
index++;
}
sbMasters.Append("\n");
sbMasters.Append($"declare -x NEON_MASTER_NAMES={sbMasterNamesArray}\n");
sbMasters.Append($"declare -x NEON_MASTER_ADDRESSES={sbMasterAddressesArray}\n");
sbMasters.Append("\n");
// Generate the master and worker NTP time sources.
var masterTimeSources = string.Empty;
var workerTimeSources = string.Empty;
if (clusterDefinition.TimeSources != null)
{
foreach (var source in clusterDefinition.TimeSources)
{
if (string.IsNullOrWhiteSpace(source))
{
continue;
}
if (masterTimeSources.Length > 0)
{
masterTimeSources += " ";
}
masterTimeSources += $"\"{source}\"";
}
}
foreach (var master in clusterDefinition.SortedMasters)
{
if (workerTimeSources.Length > 0)
{
workerTimeSources += " ";
}
workerTimeSources += $"\"{master.PrivateAddress}\"";
}
if (string.IsNullOrWhiteSpace(masterTimeSources))
{
// Default to a reasonable public time source.
masterTimeSources = "\"pool.ntp.org\"";
}
// Set the variables.
preprocessReader.Set("load-cluster-conf", KubeHostFolders.Config + "/cluster.conf.sh --echo-summary");
preprocessReader.Set("load-cluster-conf-quiet", KubeHostFolders.Config + "/cluster.conf.sh");
SetBashVariable(preprocessReader, "cluster.provisioner", clusterDefinition.Provisioner);
SetBashVariable(preprocessReader, "node.driveprefix", clusterDefinition.DrivePrefix);
SetBashVariable(preprocessReader, "neon.folders.archive", KubeHostFolders.Archive(KubeConst.SysAdminUser));
SetBashVariable(preprocessReader, "neon.folders.bin", KubeHostFolders.Bin);
SetBashVariable(preprocessReader, "neon.folders.exec", KubeHostFolders.Exec(KubeConst.SysAdminUser));
SetBashVariable(preprocessReader, "neon.folders.config", KubeHostFolders.Config);
SetBashVariable(preprocessReader, "neon.folders.setup", KubeHostFolders.Setup);
SetBashVariable(preprocessReader, "neon.folders.state", KubeHostFolders.State);
SetBashVariable(preprocessReader, "neon.folders.tmpfs", KubeHostFolders.Tmpfs);
SetBashVariable(preprocessReader, "neon.folders.tools", KubeHostFolders.Bin);
SetBashVariable(preprocessReader, "nodes.master.count", clusterDefinition.Masters.Count());
preprocessReader.Set("nodes.masters", sbMasters);
preprocessReader.Set("nodes.masters.summary", sbMasterNodesSummary);
SetBashVariable(preprocessReader, "ntp.master.sources", masterTimeSources);
NewMethod(preprocessReader, workerTimeSources);
SetBashVariable(preprocessReader, "docker.packageuri", kubeSetupInfo.DockerPackageUbuntuUri);
SetBashVariable(preprocessReader, "neon.kube.kubeadm.package_version", kubeSetupInfo.KubeAdmPackageUbuntuVersion);
SetBashVariable(preprocessReader, "neon.kube.kubectl.package_version", kubeSetupInfo.KubeCtlPackageUbuntuVersion);
SetBashVariable(preprocessReader, "neon.kube.kubelet.package_version", kubeSetupInfo.KubeletPackageUbuntuVersion);
//-----------------------------------------------------------------
// Configure the variables for the [setup-disk.sh] script.
switch (clusterDefinition.Hosting.Environment)
{
case HostingEnvironments.Aws:
throw new NotImplementedException("$todo(jefflill)");
case HostingEnvironments.Azure:
// The primary Azure data drive is [/dev/sdb] so any mounted drive will be [/dev/sdc].
if (nodeDefinition.Azure.HardDriveCount == 0)
{
SetBashVariable(preprocessReader, "data.disk", "PRIMARY");
}
else
{
SetBashVariable(preprocessReader, "data.disk", "/dev/sdc");
}
break;
case HostingEnvironments.Google:
throw new NotImplementedException("$todo(jefflill)");
case HostingEnvironments.HyperV:
case HostingEnvironments.HyperVLocal:
case HostingEnvironments.Machine:
case HostingEnvironments.Unknown:
case HostingEnvironments.XenServer:
// VMs for all of these environments simply host their data on the
// primary OS disk only for now, the idea being that this disk
// can be sized up as necessary. There are valid scenarios where
// folks would like the data on a different drive (e.g. for better
// performance). I'm putting support for that on the backlog.
SetBashVariable(preprocessReader, "data.disk", "PRIMARY");
break;
default:
throw new NotImplementedException($"The [{clusterDefinition.Hosting.Environment}] hosting environment is not implemented.");
}
}