/// <summary>
/// Initializes a near virgin server with the basic capabilities required
/// for a cluster host node.
/// </summary>
/// <param name="node">The target cluster node.</param>
/// <param name="clusterDefinition">The cluster definition.</param>
/// <param name="kubeSetupInfo">Kubernetes setup details.</param>
/// <param name="shutdown">Optionally shuts down the node.</param>
public static void PrepareNode(SshProxy <NodeDefinition> node, ClusterDefinition clusterDefinition, KubeSetupInfo kubeSetupInfo, bool shutdown = false)
{
Covenant.Requires <ArgumentNullException>(node != null);
Covenant.Requires <ArgumentNullException>(clusterDefinition != null);
Covenant.Requires <ArgumentNullException>(kubeSetupInfo != null);
if (node.FileExists($"{KubeHostFolders.State}/setup/prepared"))
{
return; // Already prepared
}
//-----------------------------------------------------------------
// Ensure that the cluster host folders exist.
node.CreateHostFolders();
//-----------------------------------------------------------------
// Package manager configuration.
if (!clusterDefinition.NodeOptions.AllowPackageManagerIPv6)
{
// Restrict the [apt] package manager to using IPv4 to communicate
// with the package mirrors, since IPv6 often doesn't work.
node.UploadText("/etc/apt/apt.conf.d/99-force-ipv4-transport", "Acquire::ForceIPv4 \"true\";");
node.SudoCommand("chmod 644 /etc/apt/apt.conf.d/99-force-ipv4-transport");
}
// Configure [apt] to retry.
node.UploadText("/etc/apt/apt.conf.d/99-retries", $"APT::Acquire::Retries \"{clusterDefinition.NodeOptions.PackageManagerRetries}\";");
node.SudoCommand("chmod 644 /etc/apt/apt.conf.d/99-retries");
//-----------------------------------------------------------------
// Other configuration.
ConfigureOpenSSH(node, TimeSpan.Zero);
node.UploadConfigFiles(clusterDefinition, kubeSetupInfo);
node.UploadResources(clusterDefinition, kubeSetupInfo);
if (clusterDefinition != null)
{
ConfigureEnvironmentVariables(node, clusterDefinition);
}
node.SudoCommand("safe-apt-get update");
node.InvokeIdempotentAction("setup/prep-node",
() =>
{
node.Status = "preparing";
node.SudoCommand("setup-prep.sh");
node.Reboot(wait: true);
});
// We need to upload the cluster configuration and initialize drives attached
// to the node. We're going to assume that these are not already initialized.
// $todo(jeff.lill):
//
// We may need an option that allows an operator to pre-build a hardware
// based drive array or something. I'm going to defer this to later and
// concentrate on commodity hardware and cloud deployments for now.
CommonSteps.ConfigureEnvironmentVariables(node, clusterDefinition);
node.Status = "setup: disk";
node.SudoCommand("setup-disk.sh");
// Clear any DHCP leases to be super sure that cloned node
// VMs will obtain fresh IP addresses.
node.Status = "clear: DHCP leases";
node.SudoCommand("rm -f /var/lib/dhcp/*");
// Indicate that the node has been fully prepared.
node.SudoCommand($"touch {KubeHostFolders.State}/setup/prepared");
// Shutdown the node if requested.
if (shutdown)
{
node.Status = "shutdown";
node.SudoCommand("shutdown 0", RunOptions.Defaults | RunOptions.Shutdown);
}
}
/// <summary>
/// Uploads the setup and other scripts and tools for the target operating system to the server.
/// </summary>
/// <typeparam name="TMetadata">The server's metadata type.</typeparam>
/// <param name="server">The remote server.</param>
/// <param name="clusterDefinition">The cluster definition.</param>
/// <param name="kubeSetupInfo">The Kubernetes setup details.</param>
public static void UploadResources <TMetadata>(this SshProxy <TMetadata> server, ClusterDefinition clusterDefinition, KubeSetupInfo kubeSetupInfo)
where TMetadata : class
{
Covenant.Requires <ArgumentNullException>(server != null, nameof(server));
Covenant.Requires <ArgumentNullException>(clusterDefinition != null, nameof(clusterDefinition));
Covenant.Requires <ArgumentNullException>(kubeSetupInfo != null, nameof(kubeSetupInfo));
//-----------------------------------------------------------------
// Upload resource files to the setup folder.
server.Status = $"clear: {KubeHostFolders.Setup}";
server.SudoCommand($"rm -rf {KubeHostFolders.Setup}/*.*");
// Upload the setup files.
server.Status = "upload: setup scripts";
foreach (var file in Program.LinuxFolder.GetFolder("setup").Files())
{
server.UploadFile(clusterDefinition, kubeSetupInfo, file, $"{KubeHostFolders.Setup}/{file.Name}");
}
// Make the setup scripts executable.
server.SudoCommand($"chmod 744 {KubeHostFolders.Setup}/*");
//-----------------------------------------------------------------
// Upload files to the bin folder.
server.Status = $"clear: {KubeHostFolders.Bin}";
server.SudoCommand($"rm -rf {KubeHostFolders.Bin}/*.*");
// Upload the tool files. Note that we're going to strip out the [.sh]
// file type to make these easier to run.
server.Status = "upload: binary files";
foreach (var file in Program.LinuxFolder.GetFolder("binary").Files())
{
server.UploadFile(clusterDefinition, kubeSetupInfo, file, $"{KubeHostFolders.Bin}/{file.Name.Replace(".sh", string.Empty)}");
}
// Make the scripts executable.
server.SudoCommand($"chmod 744 {KubeHostFolders.Bin}/*");
}
/// <summary>
/// Uploads the configuration files for the target operating system to the server.
/// </summary>
/// <typeparam name="Metadata">The node metadata type.</typeparam>
/// <param name="node">The remote node.</param>
/// <param name="clusterDefinition">The cluster definition.</param>
/// <param name="kubeSetupInfo">The Kubernetes setup details.</param>
public static void UploadConfigFiles <Metadata>(this SshProxy <Metadata> node, ClusterDefinition clusterDefinition, KubeSetupInfo kubeSetupInfo)
where Metadata : class
{
Covenant.Requires <ArgumentNullException>(node != null, nameof(node));
Covenant.Requires <ArgumentNullException>(clusterDefinition != null, nameof(clusterDefinition));
Covenant.Requires <ArgumentNullException>(kubeSetupInfo != null, nameof(kubeSetupInfo));
// Clear the contents of the configuration folder.
node.Status = $"clear: {KubeHostFolders.Config}";
node.SudoCommand($"rm -rf {KubeHostFolders.Config}/*.*");
// Upload the files.
node.Status = "upload: config files";
foreach (var file in Program.LinuxFolder.GetFolder("conf").Files())
{
node.UploadFile(clusterDefinition, kubeSetupInfo, file, $"{KubeHostFolders.Config}/{file.Name}");
}
// Secure the files and make the scripts executable.
node.SudoCommand($"chmod 644 {KubeHostFolders.Config}/*.*");
node.SudoCommand($"chmod 744 {KubeHostFolders.Config}/*.sh");
node.Status = "copied";
}
/// <summary>
/// Uploads a resource file to the remote server after performing any necessary preprocessing.
/// </summary>
/// <typeparam name="TMetadata">The node metadata type.</typeparam>
/// <param name="node">The remote node.</param>
/// <param name="clusterDefinition">The cluster definition or <c>null</c>.</param>
/// <param name="kubeSetupInfo">The Kubernetes setup details.</param>
/// <param name="file">The resource file.</param>
/// <param name="targetPath">The target path on the remote server.</param>
private static void UploadFile <TMetadata>(this SshProxy <TMetadata> node, ClusterDefinition clusterDefinition, KubeSetupInfo kubeSetupInfo, ResourceFiles.File file, string targetPath)
where TMetadata : class
{
using (var input = file.ToStream())
{
if (file.HasVariables)
{
// We need to expand any variables. Note that if we don't have a
// cluster definition or for undefined variables, we're going to
// have the variables expand to the empty string.
using (var msExpanded = new MemoryStream())
{
using (var writer = new StreamWriter(msExpanded))
{
var preprocessReader =
new PreprocessReader(new StreamReader(input))
{
DefaultVariable = string.Empty,
ExpandVariables = true,
ProcessStatements = false,
StripComments = false
};
if (clusterDefinition != null)
{
SetClusterVariables(preprocessReader, clusterDefinition, kubeSetupInfo, node.Metadata as NodeDefinition);
}
foreach (var line in preprocessReader.Lines())
{
writer.WriteLine(line);
}
writer.Flush();
msExpanded.Position = 0;
node.UploadText(targetPath, msExpanded, tabStop: 4, outputEncoding: Encoding.UTF8);
}
}
}
else
{
node.UploadText(targetPath, input, tabStop: 4, outputEncoding: Encoding.UTF8);
}
}
}
/// <summary>
/// Sets cluster definition related variables for a <see cref="PreprocessReader"/>.
/// </summary>
/// <param name="preprocessReader">The reader.</param>
/// <param name="clusterDefinition">The cluster definition.</param>
/// <param name="kubeSetupInfo">The Kubernetes setup details.</param>
/// <param name="nodeDefinition">The target node definition.</param>
private static void SetClusterVariables(PreprocessReader preprocessReader, ClusterDefinition clusterDefinition, KubeSetupInfo kubeSetupInfo, NodeDefinition nodeDefinition)
{
Covenant.Requires <ArgumentNullException>(preprocessReader != null, nameof(preprocessReader));
Covenant.Requires <ArgumentNullException>(clusterDefinition != null, nameof(clusterDefinition));
Covenant.Requires <ArgumentNullException>(kubeSetupInfo != null, nameof(kubeSetupInfo));
// Generate the master node variables in sorted order. The variable
// names will be formatted as:
//
// NEON_MASTER_#
//
// where [#] is the zero-based index of the node. This is compatible
// with the [getmaster] function included the script.
//
// Each variable defines an associative array with [name] and [address]
// properties.
//
// Then generate the NEON_MASTER_NAMES and NEON_MASTER_ADDRESSES arrays.
//
// NOTE: We need to use Linux-style line endings.
var sbMasters = new StringBuilder();
var sbMasterNamesArray = new StringBuilder();
var sbMasterAddressesArray = new StringBuilder();
var sbPeerMasterAddressesArray = new StringBuilder();
var sbMasterNodesSummary = new StringBuilder();
var index = 0;
var masterNameWidth = 0;
sbMasterNamesArray.Append("(");
sbMasterAddressesArray.Append("(");
sbPeerMasterAddressesArray.Append("(");
foreach (var master in clusterDefinition.SortedMasters)
{
sbMasters.Append($"declare -x -A NEON_MASTER_{index}\n");
sbMasters.Append($"NEON_MASTER_{index}=( [\"name\"]=\"{master.Name}\" [\"address\"]=\"{master.PrivateAddress}\" )\n");
sbMasters.Append("\n");
index++;
sbMasterNamesArray.Append($" \"{master.Name}\"");
sbMasterAddressesArray.Append($" \"{master.PrivateAddress}\"");
if (master != nodeDefinition)
{
sbPeerMasterAddressesArray.Append($" \"{master.PrivateAddress}\"");
}
masterNameWidth = Math.Max(master.Name.Length, masterNameWidth);
}
sbMasterNamesArray.Append(" )");
sbMasterAddressesArray.Append(" )");
sbPeerMasterAddressesArray.Append(" )");
foreach (var master in clusterDefinition.SortedMasters)
{
var nameField = master.Name;
if (nameField.Length < masterNameWidth)
{
nameField += new string(' ', masterNameWidth - nameField.Length);
}
// The blanks below are just enough so that the "=" sign lines up
// with the summary output from [cluster.conf.sh].
if (sbMasterNodesSummary.Length == 0)
{
sbMasterNodesSummary.Append($" echo \"NEON_MASTER_NODES = {nameField}: {master.PrivateAddress}\" 1>&2\n");
}
else
{
sbMasterNodesSummary.Append($" echo \" {nameField}: {master.PrivateAddress}\" 1>&2\n");
}
}
foreach (var master in clusterDefinition.SortedMasters)
{
sbMasters.Append($"declare -x -A NEON_MASTER_{index}\n");
sbMasters.Append($"NEON_MASTER_{index}=( [\"name\"]=\"{master.Name}\" [\"address\"]=\"{master.PrivateAddress}\" )\n");
index++;
}
sbMasters.Append("\n");
sbMasters.Append($"declare -x NEON_MASTER_NAMES={sbMasterNamesArray}\n");
sbMasters.Append($"declare -x NEON_MASTER_ADDRESSES={sbMasterAddressesArray}\n");
sbMasters.Append("\n");
// Generate the master and worker NTP time sources.
var masterTimeSources = string.Empty;
var workerTimeSources = string.Empty;
if (clusterDefinition.TimeSources != null)
{
foreach (var source in clusterDefinition.TimeSources)
{
if (string.IsNullOrWhiteSpace(source))
{
continue;
}
if (masterTimeSources.Length > 0)
{
masterTimeSources += " ";
}
masterTimeSources += $"\"{source}\"";
}
}
foreach (var master in clusterDefinition.SortedMasters)
{
if (workerTimeSources.Length > 0)
{
workerTimeSources += " ";
}
workerTimeSources += $"\"{master.PrivateAddress}\"";
}
if (string.IsNullOrWhiteSpace(masterTimeSources))
{
// Default to a reasonable public time source.
masterTimeSources = "\"pool.ntp.org\"";
}
// Set the variables.
preprocessReader.Set("load-cluster-conf", KubeHostFolders.Config + "/cluster.conf.sh --echo-summary");
preprocessReader.Set("load-cluster-conf-quiet", KubeHostFolders.Config + "/cluster.conf.sh");
SetBashVariable(preprocessReader, "cluster.provisioner", clusterDefinition.Provisioner);
SetBashVariable(preprocessReader, "node.driveprefix", clusterDefinition.DrivePrefix);
SetBashVariable(preprocessReader, "neon.folders.archive", KubeHostFolders.Archive(KubeConst.SysAdminUser));
SetBashVariable(preprocessReader, "neon.folders.bin", KubeHostFolders.Bin);
SetBashVariable(preprocessReader, "neon.folders.exec", KubeHostFolders.Exec(KubeConst.SysAdminUser));
SetBashVariable(preprocessReader, "neon.folders.config", KubeHostFolders.Config);
SetBashVariable(preprocessReader, "neon.folders.setup", KubeHostFolders.Setup);
SetBashVariable(preprocessReader, "neon.folders.state", KubeHostFolders.State);
SetBashVariable(preprocessReader, "neon.folders.tmpfs", KubeHostFolders.Tmpfs);
SetBashVariable(preprocessReader, "neon.folders.tools", KubeHostFolders.Bin);
SetBashVariable(preprocessReader, "nodes.master.count", clusterDefinition.Masters.Count());
preprocessReader.Set("nodes.masters", sbMasters);
preprocessReader.Set("nodes.masters.summary", sbMasterNodesSummary);
SetBashVariable(preprocessReader, "ntp.master.sources", masterTimeSources);
NewMethod(preprocessReader, workerTimeSources);
SetBashVariable(preprocessReader, "docker.packageuri", kubeSetupInfo.DockerPackageUbuntuUri);
SetBashVariable(preprocessReader, "neon.kube.kubeadm.package_version", kubeSetupInfo.KubeAdmPackageUbuntuVersion);
SetBashVariable(preprocessReader, "neon.kube.kubectl.package_version", kubeSetupInfo.KubeCtlPackageUbuntuVersion);
SetBashVariable(preprocessReader, "neon.kube.kubelet.package_version", kubeSetupInfo.KubeletPackageUbuntuVersion);
//-----------------------------------------------------------------
// Configure the variables for the [setup-disk.sh] script.
switch (clusterDefinition.Hosting.Environment)
{
case HostingEnvironments.Aws:
throw new NotImplementedException("$todo(jefflill)");
case HostingEnvironments.Azure:
// The primary Azure data drive is [/dev/sdb] so any mounted drive will be [/dev/sdc].
if (nodeDefinition.Azure.HardDriveCount == 0)
{
SetBashVariable(preprocessReader, "data.disk", "PRIMARY");
}
else
{
SetBashVariable(preprocessReader, "data.disk", "/dev/sdc");
}
break;
case HostingEnvironments.Google:
throw new NotImplementedException("$todo(jefflill)");
case HostingEnvironments.HyperV:
case HostingEnvironments.HyperVLocal:
case HostingEnvironments.Machine:
case HostingEnvironments.Unknown:
case HostingEnvironments.XenServer:
// VMs for all of these environments simply host their data on the
// primary OS disk only for now, the idea being that this disk
// can be sized up as necessary. There are valid scenarios where
// folks would like the data on a different drive (e.g. for better
// performance). I'm putting support for that on the backlog.
SetBashVariable(preprocessReader, "data.disk", "PRIMARY");
break;
default:
throw new NotImplementedException($"The [{clusterDefinition.Hosting.Environment}] hosting environment is not implemented.");
}
}
/// <inheritdoc/>
public override void Run(CommandLine commandLine)
{
if (commandLine.HasHelpOption)
{
Help();
Program.Exit(0);
}
// Special-case handling of the [--remove-templates] option.
if (commandLine.HasOption("--remove-templates"))
{
Console.WriteLine("Removing cached virtual machine templates.");
foreach (var fileName in Directory.GetFiles(KubeHelper.VmTemplatesFolder, "*.*", SearchOption.TopDirectoryOnly))
{
File.Delete(fileName);
}
Program.Exit(0);
}
// Implement the command.
if (KubeHelper.CurrentContext != null)
{
Console.Error.WriteLine("*** ERROR: You are logged into a cluster. You need to logout before preparing another.");
Program.Exit(1);
}
if (commandLine.Arguments.Length == 0)
{
Console.Error.WriteLine($"*** ERROR: CLUSTER-DEF expected.");
Program.Exit(1);
}
clusterDefPath = commandLine.Arguments[0];
force = commandLine.GetFlag("--force");
ClusterDefinition.ValidateFile(clusterDefPath, strict: true);
var clusterDefinition = ClusterDefinition.FromFile(clusterDefPath, strict: true);
clusterDefinition.Provisioner = $"neon-cli:{Program.Version}"; // Identify this tool/version as the cluster provisioner
// NOTE:
//
// Azure has a more restrictive password policy and our default
// machine password does not meet the requirements:
//
// The supplied password must be between 6-72 characters long and must
// satisfy at least 3 of password complexity requirements from the following:
//
// 1. Contains an uppercase character
// 2. Contains a lowercase character
// 3. Contains a numeric digit
// 4. Contains a special character
// 5. Control characters are not allowed
//
// It's also probably not a great idea to use a static password when
// provisioning VMs in public clouds because it might be possible for
// somebody to use this fact the SSH into nodes while the cluster is
// being setup and before we set the secure password at the end.
//
// This is less problematic for non-cloud environments because it's
// likely that the hosts won't initially be able to receive inbound
// Internet traffic and besides, we need to have a known password
// embedded into the VM templates.
//
// We're going to handle this for cloud environments by looking
// at [Program.MachinePassword]. If this is set to the default
// machine password then we're going to replace it with a randomlly
// generated password with a few extra characters to ensure that
// it meets the target cloud's password requirements. We'll use
// a non-default password if the operator specified one.
if (clusterDefinition.Hosting.IsCloudProvider && Program.MachinePassword == KubeConst.DefaulVmTemplatePassword)
{
Program.MachinePassword = NeonHelper.GetCryptoRandomPassword(20);
// Append a string that guarantees that the generated password meets
// cloud minimum requirements.
Program.MachinePassword += ".Aa0";
}
// NOTE: Cluster prepare starts new log files.
cluster = new ClusterProxy(clusterDefinition, Program.CreateNodeProxy <NodeDefinition>, appendToLog: false, defaultRunOptions: RunOptions.LogOutput | RunOptions.FaultOnError);
if (KubeHelper.Config.GetContext(cluster.Definition.Name) != null)
{
Console.Error.WriteLine($"*** ERROR: A context named [{cluster.Definition.Name}] already exists.");
Program.Exit(1);
}
// Configure global options.
if (commandLine.HasOption("--unredacted"))
{
cluster.SecureRunOptions = RunOptions.None;
}
var failed = false;
try
{
KubeHelper.Desktop.StartOperationAsync($"Preparing [{cluster.Name}]").Wait();
//-----------------------------------------------------------------
// Try to ensure that no servers are already deployed on the IP addresses defined
// for cluster nodes because provisoning over an existing cluster will likely
// corrupt the existing cluster and also probably prevent the new cluster from
// provisioning correctly.
//
// Note that we're not going to perform this check for the [Machine] hosting
// environment because we're expecting the bare machines to be already running
// with the assigned addresses and we're also not going to do this for cloud
// environments because we're assuming that the cluster will run in its own
// private network so there'll ne no possibility of conflicts.
if (cluster.Definition.Hosting.Environment != HostingEnvironments.Machine &&
!cluster.Definition.Hosting.IsCloudProvider)
{
Console.WriteLine();
Console.WriteLine(" Scanning for IP address conflicts...");
Console.WriteLine();
var pingOptions = new PingOptions(ttl: 32, dontFragment: true);
var pingTimeout = TimeSpan.FromSeconds(2);
var pingConflicts = new List <NodeDefinition>();
var pingAttempts = 2;
// I'm going to use up to 20 threads at a time here for simplicity
// rather then doing this as async operations.
var parallelOptions = new ParallelOptions()
{
MaxDegreeOfParallelism = 20
};
Parallel.ForEach(cluster.Definition.NodeDefinitions.Values, parallelOptions,
node =>
{
using (var pinger = new Pinger())
{
// We're going to try pinging up to [pingAttempts] times for each node
// just in case the network it sketchy and we're losing reply packets.
for (int i = 0; i < pingAttempts; i++)
{
var reply = pinger.SendPingAsync(node.PrivateAddress, (int)pingTimeout.TotalMilliseconds).Result;
if (reply.Status == IPStatus.Success)
{
lock (pingConflicts)
{
pingConflicts.Add(node);
}
break;
}
}
}
});
if (pingConflicts.Count > 0)
{
Console.Error.WriteLine($"*** ERROR: Cannot provision the cluster because [{pingConflicts.Count}] other");
Console.Error.WriteLine($"*** machines conflict with the following cluster nodes:");
Console.Error.WriteLine();
foreach (var node in pingConflicts.OrderBy(n => NetHelper.AddressToUint(IPAddress.Parse(n.PrivateAddress))))
{
Console.Error.WriteLine($"{node.PrivateAddress, 16}: {node.Name}");
}
Program.Exit(1);
}
}
//-----------------------------------------------------------------
// Perform basic environment provisioning. This creates basic cluster components
// such as virtual machines, networks, load balancers, public IP addresses, security
// groups,... as required for the environment.
hostingManager = new HostingManagerFactory(() => HostingLoader.Initialize()).GetMaster(cluster, Program.LogPath);
if (hostingManager == null)
{
Console.Error.WriteLine($"*** ERROR: No hosting manager for the [{cluster.Definition.Hosting.Environment}] hosting environment could be located.");
Program.Exit(1);
}
hostingManager.HostUsername = Program.MachineUsername;
hostingManager.HostPassword = Program.MachinePassword;
hostingManager.ShowStatus = !Program.Quiet;
hostingManager.MaxParallel = Program.MaxParallel;
hostingManager.WaitSeconds = Program.WaitSeconds;
if (hostingManager.RequiresAdminPrivileges)
{
Program.VerifyAdminPrivileges($"Provisioning to [{cluster.Definition.Hosting.Environment}] requires elevated administrator privileges.");
}
if (!hostingManager.Provision(force))
{
Program.Exit(1);
}
// Get the mounted drive prefix from the hosting manager.
cluster.Definition.DrivePrefix = hostingManager.DrivePrefix;
// Ensure that the nodes have valid IP addresses.
cluster.Definition.ValidatePrivateNodeAddresses();
var ipAddressToServer = new Dictionary <IPAddress, SshProxy <NodeDefinition> >();
foreach (var node in cluster.Nodes.OrderBy(n => n.Name))
{
SshProxy <NodeDefinition> duplicateServer;
if (node.PrivateAddress == IPAddress.Any)
{
throw new ArgumentException($"Node [{node.Name}] has not been assigned an IP address.");
}
if (ipAddressToServer.TryGetValue(node.PrivateAddress, out duplicateServer))
{
throw new ArgumentException($"Nodes [{duplicateServer.Name}] and [{node.Name}] have the same IP address [{node.Metadata.PrivateAddress}].");
}
ipAddressToServer.Add(node.PrivateAddress, node);
}
// We're going to use the masters as package caches unless the user
// specifies something else.
packageCaches = commandLine.GetOption("--package-cache"); // This overrides the cluster definition, if specified.
if (!string.IsNullOrEmpty(packageCaches))
{
cluster.Definition.PackageProxy = packageCaches;
}
if (string.IsNullOrEmpty(cluster.Definition.PackageProxy))
{
var sbProxies = new StringBuilder();
foreach (var master in cluster.Masters)
{
sbProxies.AppendWithSeparator($"{master.PrivateAddress}:{NetworkPorts.AppCacherNg}");
}
cluster.Definition.PackageProxy = sbProxies.ToString();
}
//-----------------------------------------------------------------
// Prepare the cluster.
// Write the operation begin marker to all cluster node logs.
cluster.LogLine(logBeginMarker);
var nodesText = cluster.Nodes.Count() == 1 ? "node" : "nodes";
var operation = $"Preparing [{cluster.Definition.Name}] {nodesText}";
var controller =
new SetupController <NodeDefinition>(operation, cluster.Nodes)
{
ShowStatus = !Program.Quiet,
MaxParallel = Program.MaxParallel
};
controller.AddGlobalStep("setup details",
() =>
{
using (var client = new HeadendClient())
{
kubeSetupInfo = client.GetSetupInfoAsync(cluster.Definition).Result;
}
});
// Prepare the nodes.
controller.AddWaitUntilOnlineStep(timeout: TimeSpan.FromMinutes(15));
hostingManager.AddPostProvisionSteps(controller);
controller.AddStep("verify OS", CommonSteps.VerifyOS);
controller.AddStep("prepare",
(node, stepDelay) =>
{
Thread.Sleep(stepDelay);
CommonSteps.PrepareNode(node, cluster.Definition, kubeSetupInfo, shutdown: false);
},
stepStaggerSeconds: cluster.Definition.Setup.StepStaggerSeconds);
if (!controller.Run())
{
// Write the operation end/failed marker to all cluster node logs.
cluster.LogLine(logFailedMarker);
Console.Error.WriteLine("*** ERROR: One or more configuration steps failed.");
Program.Exit(1);
}
// Persist the cluster context extension.
var contextExtensionsPath = KubeHelper.GetContextExtensionPath((KubeContextName)$"{KubeConst.RootUser}@{clusterDefinition.Name}");
var contextExtension = new KubeContextExtension(contextExtensionsPath)
{
ClusterDefinition = clusterDefinition,
SshUsername = Program.MachineUsername,
SshPassword = Program.MachinePassword,
SetupDetails = new KubeSetupDetails()
{
SetupPending = true
}
};
contextExtension.Save();
// Write the operation end marker to all cluster node logs.
cluster.LogLine(logEndMarker);
}
catch
{
failed = true;
throw;
}
finally
{
if (!failed)
{
KubeHelper.Desktop.EndOperationAsync($"Cluster [{cluster.Name}] has been prepared and is ready for setup.").Wait();
}
else
{
KubeHelper.Desktop.EndOperationAsync($"Cluster [{cluster.Name}] prepare has failed.", failed: true).Wait();
}
}
}
