Example #1
0
        /// <inheritdoc/>
        public override void AddProvisioningSteps(SetupController <NodeDefinition> controller)
        {
            Covenant.Requires <ArgumentNullException>(controller != null, nameof(controller));
            Covenant.Assert(cluster != null, $"[{nameof(BareMetalHostingManager)}] was created with the wrong constructor.");

            this.controller = controller;

            throw new NotImplementedException();
        }
Example #2
0
        /// <inheritdoc/>
        public override bool Provision(bool force)
        {
            // Perform the provisioning operations.

            controller = new SetupController <NodeDefinition>($"Provisioning [{cluster.Definition.Name}] cluster", cluster.Nodes)
            {
                ShowStatus  = this.ShowStatus,
                MaxParallel = this.MaxParallel
            };

            controller.AddStep("node labels", (node, stepDelay) => SetLabels(node));

            if (!controller.Run())
            {
                Console.Error.WriteLine("*** ERROR: One or more configuration steps failed.");
                return(false);
            }

            return(true);
        }
Example #3
0
 /// <inheritdoc/>
 public override void AddPostProvisionSteps(SetupController <NodeDefinition> controller)
 {
 }
Example #4
0
        /// <summary>
        /// Constructs the <see cref="ISetupController"/> to be used for preparing a cluster.
        /// </summary>
        /// <param name="clusterDefinition">The cluster definition.</param>
        /// <param name="nodeImageUri">
        /// <para>
        /// Optionally specifies the node image URI.
        /// </para>
        /// <note>
        /// One of <paramref name="nodeImageUri"/> or <paramref name="nodeImagePath"/> must be specified for
        /// on-premise hypervisor based environments.  This is ignored for cloud hosting.
        /// </note>
        /// </param>
        /// <param name="nodeImagePath">
        /// <para>
        /// Optionally specifies the node image path.
        /// </para>
        /// <note>
        /// One of <paramref name="nodeImageUri"/> or <paramref name="nodeImagePath"/> must be specified for
        /// on-premise hypervisor based environments.  This is ignored for cloud hosting.
        /// </note>
        /// </param>
        /// <param name="maxParallel">
        /// Optionally specifies the maximum number of node operations to be performed in parallel.
        /// This <b>defaults to 500</b> which is effectively infinite.
        /// </param>
        /// <param name="packageCacheEndpoints">
        /// <para>
        /// Optionally specifies the IP endpoints for the APT package caches to be used by
        /// the cluster, overriding the cluster definition settings.  This is useful when
        /// package caches are already deployed in an environment.
        /// </para>
        /// <note>
        /// Package cache servers are deployed to the control-plane nodes by default.
        /// </note>
        /// </param>
        /// <param name="unredacted">
        /// Optionally indicates that sensitive information <b>won't be redacted</b> from the setup logs
        /// (typically used when debugging).
        /// </param>
        /// <param name="debugMode">Optionally indicates that the cluster will be prepared in debug mode.</param>
        /// <param name="baseImageName">Optionally specifies the base image name to use for debug mode.</param>
        /// <param name="clusterspace">Optionally specifies the clusterspace for the operation.</param>
        /// <param name="neonCloudHeadendUri">Optionally overrides the headend service URI.  This defaults to <see cref="KubeConst.NeonCloudHeadendUri"/>.</param>
        /// <param name="removeExisting">Optionally remove any existing cluster with the same name in the target environment.</param>
        /// <param name="disableConsoleOutput">
        /// Optionally disables status output to the console.  This is typically
        /// enabled for non-console applications.
        /// </param>
        /// <returns>The <see cref="ISetupController"/>.</returns>
        /// <exception cref="NeonKubeException">Thrown when there's a problem.</exception>
        public static ISetupController CreateClusterPrepareController(
            ClusterDefinition clusterDefinition,
            string nodeImageUri  = null,
            string nodeImagePath = null,
            int maxParallel      = 500,
            IEnumerable <IPEndPoint> packageCacheEndpoints = null,
            bool unredacted            = false,
            bool debugMode             = false,
            string baseImageName       = null,
            string clusterspace        = null,
            string neonCloudHeadendUri = null,
            bool removeExisting        = false,
            bool disableConsoleOutput  = false)
        {
            Covenant.Requires <ArgumentNullException>(clusterDefinition != null, nameof(clusterDefinition));

            if (KubeHelper.IsOnPremiseHypervisorEnvironment(clusterDefinition.Hosting.Environment))
            {
                Covenant.Requires <ArgumentNullException>(!string.IsNullOrEmpty(nodeImageUri) || !string.IsNullOrEmpty(nodeImagePath), $"{nameof(nodeImageUri)}/{nameof(nodeImagePath)}");
            }

            Covenant.Requires <ArgumentException>(maxParallel > 0, nameof(maxParallel));
            Covenant.Requires <ArgumentNullException>(!debugMode || !string.IsNullOrEmpty(baseImageName), nameof(baseImageName));

            neonCloudHeadendUri ??= KubeConst.NeonCloudHeadendUri;

            clusterDefinition.Validate();

            if (!string.IsNullOrEmpty(nodeImagePath))
            {
                if (!File.Exists(nodeImagePath))
                {
                    throw new NeonKubeException($"No node image file exists at: {nodeImagePath}");
                }
            }

            // Determine where the log files should go.

            var logFolder = KubeHelper.LogFolder;

            // Remove any log files left over from a previous prepare/setup operation.

            foreach (var file in Directory.GetFiles(logFolder, "*.*", SearchOption.AllDirectories))
            {
                File.Delete(file);
            }

            // Initialize the cluster proxy.

            var cluster = new ClusterProxy(
                clusterDefinition:      clusterDefinition,
                hostingManagerFactory:  new HostingManagerFactory(() => HostingLoader.Initialize()),
                operation:              ClusterProxy.Operation.Prepare,
                nodeImageUri:           nodeImageUri,
                nodeImagePath:          nodeImagePath,
                nodeProxyCreator:       (nodeName, nodeAddress) =>
            {
                var logStream      = new FileStream(Path.Combine(logFolder, $"{nodeName}.log"), FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite);
                var logWriter      = new StreamWriter(logStream);
                var sshCredentials = SshCredentials.FromUserPassword(KubeConst.SysAdminUser, KubeConst.SysAdminPassword);

                return(new NodeSshProxy <NodeDefinition>(nodeName, nodeAddress, sshCredentials, logWriter: logWriter));
            });

            if (unredacted)
            {
                cluster.SecureRunOptions = RunOptions.None;
            }

            var hostingManager = cluster.HostingManager;

            // Ensure that the nodes have valid IP addresses.

            cluster.Definition.ValidatePrivateNodeAddresses();

            // Override the cluster definition package caches when requested.

            if (packageCacheEndpoints != null && packageCacheEndpoints.Count() > 0)
            {
                var sb = new StringBuilder();

                foreach (var endpoint in packageCacheEndpoints)
                {
                    sb.AppendWithSeparator($"{endpoint.Address}:{endpoint.Port}");
                }

                clusterDefinition.PackageProxy = sb.ToString();
            }

            // Configure the setup controller.

            var controller = new SetupController <NodeDefinition>($"Preparing [{cluster.Definition.Name}] cluster infrastructure", cluster.Nodes, KubeHelper.LogFolder, disableConsoleOutput: disableConsoleOutput)
            {
                MaxParallel     = maxParallel,
                LogBeginMarker  = "# CLUSTER-BEGIN-PREPARE #######################################################",
                LogEndMarker    = "# CLUSTER-END-PREPARE-SUCCESS #################################################",
                LogFailedMarker = "# CLUSTER-END-PREPARE-FAILED ##################################################"
            };

            // Load the cluster login information if it exists and when it indicates that
            // setup is still pending, we'll use that information (especially the generated
            // secure SSH password).
            //
            // Otherwise, we'll fail the cluster prepare to avoid the possiblity of overwriting
            // the login for an active cluster.

            var contextName      = $"{KubeConst.RootUser}@{clusterDefinition.Name}";
            var clusterLoginPath = KubeHelper.GetClusterLoginPath((KubeContextName)contextName);
            var clusterLogin     = ClusterLogin.Load(clusterLoginPath);

            if (clusterLogin == null || !clusterLogin.SetupDetails.SetupPending)
            {
                clusterLogin = new ClusterLogin(clusterLoginPath)
                {
                    ClusterDefinition = clusterDefinition,
                    SshUsername       = KubeConst.SysAdminUser,
                    SetupDetails      = new KubeSetupDetails()
                    {
                        SetupPending = true
                    }
                };

                clusterLogin.Save();
            }
            else
            {
                throw new InvalidOperationException($"Cannot overwrite existing cluster login [{KubeConst.RootUser}@{clusterDefinition.Name}].  Remove the login first when you're VERY SURE IT'S NOT IMPORTANT!");
            }

            // Configure the setup controller state.

            controller.Add(KubeSetupProperty.Preparing, true);
            controller.Add(KubeSetupProperty.ReleaseMode, KubeHelper.IsRelease);
            controller.Add(KubeSetupProperty.DebugMode, debugMode);
            controller.Add(KubeSetupProperty.BaseImageName, baseImageName);
            controller.Add(KubeSetupProperty.MaintainerMode, !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("NC_ROOT")));
            controller.Add(KubeSetupProperty.ClusterProxy, cluster);
            controller.Add(KubeSetupProperty.ClusterLogin, clusterLogin);
            controller.Add(KubeSetupProperty.HostingManager, cluster.HostingManager);
            controller.Add(KubeSetupProperty.HostingEnvironment, cluster.HostingManager.HostingEnvironment);
            controller.Add(KubeSetupProperty.ClusterspaceFolder, clusterspace);
            controller.Add(KubeSetupProperty.NeonCloudHeadendUri, neonCloudHeadendUri);
            controller.Add(KubeSetupProperty.DisableImageDownload, !string.IsNullOrEmpty(nodeImagePath));
            controller.Add(KubeSetupProperty.Redact, !unredacted);

            // Configure the cluster preparation steps.

            controller.AddGlobalStep("configure hosting manager",
                                     controller =>
            {
                controller.SetGlobalStepStatus("configure: hosting manager");

                hostingManager.MaxParallel = maxParallel;
                hostingManager.WaitSeconds = 60;
            });

            // Delete any existing cluster in the environment when requested.

            if (removeExisting)
            {
                controller.AddGlobalStep("remove existing cluster",
                                         async controller =>
                {
                    await hostingManager.RemoveClusterAsync(removeOrphans: true);
                });
            }

            controller.AddGlobalStep("generate credentials",
                                     controller =>
            {
                // We're going to generate a secure random password and we're going to append
                // an extra 4-character string to ensure that the password meets Azure (and probably
                // other cloud) minimum requirements:
                //
                // The supplied password must be between 6-72 characters long and must
                // satisfy at least 3 of password complexity requirements from the following:
                //
                //      1. Contains an uppercase character
                //      2. Contains a lowercase character
                //      3. Contains a numeric digit
                //      4. Contains a special character
                //      5. Control characters are not allowed
                //
                // We're going to use the cloud API to configure this secure password
                // when creating the VMs.  For on-premise hypervisor environments such
                // as Hyper-V and XenServer, we're going use the [neon-init]
                // service to mount a virtual DVD that will change the password before
                // configuring the network on first boot.

                var hostingManager = controller.Get <IHostingManager>(KubeSetupProperty.HostingManager);
                var clusterLogin   = controller.Get <ClusterLogin>(KubeSetupProperty.ClusterLogin);

                controller.SetGlobalStepStatus("generate: SSH password");

                // Generate a secure SSH password and append a string that guarantees that
                // the generated password meets minimum cloud requirements.

                clusterLogin.SshPassword  = NeonHelper.GetCryptoRandomPassword(clusterDefinition.Security.PasswordLength);
                clusterLogin.SshPassword += ".Aa0";

                // We're also going to generate the server's SSH key here and pass that to the hosting
                // manager's provisioner.  We need to do this up front because some hosting environments
                // like Azure don't allow SSH password authentication by default, so we'll need the SSH key
                // to initialize the nodes after they've been provisioned for those environments.

                if (clusterLogin.SshKey == null)
                {
                    // Generate a 2048 bit SSH key pair.

                    controller.SetGlobalStepStatus("generate: SSH client key pair");

                    clusterLogin.SshKey = KubeHelper.GenerateSshKey(cluster.Name, KubeConst.SysAdminUser);
                }

                // We also need to generate the root SSO password when necessary and add this
                // to the cluster login.

                controller.SetGlobalStepStatus("generate: SSO password");

                clusterLogin.SsoUsername = "******";
                clusterLogin.SsoPassword = cluster.Definition.RootPassword ?? NeonHelper.GetCryptoRandomPassword(cluster.Definition.Security.PasswordLength);

                clusterLogin.Save();

                // Update node proxies with the generated SSH credentials.

                foreach (var node in cluster.Nodes)
                {
                    node.UpdateCredentials(clusterLogin.SshCredentials);
                }
            });

            // Have the hosting manager add any custom proviosioning steps.

            hostingManager.AddProvisioningSteps(controller);

            // Add the provisioning steps.

            controller.AddWaitUntilOnlineStep(timeout: TimeSpan.FromMinutes(15));
            controller.AddNodeStep("check node OS", (controller, node) => node.VerifyNodeOS());

            controller.AddNodeStep("delete boot script",
                                   (controller, node) =>
            {
                // Hosting managers may use [cloud-init] to execute custom scripts
                // when node virtual machine first boots to configure networking and
                // also to set a secure SSH password.
                //
                // We need to delete this script file since it includes the SSH password.
                // If present, the script writes the path to itself to:
                //
                //      /etc/neonkube/cloud-init/boot-script-path
                //
                // We're going to read this file if it exists and delete the script.

                var scriptPath = "/etc/neonkube/cloud-init/boot-script-path";

                if (node.FileExists(scriptPath))
                {
                    scriptPath = node.DownloadText(scriptPath);

                    if (!string.IsNullOrEmpty(scriptPath))
                    {
                        node.SudoCommand("rm", "-f", scriptPath.Trim());
                    }
                }
            });

            controller.AddNodeStep("check image version",
                                   (controller, node) =>
            {
                // Ensure that the node image version matches the current neonKUBE version.

                var imageVersion = node.ImageVersion;

                if (imageVersion == null)
                {
                    throw new Exception($"Node image is not stamped with the image version file: {KubeConst.ImageVersionPath}");
                }

                if (imageVersion != SemanticVersion.Parse(KubeVersions.NeonKube))
                {
                    throw new Exception($"Node image version [{imageVersion}] does not match the neonKUBE version [{KubeVersions.NeonKube}] implemented by the current build.");
                }
            });

            controller.AddNodeStep("node credentials",
                                   (controller, node) =>
            {
                node.ConfigureSshKey(controller);
            });

            controller.AddNodeStep("prepare nodes",
                                   (controller, node) =>
            {
                node.PrepareNode(controller);
            });

            controller.AddGlobalStep("neoncluster.io domain",
                                     async controller =>
            {
                controller.SetGlobalStepStatus("create: *.neoncluster.io domain (for TLS)");

                if (string.IsNullOrEmpty(cluster.Definition.Domain))
                {
                    var hostingEnvironment = controller.Get <HostingEnvironment>(KubeSetupProperty.HostingEnvironment);
                    var clusterAddresses   = cluster.HostingManager.GetClusterAddresses();

                    using (var jsonClient = new JsonClient())
                    {
                        jsonClient.BaseAddress = new Uri(controller.Get <string>(KubeSetupProperty.NeonCloudHeadendUri));

                        clusterLogin.ClusterDefinition.Domain = await jsonClient.PostAsync <string>($"/cluster/domain?addresses={string.Join(',', clusterAddresses)}");

                        // $hack(jefflill):
                        //
                        // I'm going to parse the cluster ID from the domain returned.
                        //
                        // https://github.com/nforgeio/neonKUBE/issues/16407
                        //
                        // $todo(marcusbooyah):
                        //
                        // You need to modify the headend API to return an object with the cluster ID
                        // and domain being returned as separate properties and then replace this code.
                        //
                        // Note that the ID should include some dashes to make it easier to read, like:
                        //
                        //      4FCA-0F7A-F7F3-4FC0
                        //
                        // Cluster IDs are probably going to end up being important for customer support
                        // and I suspect that we'll be asking users for this ID when they call in.

                        var fields = clusterLogin.ClusterDefinition.Domain.Split('.');

                        clusterLogin.ClusterDefinition.Id = fields[1];
                    }
                }

                clusterLogin.Save();
            });

            // Some hosting managers may have to some additional work after
            // the cluster has been otherwise prepared.

            hostingManager.AddPostProvisioningSteps(controller);

            // Indicate that cluster prepare succeeded by creating [prepare-ok] file to
            // the log folder.  Cluster setup will verify that this file exists before
            // proceeding.

            controller.AddGlobalStep("finish",
                                     controller =>
            {
                File.Create(Path.Combine(logFolder, "prepare-ok"));
            },
                                     quiet: true);

            // We need to dispose this after the setup controller runs.

            controller.AddDisposable(cluster);

            return(controller);
        }
Example #5
0
        /// <summary>
        /// Constructs the <see cref="ISetupController"/> to be used for setting up a cluster.
        /// </summary>
        /// <param name="clusterDefinition">The cluster definition.</param>
        /// <param name="maxParallel">
        /// Optionally specifies the maximum number of node operations to be performed in parallel.
        /// This <b>defaults to 500</b> which is effectively infinite.
        /// </param>
        /// <param name="unredacted">
        /// Optionally indicates that sensitive information <b>won't be redacted</b> from the setup logs
        /// (typically used when debugging).
        /// </param>
        /// <param name="debugMode">Optionally indicates that the cluster will be prepared in debug mode.</param>
        /// <param name="uploadCharts">
        /// <para>
        /// Optionally specifies that the current Helm charts should be uploaded to replace the charts in the base image.
        /// </para>
        /// <note>
        /// This will be treated as <c>true</c> when <paramref name="debugMode"/> is passed as <c>true</c>.
        /// </note>
        /// </param>
        /// <param name = "clusterspace" > Optionally specifies the clusterspace for the operation.</param>
        /// <param name="neonCloudHeadendUri">Optionally overrides the neonCLOUD headend service URI.  This defaults to <see cref="KubeConst.NeonCloudHeadendUri"/>.</param>
        /// <param name="disableConsoleOutput">
        /// Optionally disables status output to the console.  This is typically
        /// enabled for non-console applications.
        /// </param>
        /// <returns>The <see cref="ISetupController"/>.</returns>
        /// <exception cref="NeonKubeException">Thrown when there's a problem.</exception>
        public static ISetupController CreateClusterSetupController(
            ClusterDefinition clusterDefinition,
            int maxParallel            = 500,
            bool unredacted            = false,
            bool debugMode             = false,
            bool uploadCharts          = false,
            string clusterspace        = null,
            string neonCloudHeadendUri = null,
            bool disableConsoleOutput  = false)
        {
            Covenant.Requires <ArgumentNullException>(clusterDefinition != null, nameof(clusterDefinition));
            Covenant.Requires <ArgumentException>(maxParallel > 0, nameof(maxParallel));

            neonCloudHeadendUri ??= KubeConst.NeonCloudHeadendUri;

            clusterDefinition.Validate();

            // Determine where the log files should go.

            var logFolder = KubeHelper.LogFolder;

            // Ensure that the [prepare-ok] file in the log folder exists, indicating that
            // the last prepare operation succeeded.

            var prepareOkPath = Path.Combine(logFolder, "prepare-ok");

            if (!File.Exists(prepareOkPath))
            {
                throw new NeonKubeException($"Cannot locate the [{prepareOkPath}] file.  Cluster prepare must have failed.");
            }

            // Clear the log folder except for the [prepare-ok] file.

            if (Directory.Exists(logFolder))
            {
                foreach (var file in Directory.GetFiles(logFolder, "*", SearchOption.TopDirectoryOnly))
                {
                    if (Path.GetFileName(file) != "prepare-ok")
                    {
                        NeonHelper.DeleteFile(file);
                    }
                }
            }
            else
            {
                throw new DirectoryNotFoundException(logFolder);
            }

            // Reload the any KubeConfig file to ensure we're up-to-date.

            KubeHelper.LoadConfig();

            // Do some quick checks to ensure that component versions look reasonable.

            //var kubernetesVersion = new Version(KubeVersions.Kubernetes);
            //var crioVersion       = new Version(KubeVersions.Crio);

            //if (crioVersion.Major != kubernetesVersion.Major || crioVersion.Minor != kubernetesVersion.Minor)
            //{
            //    throw new NeonKubeException($"[{nameof(KubeConst)}.{nameof(KubeVersions.Crio)}={KubeVersions.Crio}] major and minor versions don't match [{nameof(KubeConst)}.{nameof(KubeVersions.Kubernetes)}={KubeVersions.Kubernetes}].");
            //}

            // Initialize the cluster proxy.

            var contextName = KubeContextName.Parse($"root@{clusterDefinition.Name}");
            var kubeContext = new KubeConfigContext(contextName);

            KubeHelper.InitContext(kubeContext);

            ClusterProxy cluster = null;

            cluster = new ClusterProxy(
                hostingManagerFactory:  new HostingManagerFactory(() => HostingLoader.Initialize()),
                operation:              ClusterProxy.Operation.Setup,
                clusterDefinition:      clusterDefinition,
                nodeProxyCreator:       (nodeName, nodeAddress) =>
            {
                var logStream      = new FileStream(Path.Combine(logFolder, $"{nodeName}.log"), FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite);
                var logWriter      = new StreamWriter(logStream);
                var context        = KubeHelper.CurrentContext;
                var sshCredentials = context.Extension.SshCredentials ?? SshCredentials.FromUserPassword(KubeConst.SysAdminUser, KubeConst.SysAdminPassword);

                return(new NodeSshProxy <NodeDefinition>(nodeName, nodeAddress, sshCredentials, logWriter: logWriter));
            });

            if (unredacted)
            {
                cluster.SecureRunOptions = RunOptions.None;
            }

            // Configure the setup controller.

            var controller = new SetupController <NodeDefinition>($"Setup [{cluster.Definition.Name}] cluster", cluster.Nodes, KubeHelper.LogFolder, disableConsoleOutput: disableConsoleOutput)
            {
                MaxParallel     = maxParallel,
                LogBeginMarker  = "# CLUSTER-BEGIN-SETUP #########################################################",
                LogEndMarker    = "# CLUSTER-END-SETUP-SUCCESS ###################################################",
                LogFailedMarker = "# CLUSTER-END-SETUP-FAILED ####################################################"
            };

            // Load the cluster login information if it exists and when it indicates that
            // setup is still pending, we'll use that information (especially the generated
            // secure SSH password).
            //
            // Otherwise, we'll write (or overwrite) the context file with a fresh context.

            var clusterLoginPath = KubeHelper.GetClusterLoginPath((KubeContextName)$"{KubeConst.RootUser}@{clusterDefinition.Name}");
            var clusterLogin     = ClusterLogin.Load(clusterLoginPath);

            if (clusterLogin == null || !clusterLogin.SetupDetails.SetupPending)
            {
                clusterLogin = new ClusterLogin(clusterLoginPath)
                {
                    ClusterDefinition = clusterDefinition,
                    SshUsername       = KubeConst.SysAdminUser,
                    SetupDetails      = new KubeSetupDetails()
                    {
                        SetupPending = true
                    }
                };

                clusterLogin.Save();
            }

            // Update the cluster node SSH credentials to use the secure password.

            var sshCredentials = SshCredentials.FromUserPassword(KubeConst.SysAdminUser, clusterLogin.SshPassword);

            foreach (var node in cluster.Nodes)
            {
                node.UpdateCredentials(sshCredentials);
            }

            // Configure the setup controller state.

            controller.Add(KubeSetupProperty.Preparing, false);
            controller.Add(KubeSetupProperty.ReleaseMode, KubeHelper.IsRelease);
            controller.Add(KubeSetupProperty.DebugMode, debugMode);
            controller.Add(KubeSetupProperty.MaintainerMode, !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("NC_ROOT")));
            controller.Add(KubeSetupProperty.ClusterProxy, cluster);
            controller.Add(KubeSetupProperty.ClusterLogin, clusterLogin);
            controller.Add(KubeSetupProperty.HostingManager, cluster.HostingManager);
            controller.Add(KubeSetupProperty.HostingEnvironment, cluster.HostingManager.HostingEnvironment);
            controller.Add(KubeSetupProperty.ClusterspaceFolder, clusterspace);
            controller.Add(KubeSetupProperty.NeonCloudHeadendUri, neonCloudHeadendUri);
            controller.Add(KubeSetupProperty.Redact, !unredacted);

            // Configure the setup steps.

            controller.AddGlobalStep("resource requirements", KubeSetup.CalculateResourceRequirements);

            cluster.HostingManager.AddSetupSteps(controller);

            controller.AddWaitUntilOnlineStep("connect nodes");
            controller.AddNodeStep("check node OS", (controller, node) => node.VerifyNodeOS());

            controller.AddNodeStep("check image version",
                                   (controller, node) =>
            {
                // Ensure that the node image version matches the current neonKUBE (build) version.

                var imageVersion = node.ImageVersion;

                if (imageVersion == null)
                {
                    throw new Exception($"Node image is not stamped with the image version file: {KubeConst.ImageVersionPath}");
                }

                if (imageVersion != SemanticVersion.Parse(KubeVersions.NeonKube))
                {
                    throw new Exception($"Node image version [{imageVersion}] does not match the neonKUBE version [{KubeVersions.NeonKube}] implemented by the current build.");
                }
            });

            controller.AddNodeStep("disable cloud-init", (controller, node) => node.SudoCommand("touch /etc/cloud/cloud-init.disabled"));
            controller.AddNodeStep("node basics", (controller, node) => node.BaseInitialize(controller, upgradeLinux: false));  // $todo(jefflill): We don't support Linux distribution upgrades yet.
            controller.AddNodeStep("root certificates", (controller, node) => node.UpdateRootCertificates());
            controller.AddNodeStep("setup ntp", (controller, node) => node.SetupConfigureNtp(controller));
            controller.AddNodeStep("cluster metadata", ConfigureMetadataAsync);

            // Perform common configuration for the bootstrap node first.
            // We need to do this so the the package cache will be running
            // when the remaining nodes are configured.

            var configureControlPlaneStepLabel = cluster.Definition.ControlNodes.Count() > 1 ? "setup first control-plane node" : "setup control-plane node";

            controller.AddNodeStep(configureControlPlaneStepLabel,
                                   (controller, node) =>
            {
                node.SetupNode(controller, KubeSetup.ClusterManifest);
            },
                                   (controller, node) => node == cluster.FirstControlNode);

            // Perform common configuration for the remaining nodes (if any).

            if (cluster.Definition.Nodes.Count() > 1)
            {
                controller.AddNodeStep("setup other nodes",
                                       (controller, node) =>
                {
                    node.SetupNode(controller, KubeSetup.ClusterManifest);
                    node.InvokeIdempotent("setup/setup-node-restart", () => node.Reboot(wait: true));
                },
                                       (controller, node) => node != cluster.FirstControlNode);
            }

            if (debugMode)
            {
                controller.AddNodeStep("load images", (controller, node) => node.NodeLoadImagesAsync(controller, downloadParallel: 5, loadParallel: 3));
            }

            controller.AddNodeStep("install helm",
                                   (controller, node) =>
            {
                node.NodeInstallHelm(controller);
            });

            controller.AddNodeStep("install kustomize",
                                   (controller, node) =>
            {
                node.NodeInstallKustomize(controller);
            });

            if (uploadCharts || debugMode)
            {
                controller.AddNodeStep("upload helm charts",
                                       (controller, node) =>
                {
                    cluster.FirstControlNode.SudoCommand($"rm -rf {KubeNodeFolder.Helm}/*");
                    cluster.FirstControlNode.NodeInstallHelmArchive(controller);

                    var zipPath = LinuxPath.Combine(KubeNodeFolder.Helm, "charts.zip");

                    cluster.FirstControlNode.SudoCommand($"unzip {zipPath} -d {KubeNodeFolder.Helm}");
                    cluster.FirstControlNode.SudoCommand($"rm -f {zipPath}");
                },
                                       (controller, node) => node == cluster.FirstControlNode);
            }

            //-----------------------------------------------------------------
            // Cluster setup.

            controller.AddGlobalStep("setup cluster", controller => KubeSetup.SetupClusterAsync(controller));
            controller.AddGlobalStep("persist state",
                                     controller =>
            {
                // Indicate that setup is complete.

                clusterLogin.ClusterDefinition.ClearSetupState();
                clusterLogin.SetupDetails.SetupPending = false;
                clusterLogin.Save();
            });

            //-----------------------------------------------------------------
            // Verify the cluster.

            controller.AddNodeStep("check control-plane nodes",
                                   (controller, node) =>
            {
                KubeDiagnostics.CheckControlNode(node, cluster.Definition);
            },
                                   (controller, node) => node.Metadata.IsControlPane);

            if (cluster.Workers.Count() > 0)
            {
                controller.AddNodeStep("check workers",
                                       (controller, node) =>
                {
                    KubeDiagnostics.CheckWorker(node, cluster.Definition);
                },
                                       (controller, node) => node.Metadata.IsWorker);
            }

            cluster.HostingManager.AddPostSetupSteps(controller);

            // We need to dispose this after the setup controller runs.

            controller.AddDisposable(cluster);

            return(controller);
        }
        /// <inheritdoc/>
        public override bool Provision(bool force)
        {
            if (IsProvisionNOP)
            {
                // There's nothing to do here.

                return(true);
            }

            // Update the node labels with the actual capabilities of the
            // virtual machines being provisioned.

            foreach (var node in cluster.Definition.Nodes)
            {
                if (string.IsNullOrEmpty(node.Labels.PhysicalMachine))
                {
                    node.Labels.PhysicalMachine = Environment.MachineName;
                }

                if (node.Labels.ComputeCores == 0)
                {
                    node.Labels.ComputeCores = cluster.Definition.Hosting.VmProcessors;
                }

                if (node.Labels.ComputeRam == 0)
                {
                    node.Labels.ComputeRam = (int)(ClusterDefinition.ValidateSize(cluster.Definition.Hosting.VmMemory, typeof(HostingOptions), nameof(HostingOptions.VmMemory)) / ByteUnits.MebiBytes);
                }

                if (string.IsNullOrEmpty(node.Labels.StorageSize))
                {
                    node.Labels.StorageSize = ByteUnits.ToGiString(node.GetVmMemory(cluster.Definition));
                }
            }

            // If a public address isn't explicitly specified, we'll assume that we're
            // running inside the network and we can access the private address.

            foreach (var node in cluster.Definition.Nodes)
            {
                if (string.IsNullOrEmpty(node.PublicAddress))
                {
                    node.PublicAddress = node.PrivateAddress;
                }
            }

            // Perform the provisioning operations.

            controller = new SetupController <NodeDefinition>($"Provisioning [{cluster.Definition.Name}] cluster", cluster.Nodes)
            {
                ShowStatus  = this.ShowStatus,
                MaxParallel = 1     // We're only going to provision one VM at a time on a local Hyper-V instance.
            };

            controller.AddGlobalStep("prepare hyper-v", () => PrepareHyperV());
            controller.AddStep("virtual machines", (node, stepDelay) => ProvisionVM(node));
            controller.AddGlobalStep(string.Empty, () => Finish(), quiet: true);

            if (!controller.Run())
            {
                Console.Error.WriteLine("*** ERROR: One or more configuration steps failed.");
                return(false);
            }

            return(true);
        }
Example #7
0
 /// <inheritdoc/>
 public virtual void AddPostSetupSteps(SetupController <NodeDefinition> controllerd)
 {
 }
Example #8
0
 /// <inheritdoc/>
 public virtual void AddPostProvisioningSteps(SetupController <NodeDefinition> controller)
 {
 }
Example #9
0
 /// <inheritdoc/>
 public abstract void AddProvisioningSteps(SetupController <NodeDefinition> controller);
Example #10
0
        /// <inheritdoc/>
        public override bool Provision(bool force)
        {
            // $todo(jeff.lill):
            //
            // I'm not implementing [force] here.  I'm not entirely sure
            // that this makes sense for production clusters.
            //
            // Perhaps it would make more sense to replace this with a
            // [neon cluster remove] command.
            //
            //      https://github.com/nforgeio/neonKUBE/issues/156

            if (IsProvisionNOP)
            {
                // There's nothing to do here.

                return(true);
            }

            // Update the node labels with the actual capabilities of the
            // virtual machines being provisioned.

            foreach (var node in cluster.Definition.Nodes)
            {
                if (string.IsNullOrEmpty(node.Labels.PhysicalMachine))
                {
                    node.Labels.PhysicalMachine = node.VmHost;
                }

                if (node.Labels.ComputeCores == 0)
                {
                    node.Labels.ComputeCores = node.GetVmProcessors(cluster.Definition);
                }

                if (node.Labels.ComputeRam == 0)
                {
                    node.Labels.ComputeRam = (int)(node.GetVmMemory(cluster.Definition) / ByteUnits.MebiBytes);
                }

                if (string.IsNullOrEmpty(node.Labels.StorageSize))
                {
                    node.Labels.StorageSize = ByteUnits.ToGiString(node.GetVmDisk(cluster.Definition));
                }
            }

            // Build a list of [SshProxy] instances that map to the specified XenServer
            // hosts.  We'll use the [XenClient] instances as proxy metadata.

            var sshProxies = new List <SshProxy <XenClient> >();

            xenHosts = new List <XenClient>();

            foreach (var host in cluster.Definition.Hosting.VmHosts)
            {
                var hostAddress  = host.Address;
                var hostname     = host.Name;
                var hostUsername = host.Username ?? cluster.Definition.Hosting.VmHostUsername;
                var hostPassword = host.Password ?? cluster.Definition.Hosting.VmHostPassword;

                if (string.IsNullOrEmpty(hostname))
                {
                    hostname = host.Address;
                }

                var xenHost = new XenClient(hostAddress, hostUsername, hostPassword, name: host.Name, logFolder: logFolder);

                xenHosts.Add(xenHost);
                sshProxies.Add(xenHost.SshProxy);
            }

            // We're going to provision the XenServer hosts in parallel to
            // speed up cluster setup.  This works because each XenServer
            // is essentially independent from the others.

            controller = new SetupController <XenClient>($"Provisioning [{cluster.Definition.Name}] cluster", sshProxies)
            {
                ShowStatus  = this.ShowStatus,
                MaxParallel = this.MaxParallel
            };

            controller.AddWaitUntilOnlineStep();

            controller.AddStep("host folders", (node, stepDelay) => node.CreateHostFolders());
            controller.AddStep("verify readiness", (node, stepDelay) => VerifyReady(node));
            controller.AddStep("virtual machine template", (node, stepDelay) => CheckVmTemplate(node));
            controller.AddStep("virtual machines", (node, stepDelay) => ProvisionVirtualMachines(node));
            controller.AddGlobalStep(string.Empty, () => Finish(), quiet: true);

            if (!controller.Run())
            {
                Console.Error.WriteLine("*** ERROR: One or more configuration steps failed.");
                return(false);
            }

            return(true);
        }