Exemplo n.º 1
0
        /// <inheritdoc/>
        public override async Task RunAsync(CommandLine commandLine)
        {
            if (commandLine.HasHelpOption)
            {
                Help();
                Program.Exit(0);
            }

            Console.WriteLine();

            var context = KubeHelper.CurrentContext;

            if (context == null)
            {
                Console.Error.WriteLine($"*** ERROR: There is no current cluster.");
                Program.Exit(1);
            }

            using (var cluster = new ClusterProxy(context, new HostingManagerFactory()))
            {
                var clusterInfo = await cluster.GetClusterInfoAsync();

                Console.WriteLine();
                Console.WriteLine(NeonHelper.JsonSerialize(clusterInfo, Formatting.Indented));
                Console.WriteLine();
            }
        }
Exemplo n.º 2
0
        public Test_Hive(HiveFixture fixture)
        {
            if (!fixture.LoginAndInitialize(action:
                                            () =>
            {
                // This adds a [HostsFixture] to the [HiveFixture] (which inherits
                // from [TestFixtureSet]).  We'll name the HostFixture so we can use
                // it to setup local DNS entries for the tests.

                fixture.AddFixture("hosts", hosts = new HostsFixture());
            }))
            {
                // This call ensures that the hive is reset to a
                // pristine state before each test is invoked.

                fixture.Reset();

                // Retrieve the hosts fixture and reset it.

                hosts = (HostsFixture)fixture["hosts"];
                hosts.Reset();
            }

            this.hiveFixture = fixture;
            this.hive        = fixture.Hive;
        }
Exemplo n.º 3
0
        /// <inheritdoc/>
        public override async Task RunAsync(CommandLine commandLine)
        {
            if (commandLine.HasHelpOption)
            {
                Help();
                Program.Exit(0);
            }

            Console.WriteLine();

            var context = KubeHelper.CurrentContext;

            if (context == null)
            {
                Console.Error.WriteLine($"*** ERROR: There is no current cluster.");
                Program.Exit(1);
            }

            using (var cluster = new ClusterProxy(context, new HostingManagerFactory()))
            {
                var status = await cluster.GetClusterHealthAsync();

                var capabilities = cluster.Capabilities;

                switch (status.State)
                {
                case ClusterState.Off:
                case ClusterState.Paused:

                    if ((capabilities & HostingCapabilities.Stoppable) == 0)
                    {
                        Console.Error.WriteLine($"*** ERROR: Cluster does not support start/stop.");
                        Program.Exit(1);
                    }

                    Console.WriteLine($"Starting: {cluster.Name}...");

                    try
                    {
                        await cluster.StartAsync();

                        Console.WriteLine($"STARTED:  {cluster.Name}");
                    }
                    catch (TimeoutException)
                    {
                        Console.WriteLine();
                        Console.WriteLine($"*** ERROR: Timeout waiting for cluster.");
                    }
                    break;

                default:

                    Console.Error.WriteLine($"*** ERROR: Cluster is already running.");
                    Program.Exit(1);
                    break;
                }
            }
        }
Exemplo n.º 4
0
        /// <inheritdoc/>
        public override async Task RunAsync(CommandLine commandLine)
        {
            if (commandLine.HasHelpOption)
            {
                Help();
                Program.Exit(0);
            }

            Console.WriteLine();

            var context = KubeHelper.CurrentContext;

            if (context == null)
            {
                Console.Error.WriteLine($"*** ERROR: There is no current cluster.");
                Program.Exit(1);
            }

            using (var cluster = new ClusterProxy(context, new HostingManagerFactory()))
            {
                var status = await cluster.GetClusterHealthAsync();

                switch (status.State)
                {
                case ClusterState.Healthy:
                case ClusterState.Unhealthy:

                    var isLocked = await cluster.IsLockedAsync();

                    if (!isLocked.HasValue)
                    {
                        Console.Error.WriteLine($"*** ERROR: [{cluster.Name}] lock status is unknown.");
                        Program.Exit(1);
                    }

                    if (isLocked.Value)
                    {
                        Console.WriteLine($"[{cluster.Name}]: LOCKED");
                        Program.Exit(0);
                    }
                    else
                    {
                        Console.WriteLine($"[{cluster.Name}]: UNLOCKED");
                        Program.Exit(2);
                    }
                    break;

                default:

                    Console.Error.WriteLine($"*** ERROR: Cluster is not running.");
                    Program.Exit(1);
                    break;
                }
            }
        }
Exemplo n.º 5
0
        /// <summary>
        /// Initializes the test fixture to run tests against the current cluster.  This is useful
        /// when developing unit tests against a developer managed cluster.
        /// </summary>
        /// <param name="options">
        /// Optionally specifies the options that <see cref="ClusterFixture"/> will use to
        /// manage the test cluster.
        /// </param>
        /// <returns>This always returns <see cref="TestFixtureStatus.AlreadyRunning"/>.</returns>
        /// <exception cref="NeonKubeException">Thrown when there isn't a current cluster or when it's locked.</exception>
        public TestFixtureStatus StartWithCurrentCluster(ClusterFixtureOptions options = null)
        {
            options ??= new ClusterFixtureOptions();

            // Make a copy of the options and then disable any settings that don't apply to
            // running tests against the current cluster.

            options = options.Clone();

            options.RemoveClusterOnStart   = false;
            options.RemoveClusterOnDispose = false;

            this.options = options;

            // Verify that:
            //
            //      * There is a current cluster
            //      * That it's running
            //      * That it's not locked

            Cluster = new ClusterProxy(KubeHelper.CurrentContext, new HostingManagerFactory());

            try
            {
                var isLocked = Cluster.IsLockedAsync().Result;

                if (!isLocked.HasValue)
                {
                    throw new NeonKubeException("Unable to determine the cluster lock status.");
                }

                if (isLocked.Value)
                {
                    throw new NeonKubeException("Cluster is locked.  Use this command to unlock it: neon cluster unlock");
                }
            }
            catch (NeonKubeException)
            {
                throw;
            }
            catch (Exception e)
            {
                throw new NeonKubeException("Unable to connect cluster.  Is it running?", e);
            }

            started   = true;
            IsRunning = true;

            return(TestFixtureStatus.AlreadyRunning);
        }
Exemplo n.º 6
0
        /// <inheritdoc/>
        public override async Task RunAsync(CommandLine commandLine)
        {
            if (commandLine.HasHelpOption)
            {
                Help();
                Program.Exit(0);
            }

            Console.WriteLine();

            var context = KubeHelper.CurrentContext;

            if (context == null)
            {
                Console.Error.WriteLine($"*** ERROR: There is no current cluster.");
                Program.Exit(1);
            }

            using (var cluster = new ClusterProxy(context, new HostingManagerFactory()))
            {
                var status = await cluster.GetClusterHealthAsync();

                var capabilities = cluster.Capabilities;

                switch (status.State)
                {
                case ClusterState.Healthy:
                case ClusterState.Unhealthy:

                    Console.WriteLine($"Unlocking: {cluster.Name}...");
                    await cluster.UnlockAsync();

                    Console.WriteLine($"UNLOCKED:  {cluster.Name}");
                    break;

                default:

                    Console.Error.WriteLine($"*** ERROR: Cluster is not running.");
                    Program.Exit(1);
                    break;
                }
            }
        }
Exemplo n.º 7
0
        /// <summary>
        /// <para>
        /// Deploys a new test cluster as specified by the cluster definition passed or connects
        /// to a cluster previously deployed by this method when the cluster definition of the
        /// existing cluster and the definition passed here are the same.
        /// </para>
        /// </summary>
        /// <param name="clusterDefinition">The cluster definition model.</param>
        /// <param name="options">
        /// Optionally specifies the options that <see cref="ClusterFixture"/> will use to
        /// manage the test cluster.
        /// </param>
        /// <returns>
        /// <para>
        /// The <see cref="TestFixtureStatus"/>:
        /// </para>
        /// <list type="table">
        /// <item>
        ///     <term><see cref="TestFixtureStatus.Disabled"/></term>
        ///     <description>
        ///     Returned when cluster unit testing is disabled due to the <c>NEON_CLUSTER_TESTING</c> environment
        ///     variable not being present on the current machine which means that <see cref="TestHelper.IsClusterTestingEnabled"/>
        ///     returns <c>false</c>.
        ///     </description>
        /// </item>
        /// <item>
        ///     <term><see cref="TestFixtureStatus.Started"/></term>
        ///     <description>
        ///     Returned when one of the <c>Start()</c> methods is called for the first time for the fixture
        ///     instance, indicating that an existing cluster has been connected or a new cluster has been deployed.
        ///     </description>
        /// </item>
        /// <item>
        ///     <term><see cref="TestFixtureStatus.AlreadyRunning"/></term>
        ///     <description>
        ///     Returned when one of the <c>Start()</c> methods has already been called by your test
        ///     class instance.
        ///     </description>
        /// </item>
        /// </list>
        /// </returns>
        /// <exception cref="NeonKubeException">Thrown when the test cluster could not be deployed.</exception>
        /// <remarks>
        /// <para>
        /// <b>IMPORTANT:</b> Only one <see cref="ClusterFixture"/> can be run at a time on
        /// any one computer.  This is due to the fact that cluster state like the kubeconfig,
        /// neonKUBE logins, logs and other files will be written to <b>~/.neonkube/spaces/$fixture/*</b>
        /// so multiple fixture instances will be confused when trying to manage these same files.
        /// </para>
        /// <para>
        /// This means that not only will running <see cref="ClusterFixture"/> based tests in parallel
        /// within the same instance of Visual Studio fail, but running these tests in different
        /// Visual Studio instances will also fail.
        /// </para>
        /// </remarks>
        public TestFixtureStatus StartWithClusterDefinition(ClusterDefinition clusterDefinition, ClusterFixtureOptions options = null)
        {
            Covenant.Requires <ArgumentNullException>(clusterDefinition != null, nameof(clusterDefinition));

            if (clusterDefinition.IsLocked)
            {
                throw new NeonKubeException("Test clusters need to be unlocked.  Please set [isLocked: false] in your cluster definition.");
            }

            if (!TestHelper.IsClusterTestingEnabled)
            {
                return(TestFixtureStatus.Disabled);
            }

            if (started)
            {
                return(TestFixtureStatus.AlreadyRunning);
            }

            options ??= new ClusterFixtureOptions();
            this.options = options.Clone();

            if (this.Cluster != null)
            {
                return(TestFixtureStatus.AlreadyRunning);
            }

            // Set the clusterspace mode, using any previously downloaded node image unless
            // the user specifies a custom image.  We're going to host the fixture state
            // files in this fixed folder:
            //
            //      ~/.neonkube/spaces/$fixture/*

            clusterspaceFolder = KubeHelper.SetClusterSpaceMode(string.IsNullOrEmpty(options.ImageUriOrPath) ? KubeClusterspaceMode.EnabledWithSharedCache : KubeClusterspaceMode.Enabled, KubeHelper.ClusterspacePrefix("fixture"));

            // Figure out whether the user passed an image URI or file path to override
            // the default node image.

            var imageUriOrPath = options.ImageUriOrPath;
            var imageUri       = (string)null;
            var imagePath      = (string)null;

            if (string.IsNullOrEmpty(imageUriOrPath))
            {
                imageUriOrPath = KubeDownloads.GetDefaultNodeImageUri(clusterDefinition.Hosting.Environment);
            }

            if (imageUriOrPath.StartsWith("http://", StringComparison.InvariantCultureIgnoreCase) || imageUriOrPath.StartsWith("https://", StringComparison.InvariantCultureIgnoreCase))
            {
                imageUri = imageUriOrPath;
            }
            else
            {
                imagePath = imageUriOrPath;
            }

            //-------------------------------------------------------------
            // We need to deal with some scenarios here:
            //
            //  1. No cluster context or login exists for the target cluster.
            //
            //     A conflicting cluster may still exist though, having been deployed
            //     by another computer or perhaps the kubecontext/logins on the current
            //     machine may have been modified.  We need to be sure to remove any
            //     conflicting resources in this case.
            //
            //  2. Cluster context and login exist on the current machine for the target
            //     cluster but the cluster is unhealthy or locked.  We'll abort for locked
            //     clusters and remove and redeploy for unhealth clusters.
            //
            //  3.  Cluster context and login exist and the cluster is healthy.  In this case,
            //      we need to compare the deployed cluster version against the current version
            //      and remove/redeploy when the versions don't match.
            //
            //  4. Cluster context and login exist and the cluster is healthy and cluster versions
            //     match.  In this case,  We'll compare the existing cluster definition with that for
            //     the new cluster and also compare the cluster versions and if they match and
            //     [RemoveClusterOnStart=false] we'll just use the existing cluster.
            //
            //  5. The current cluster matches the target but [RemoveClusterOnStart=true].
            //     We need to remove the current cluster in this case so we'll deploy a
            //     fresh one.

            // Determine whether a test cluster with the same name exists and if
            // its cluster definition matches the test cluster's definition.
            ;
            var clusterExists      = false;
            var clusterContextName = KubeContextName.Parse($"root@{clusterDefinition.Name}");
            var clusterContext     = KubeHelper.Config.GetContext(clusterContextName);
            var clusterLogin       = KubeHelper.GetClusterLogin(clusterContextName);

            if (clusterContext != null && clusterLogin != null && !clusterLogin.SetupDetails.SetupPending)
            {
                clusterExists = ClusterDefinition.AreSimilar(clusterDefinition, clusterLogin.ClusterDefinition);
            }

            if (clusterExists && !options.RemoveClusterOnStart)
            {
                // It looks like the test cluster may already exist.  We'll verify
                // that it's running, healthy, unlocked and the cluster versions match.
                // When all of these conditions are true, we'll use the existing cluster,
                // otherwise we'll remove the cluster as well as its context/login,
                // and deploy a new cluster below.

                using (var cluster = new ClusterProxy(clusterLogin.ClusterDefinition, new HostingManagerFactory()))
                {
                    KubeHelper.SetCurrentContext(clusterContextName);

                    var isLocked      = cluster.IsLockedAsync().ResultWithoutAggregate();
                    var clusterInfo   = cluster.GetClusterInfoAsync().ResultWithoutAggregate();
                    var clusterHealth = cluster.GetClusterHealthAsync().ResultWithoutAggregate();

                    if (isLocked.HasValue && isLocked.Value)
                    {
                        throw new NeonKubeException($"Cluster is locked: {cluster.Name}");
                    }

                    if (clusterHealth.State == ClusterState.Healthy && clusterInfo.ClusterVersion == KubeVersions.NeonKube)
                    {
                        // We need to reset an existing cluster to ensure it's in a known state.

                        cluster.ResetAsync().WaitWithoutAggregate();

                        started   = true;
                        IsRunning = true;
                        Cluster   = new ClusterProxy(KubeHelper.CurrentContext, new HostingManagerFactory());

                        return(TestFixtureStatus.Started);
                    }

                    cluster.RemoveAsync(removeOrphans: true).WaitWithoutAggregate();
                }
            }
            else
            {
                // There is no known existing cluster but there still might be a cluster
                // deployed by another machine or fragments of a partially deployed cluster,
                // so we need to do a preemptive cluster remove.

                using (var cluster = new ClusterProxy(clusterDefinition, new HostingManagerFactory()))
                {
                    cluster.RemoveAsync(removeOrphans: true).WaitWithoutAggregate();
                }
            }

            // Provision the new cluster.

            WriteTestOutputLine($"PREPARE CLUSTER: {clusterDefinition.Name}");

            try
            {
                var controller = KubeSetup.CreateClusterPrepareController(
                    clusterDefinition:   clusterDefinition,
                    nodeImageUri:        imageUri,
                    nodeImagePath:       imagePath,
                    maxParallel:         options.MaxParallel,
                    unredacted:          options.Unredacted,
                    neonCloudHeadendUri: options.NeonCloudHeadendUri);

                switch (controller.RunAsync().ResultWithoutAggregate())
                {
                case SetupDisposition.Succeeded:

                    WriteTestOutputLine("CLUSTER PREPARE: SUCCESS");
                    break;

                case SetupDisposition.Failed:

                    WriteTestOutputLine("CLUSTER PREPARE: FAIL");
                    throw new NeonKubeException("Cluster prepare failed.");

                case SetupDisposition.Cancelled:
                default:

                    throw new NotImplementedException();
                }
            }
            finally
            {
                if (options.CaptureDeploymentLogs)
                {
                    CaptureDeploymentLogs();
                }
            }

            // Setup the cluster.

            WriteTestOutputLine($"SETUP CLUSTER: {clusterDefinition.Name}");

            try
            {
                var controller = KubeSetup.CreateClusterSetupController(
                    clusterDefinition: clusterDefinition,
                    maxParallel:       options.MaxParallel,
                    unredacted:        options.Unredacted);

                switch (controller.RunAsync().ResultWithoutAggregate())
                {
                case SetupDisposition.Succeeded:

                    WriteTestOutputLine("CLUSTER SETUP: SUCCESS");
                    break;

                case SetupDisposition.Failed:

                    WriteTestOutputLine("CLUSTER SETUP: FAILED");
                    throw new NeonKubeException("Cluster setup failed.");

                case SetupDisposition.Cancelled:
                default:

                    throw new NotImplementedException();
                }
            }
            finally
            {
                if (options.CaptureDeploymentLogs)
                {
                    CaptureDeploymentLogs();
                }
            }

            // NOTE: We just deployed brand new cluster so there's no need to reset it.

            started   = true;
            IsRunning = true;
            Cluster   = new ClusterProxy(KubeHelper.CurrentContext, new HostingManagerFactory());

            return(TestFixtureStatus.Started);
        }
Exemplo n.º 8
0
        /// <inheritdoc/>
        public override async Task RunAsync(CommandLine commandLine)
        {
            if (commandLine.HasHelpOption)
            {
                Help();
                Program.Exit(0);
            }

            Console.WriteLine();

            var context = KubeHelper.CurrentContext;

            if (context == null)
            {
                Console.Error.WriteLine($"*** ERROR: There is no current cluster.");
                Program.Exit(1);
            }

            var force = commandLine.HasOption("--force");

            // Determine the individual components to be reset.  We'll default to
            // resetting all of them when none of these options are specified.

            var crio       = commandLine.HasOption("--crio");
            var auth       = commandLine.HasOption("--auth");
            var harbor     = commandLine.HasOption("--harbor");
            var minio      = commandLine.HasOption("--minio");
            var monitoring = commandLine.HasOption("--monitoring");

            if (!crio && !auth && !harbor && !minio && monitoring)
            {
                crio       = true;
                auth       = true;
                harbor     = true;
                minio      = true;
                monitoring = true;
            }

            // Obtain the namespaces to be retained.

            var keep           = commandLine.GetOption("--keep-namespaces");
            var keepNamespaces = new List <string>();

            if (!string.IsNullOrEmpty(keep))
            {
                foreach (var @namespace in keep.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries))
                {
                    keepNamespaces.Add(@namespace);
                }
            }

            using (var cluster = new ClusterProxy(context, new HostingManagerFactory()))
            {
                var status = await cluster.GetClusterHealthAsync();

                switch (status.State)
                {
                case ClusterState.Healthy:
                case ClusterState.Unhealthy:

                    if (!force)
                    {
                        var isLocked = await cluster.IsLockedAsync();

                        if (!isLocked.HasValue)
                        {
                            Console.Error.WriteLine($"*** ERROR: [{cluster.Name}] lock status is unknown.");
                            Program.Exit(1);
                        }

                        if (isLocked.Value)
                        {
                            Console.Error.WriteLine($"*** ERROR: [{cluster.Name}] is locked.");
                            Program.Exit(1);
                        }

                        if (!Program.PromptYesNo($"Are you sure you want to reset: {cluster.Name}?"))
                        {
                            Program.Exit(0);
                        }
                    }

                    Console.WriteLine("Resetting cluster (this may take a while)...");

                    await cluster.ResetAsync(
                        new ClusterResetOptions()
                    {
                        KeepNamespaces  = keepNamespaces,
                        ResetCrio       = crio,
                        ResetAuth       = auth,
                        ResetHarbor     = harbor,
                        ResetMinio      = minio,
                        ResetMonitoring = monitoring
                    },
                        progressMessage => Console.WriteLine(progressMessage));

                    Console.WriteLine($"RESET: {cluster.Name}");
                    break;

                default:

                    Console.Error.WriteLine($"*** ERROR: Cluster is not running.");
                    Program.Exit(1);
                    break;
                }
            }
        }
Exemplo n.º 9
0
        /// <inheritdoc/>
        public override async Task RunAsync(CommandLine commandLine)
        {
            if (commandLine.HasHelpOption)
            {
                Help();
                Program.Exit(0);
            }

            Console.WriteLine();

            var context = KubeHelper.CurrentContext;

            if (context == null)
            {
                Console.Error.WriteLine($"*** ERROR: There is no current cluster.");
                Program.Exit(1);
            }

            var force = commandLine.HasOption("--force");

            using (var cluster = new ClusterProxy(context, new HostingManagerFactory()))
            {
                var capabilities = cluster.Capabilities;

                if ((capabilities & HostingCapabilities.Pausable) == 0)
                {
                    Console.Error.WriteLine($"*** ERROR: Cluster does not support pause/resume.");
                    Program.Exit(1);
                }

                var status = await cluster.GetClusterHealthAsync();

                switch (status.State)
                {
                case ClusterState.Healthy:
                case ClusterState.Unhealthy:

                    if (!force)
                    {
                        var isLocked = await cluster.IsLockedAsync();

                        if (!isLocked.HasValue)
                        {
                            Console.Error.WriteLine($"*** ERROR: [{cluster.Name}] lock status is unknown.");
                            Program.Exit(1);
                        }

                        if (isLocked.Value)
                        {
                            Console.Error.WriteLine($"*** ERROR: [{cluster.Name}] is locked.");
                            Program.Exit(1);
                        }

                        if (!Program.PromptYesNo($"Are you sure you want to pause: {cluster.Name}?"))
                        {
                            Program.Exit(0);
                        }
                    }

                    Console.WriteLine($"Pausing: {cluster.Name}");

                    try
                    {
                        await cluster.StopAsync(StopMode.Pause);

                        Console.WriteLine($"PAUSED:  {cluster.Name}");
                    }
                    catch (TimeoutException)
                    {
                        Console.WriteLine();
                        Console.WriteLine($"*** ERROR: Timeout waiting for cluster.");
                    }
                    break;

                default:

                    Console.Error.WriteLine($"*** ERROR: Cluster is not running.");
                    Program.Exit(1);
                    break;
                }
            }
        }
Exemplo n.º 10
0
        /// <inheritdoc/>
        public override async Task RunAsync(CommandLine commandLine)
        {
            if (commandLine.HasHelpOption)
            {
                Help();
                Program.Exit(0);
            }

            Console.WriteLine();

            var context = KubeHelper.CurrentContext;

            if (context == null)
            {
                Console.Error.WriteLine($"*** ERROR: There is no current cluster.");
                Program.Exit(1);
            }

            var force = commandLine.HasOption("--force");

            using (var cluster = new ClusterProxy(context, new HostingManagerFactory()))
            {
                var capabilities = cluster.Capabilities;

                if ((capabilities & HostingCapabilities.Removable) == 0)
                {
                    Console.Error.WriteLine($"*** ERROR: Cluster is not removable.");
                    Program.Exit(1);
                }

                if (!force)
                {
                    var isLocked = await cluster.IsLockedAsync();

                    if (!isLocked.HasValue)
                    {
                        Console.Error.WriteLine($"*** ERROR: [{cluster.Name}] lock status is unknown.");
                        Program.Exit(1);
                    }

                    if (isLocked.Value)
                    {
                        Console.Error.WriteLine($"*** ERROR: [{cluster.Name}] is locked.");
                        Program.Exit(1);
                    }

                    if (!Program.PromptYesNo($"Are you sure you want to remove: {cluster.Name}?"))
                    {
                        Program.Exit(0);
                    }
                }

                try
                {
                    Console.WriteLine($"Removing: {cluster.Name}...");
                    await cluster.RemoveAsync();

                    KubeHelper.Config.RemoveContext(context);

                    Console.WriteLine($"REMOVED:  {cluster.Name}");
                }
                catch (TimeoutException)
                {
                    Console.WriteLine();
                    Console.WriteLine($"*** ERROR: Timeout waiting for cluster.");
                }
            }
        }
Exemplo n.º 11
0
        /// <inheritdoc/>
        public override async Task RunAsync(CommandLine commandLine)
        {
            if (commandLine.HasHelpOption)
            {
                Help();
                Program.Exit(0);
            }

            Console.WriteLine();

            var context = KubeHelper.CurrentContext;

            if (context == null)
            {
                Console.Error.WriteLine($"*** ERROR: There is no current cluster.");
                Program.Exit(1);
            }

            var turnoff = commandLine.HasOption("--turnoff");
            var force   = commandLine.HasOption("--force");

            using (var cluster = new ClusterProxy(context, new HostingManagerFactory()))
            {
                var capabilities = cluster.Capabilities;

                if ((capabilities & HostingCapabilities.Stoppable) == 0)
                {
                    Console.Error.WriteLine($"*** ERROR: Cluster does not support start/stop.");
                    Program.Exit(1);
                }

                var status = await cluster.GetClusterHealthAsync();

                switch (status.State)
                {
                case ClusterState.Healthy:
                case ClusterState.Unhealthy:

                    if (!force)
                    {
                        var isLocked = await cluster.IsLockedAsync();

                        if (!isLocked.HasValue)
                        {
                            Console.Error.WriteLine($"*** ERROR: [{cluster.Name}] lock status is unknown.");
                            Program.Exit(1);
                        }

                        if (isLocked.Value)
                        {
                            Console.Error.WriteLine($"*** ERROR: [{cluster.Name}] is locked.");
                            Program.Exit(1);
                        }

                        if (!Program.PromptYesNo($"Are you sure you want to stop: {cluster.Name}?"))
                        {
                            Program.Exit(0);
                        }
                    }

                    try
                    {
                        if (turnoff)
                        {
                            Console.WriteLine($"Turning Off: {cluster.Name}");
                        }
                        else
                        {
                            Console.WriteLine($"Stopping: {cluster.Name}...");
                        }

                        await cluster.StopAsync(turnoff?StopMode.TurnOff : StopMode.Graceful);

                        Console.WriteLine($"STOPPED:  {cluster.Name}");
                    }
                    catch (TimeoutException)
                    {
                        Console.WriteLine();
                        Console.WriteLine($"*** ERROR: Timeout waiting for cluster.");
                    }
                    break;

                default:

                    Console.Error.WriteLine($"*** ERROR: Cluster is already stopped.");
                    Program.Exit(1);
                    break;
                }
            }
        }
Exemplo n.º 12
0
        /// <inheritdoc/>
        public override void Run(CommandLine commandLine)
        {
            if (commandLine.HasHelpOption)
            {
                Help();
                Program.Exit(0);
            }

            // Special-case handling of the [--remove-templates] option.

            if (commandLine.HasOption("--remove-templates"))
            {
                Console.WriteLine("Removing cached virtual machine templates.");

                foreach (var fileName in Directory.GetFiles(KubeHelper.VmTemplatesFolder, "*.*", SearchOption.TopDirectoryOnly))
                {
                    File.Delete(fileName);
                }

                Program.Exit(0);
            }

            // Implement the command.

            if (KubeHelper.CurrentContext != null)
            {
                Console.Error.WriteLine("*** ERROR: You are logged into a cluster.  You need to logout before preparing another.");
                Program.Exit(1);
            }

            if (commandLine.Arguments.Length == 0)
            {
                Console.Error.WriteLine($"*** ERROR: CLUSTER-DEF expected.");
                Program.Exit(1);
            }

            clusterDefPath = commandLine.Arguments[0];
            force          = commandLine.GetFlag("--force");

            ClusterDefinition.ValidateFile(clusterDefPath, strict: true);

            var clusterDefinition = ClusterDefinition.FromFile(clusterDefPath, strict: true);

            clusterDefinition.Provisioner = $"neon-cli:{Program.Version}";  // Identify this tool/version as the cluster provisioner

            // NOTE:
            //
            // Azure has a more restrictive password policy and our default
            // machine password does not meet the requirements:
            //
            // The supplied password must be between 6-72 characters long and must
            // satisfy at least 3 of password complexity requirements from the following:
            //
            //      1. Contains an uppercase character
            //      2. Contains a lowercase character
            //      3. Contains a numeric digit
            //      4. Contains a special character
            //      5. Control characters are not allowed
            //
            // It's also probably not a great idea to use a static password when
            // provisioning VMs in public clouds because it might be possible for
            // somebody to use this fact the SSH into nodes while the cluster is
            // being setup and before we set the secure password at the end.
            //
            // This is less problematic for non-cloud environments because it's
            // likely that the hosts won't initially be able to receive inbound
            // Internet traffic and besides, we need to have a known password
            // embedded into the VM templates.
            //
            // We're going to handle this for cloud environments by looking
            // at [Program.MachinePassword].  If this is set to the default
            // machine password then we're going to replace it with a randomlly
            // generated password with a few extra characters to ensure that
            // it meets the target cloud's password requirements.  We'll use
            // a non-default password if the operator specified one.

            if (clusterDefinition.Hosting.IsCloudProvider && Program.MachinePassword == KubeConst.DefaulVmTemplatePassword)
            {
                Program.MachinePassword = NeonHelper.GetCryptoRandomPassword(20);

                // Append a string that guarantees that the generated password meets
                // cloud minimum requirements.

                Program.MachinePassword += ".Aa0";
            }

            // NOTE: Cluster prepare starts new log files.

            cluster = new ClusterProxy(clusterDefinition, Program.CreateNodeProxy <NodeDefinition>, appendToLog: false, defaultRunOptions: RunOptions.LogOutput | RunOptions.FaultOnError);

            if (KubeHelper.Config.GetContext(cluster.Definition.Name) != null)
            {
                Console.Error.WriteLine($"*** ERROR: A context named [{cluster.Definition.Name}] already exists.");
                Program.Exit(1);
            }

            // Configure global options.

            if (commandLine.HasOption("--unredacted"))
            {
                cluster.SecureRunOptions = RunOptions.None;
            }

            var failed = false;

            try
            {
                KubeHelper.Desktop.StartOperationAsync($"Preparing [{cluster.Name}]").Wait();

                //-----------------------------------------------------------------
                // Try to ensure that no servers are already deployed on the IP addresses defined
                // for cluster nodes because provisoning over an existing cluster will likely
                // corrupt the existing cluster and also probably prevent the new cluster from
                // provisioning correctly.
                //
                // Note that we're not going to perform this check for the [Machine] hosting
                // environment because we're expecting the bare machines to be already running
                // with the assigned addresses and we're also not going to do this for cloud
                // environments because we're assuming that the cluster will run in its own
                // private network so there'll ne no possibility of conflicts.

                if (cluster.Definition.Hosting.Environment != HostingEnvironments.Machine &&
                    !cluster.Definition.Hosting.IsCloudProvider)
                {
                    Console.WriteLine();
                    Console.WriteLine(" Scanning for IP address conflicts...");
                    Console.WriteLine();

                    var pingOptions   = new PingOptions(ttl: 32, dontFragment: true);
                    var pingTimeout   = TimeSpan.FromSeconds(2);
                    var pingConflicts = new List <NodeDefinition>();
                    var pingAttempts  = 2;

                    // I'm going to use up to 20 threads at a time here for simplicity
                    // rather then doing this as async operations.

                    var parallelOptions = new ParallelOptions()
                    {
                        MaxDegreeOfParallelism = 20
                    };

                    Parallel.ForEach(cluster.Definition.NodeDefinitions.Values, parallelOptions,
                                     node =>
                    {
                        using (var pinger = new Pinger())
                        {
                            // We're going to try pinging up to [pingAttempts] times for each node
                            // just in case the network it sketchy and we're losing reply packets.

                            for (int i = 0; i < pingAttempts; i++)
                            {
                                var reply = pinger.SendPingAsync(node.PrivateAddress, (int)pingTimeout.TotalMilliseconds).Result;

                                if (reply.Status == IPStatus.Success)
                                {
                                    lock (pingConflicts)
                                    {
                                        pingConflicts.Add(node);
                                    }

                                    break;
                                }
                            }
                        }
                    });

                    if (pingConflicts.Count > 0)
                    {
                        Console.Error.WriteLine($"*** ERROR: Cannot provision the cluster because [{pingConflicts.Count}] other");
                        Console.Error.WriteLine($"***        machines conflict with the following cluster nodes:");
                        Console.Error.WriteLine();

                        foreach (var node in pingConflicts.OrderBy(n => NetHelper.AddressToUint(IPAddress.Parse(n.PrivateAddress))))
                        {
                            Console.Error.WriteLine($"{node.PrivateAddress, 16}:    {node.Name}");
                        }

                        Program.Exit(1);
                    }
                }

                //-----------------------------------------------------------------
                // Perform basic environment provisioning.  This creates basic cluster components
                // such as virtual machines, networks, load balancers, public IP addresses, security
                // groups,... as required for the environment.

                hostingManager = new HostingManagerFactory(() => HostingLoader.Initialize()).GetMaster(cluster, Program.LogPath);

                if (hostingManager == null)
                {
                    Console.Error.WriteLine($"*** ERROR: No hosting manager for the [{cluster.Definition.Hosting.Environment}] hosting environment could be located.");
                    Program.Exit(1);
                }

                hostingManager.HostUsername = Program.MachineUsername;
                hostingManager.HostPassword = Program.MachinePassword;
                hostingManager.ShowStatus   = !Program.Quiet;
                hostingManager.MaxParallel  = Program.MaxParallel;
                hostingManager.WaitSeconds  = Program.WaitSeconds;

                if (hostingManager.RequiresAdminPrivileges)
                {
                    Program.VerifyAdminPrivileges($"Provisioning to [{cluster.Definition.Hosting.Environment}] requires elevated administrator privileges.");
                }

                if (!hostingManager.Provision(force))
                {
                    Program.Exit(1);
                }

                // Get the mounted drive prefix from the hosting manager.

                cluster.Definition.DrivePrefix = hostingManager.DrivePrefix;

                // Ensure that the nodes have valid IP addresses.

                cluster.Definition.ValidatePrivateNodeAddresses();

                var ipAddressToServer = new Dictionary <IPAddress, SshProxy <NodeDefinition> >();

                foreach (var node in cluster.Nodes.OrderBy(n => n.Name))
                {
                    SshProxy <NodeDefinition> duplicateServer;

                    if (node.PrivateAddress == IPAddress.Any)
                    {
                        throw new ArgumentException($"Node [{node.Name}] has not been assigned an IP address.");
                    }

                    if (ipAddressToServer.TryGetValue(node.PrivateAddress, out duplicateServer))
                    {
                        throw new ArgumentException($"Nodes [{duplicateServer.Name}] and [{node.Name}] have the same IP address [{node.Metadata.PrivateAddress}].");
                    }

                    ipAddressToServer.Add(node.PrivateAddress, node);
                }

                // We're going to use the masters as package caches unless the user
                // specifies something else.

                packageCaches = commandLine.GetOption("--package-cache");     // This overrides the cluster definition, if specified.

                if (!string.IsNullOrEmpty(packageCaches))
                {
                    cluster.Definition.PackageProxy = packageCaches;
                }

                if (string.IsNullOrEmpty(cluster.Definition.PackageProxy))
                {
                    var sbProxies = new StringBuilder();

                    foreach (var master in cluster.Masters)
                    {
                        sbProxies.AppendWithSeparator($"{master.PrivateAddress}:{NetworkPorts.AppCacherNg}");
                    }

                    cluster.Definition.PackageProxy = sbProxies.ToString();
                }

                //-----------------------------------------------------------------
                // Prepare the cluster.

                // Write the operation begin marker to all cluster node logs.

                cluster.LogLine(logBeginMarker);

                var nodesText = cluster.Nodes.Count() == 1 ? "node" : "nodes";
                var operation = $"Preparing [{cluster.Definition.Name}] {nodesText}";

                var controller =
                    new SetupController <NodeDefinition>(operation, cluster.Nodes)
                {
                    ShowStatus  = !Program.Quiet,
                    MaxParallel = Program.MaxParallel
                };

                controller.AddGlobalStep("setup details",
                                         () =>
                {
                    using (var client = new HeadendClient())
                    {
                        kubeSetupInfo = client.GetSetupInfoAsync(cluster.Definition).Result;
                    }
                });

                // Prepare the nodes.

                controller.AddWaitUntilOnlineStep(timeout: TimeSpan.FromMinutes(15));
                hostingManager.AddPostProvisionSteps(controller);
                controller.AddStep("verify OS", CommonSteps.VerifyOS);

                controller.AddStep("prepare",
                                   (node, stepDelay) =>
                {
                    Thread.Sleep(stepDelay);
                    CommonSteps.PrepareNode(node, cluster.Definition, kubeSetupInfo, shutdown: false);
                },
                                   stepStaggerSeconds: cluster.Definition.Setup.StepStaggerSeconds);

                if (!controller.Run())
                {
                    // Write the operation end/failed marker to all cluster node logs.

                    cluster.LogLine(logFailedMarker);

                    Console.Error.WriteLine("*** ERROR: One or more configuration steps failed.");
                    Program.Exit(1);
                }

                // Persist the cluster context extension.

                var contextExtensionsPath = KubeHelper.GetContextExtensionPath((KubeContextName)$"{KubeConst.RootUser}@{clusterDefinition.Name}");
                var contextExtension      = new KubeContextExtension(contextExtensionsPath)
                {
                    ClusterDefinition = clusterDefinition,
                    SshUsername       = Program.MachineUsername,
                    SshPassword       = Program.MachinePassword,
                    SetupDetails      = new KubeSetupDetails()
                    {
                        SetupPending = true
                    }
                };

                contextExtension.Save();

                // Write the operation end marker to all cluster node logs.

                cluster.LogLine(logEndMarker);
            }
            catch
            {
                failed = true;
                throw;
            }
            finally
            {
                if (!failed)
                {
                    KubeHelper.Desktop.EndOperationAsync($"Cluster [{cluster.Name}] has been prepared and is ready for setup.").Wait();
                }
                else
                {
                    KubeHelper.Desktop.EndOperationAsync($"Cluster [{cluster.Name}] prepare has failed.", failed: true).Wait();
                }
            }
        }
Exemplo n.º 13
0
        /// <inheritdoc/>
        public override async Task RunAsync(CommandLine commandLine)
        {
            if (commandLine.HasHelpOption)
            {
                Help();
                Program.Exit(0);
            }

            Console.WriteLine();

            // Cluster prepare/setup uses the [ProfileClient] to retrieve secrets and profile values.
            // We need to inject an implementation for [PreprocessReader] so it will be able to
            // perform the lookups.

            NeonHelper.ServiceContainer.AddSingleton <IProfileClient>(new ProfileClient());

            // Handle the [--remove-templates] option.

            if (commandLine.HasOption("--remove-templates"))
            {
                Console.WriteLine("Removing cached virtual machine templates.");

                foreach (var fileName in Directory.GetFiles(KubeHelper.NodeImageFolder, "*.*", SearchOption.TopDirectoryOnly))
                {
                    File.Delete(fileName);
                }
            }

            var nodeImageUri      = commandLine.GetOption("--node-image-uri");
            var nodeImagePath     = commandLine.GetOption("--node-image-path");
            var debug             = commandLine.HasOption("--debug");
            var baseImageName     = commandLine.GetOption("--base-image-name");
            var clusterspace      = commandLine.GetOption("--clusterspace");
            var headendUri        = commandLine.GetOption("--headend-uri") ?? KubeConst.NeonCloudHeadendUri;
            var maxParallelOption = commandLine.GetOption("--max-parallel", "6");
            var disablePending    = commandLine.HasOption("--disable-pending");

            if (!int.TryParse(maxParallelOption, out var maxParallel) || maxParallel <= 0)
            {
                Console.Error.WriteLine($"*** ERROR: [--max-parallel={maxParallelOption}] is not valid.");
                Program.Exit(1);
            }

            if (debug && string.IsNullOrEmpty(baseImageName))
            {
                Console.Error.WriteLine($"*** ERROR: [--base-image-name] is required for [--debug] mode.");
                Program.Exit(1);
            }

            // Implement the command.

            if (KubeHelper.CurrentContext != null)
            {
                Console.Error.WriteLine("*** ERROR: You are logged into a cluster.  You need to logout before preparing another.");
                Program.Exit(1);
            }

            if (commandLine.Arguments.Length == 0)
            {
                Console.Error.WriteLine($"*** ERROR: CLUSTER-DEF expected.");
                Program.Exit(1);
            }

            // Obtain the cluster definition.

            var clusterDefPath    = commandLine.Arguments[0];
            var clusterDefinition = (ClusterDefinition)null;

            ClusterDefinition.ValidateFile(clusterDefPath, strict: true);

            clusterDefinition = ClusterDefinition.FromFile(clusterDefPath, strict: true);

            // Do a quick sanity check to ensure that the hosting environment has enough
            // resources (memory and disk) to actually host the cluster.

            using (var cluster = new ClusterProxy(clusterDefinition, new HostingManagerFactory()))
            {
                var status = await cluster.GetResourceAvailabilityAsync();

                if (!status.CanBeDeployed)
                {
                    Console.Error.WriteLine();
                    Console.Error.WriteLine($"*** ERROR: Insufficent resources available to deploy cluster.");
                    Console.Error.WriteLine();

                    foreach (var entity in status.Constraints.Keys
                             .OrderBy(key => key, StringComparer.InvariantCultureIgnoreCase))
                    {
                        Console.Error.WriteLine();
                        Console.Error.WriteLine($"{entity}:");

                        foreach (var constraint in status.Constraints[entity])
                        {
                            Console.Error.WriteLine($"    {constraint.ResourceType.ToString().ToUpperInvariant()}: {constraint.Details}");
                        }
                    }

                    Console.Error.WriteLine();
                    Program.Exit(1);
                }
            }

            if (KubeHelper.IsOnPremiseHypervisorEnvironment(clusterDefinition.Hosting.Environment))
            {
                // Use the default node image for the hosting environment unless [--node-image-uri]
                // or [--node-image-path] was specified.

                if (string.IsNullOrEmpty(nodeImageUri) && string.IsNullOrEmpty(nodeImagePath))
                {
                    nodeImageUri = KubeDownloads.GetDefaultNodeImageUri(clusterDefinition.Hosting.Environment);
                }
            }

            // Parse any specified package cache endpoints.

            var packageCaches         = commandLine.GetOption("--package-caches", null);
            var packageCacheEndpoints = new List <IPEndPoint>();

            if (!string.IsNullOrEmpty(packageCaches))
            {
                foreach (var item in packageCaches.Split(' ', StringSplitOptions.RemoveEmptyEntries))
                {
                    if (!NetHelper.TryParseIPv4Endpoint(item, out var endpoint))
                    {
                        Console.Error.WriteLine($"*** ERROR: [{item}] is not a valid package cache IPv4 endpoint.");
                        Program.Exit(1);
                    }

                    packageCacheEndpoints.Add(endpoint);
                }
            }

            // Create and run the cluster prepare controller.

            var controller = KubeSetup.CreateClusterPrepareController(
                clusterDefinition,
                nodeImageUri:           nodeImageUri,
                nodeImagePath:          nodeImagePath,
                maxParallel:            maxParallel,
                packageCacheEndpoints:  packageCacheEndpoints,
                unredacted:             commandLine.HasOption("--unredacted"),
                debugMode:              debug,
                baseImageName:          baseImageName,
                clusterspace:           clusterspace,
                neonCloudHeadendUri:    headendUri);

            controller.DisablePendingTasks = disablePending;

            controller.StatusChangedEvent +=
                status =>
            {
                status.WriteToConsole();
            };

            switch (await controller.RunAsync())
            {
            case SetupDisposition.Succeeded:

                var pendingGroups = controller.GetPendingGroups();

                if (pendingGroups.Count > 0)
                {
                    Console.WriteLine($"*** ERROR: [{pendingGroups.Count}] pending task groups have not been awaited:");
                    Console.WriteLine();

                    foreach (var groupName in pendingGroups)
                    {
                        Console.WriteLine($"   {groupName}");
                    }

                    Program.Exit(1);
                }

                Console.WriteLine();
                Console.WriteLine($" [{clusterDefinition.Name}] cluster is prepared.");
                Console.WriteLine();
                Program.Exit(0);
                break;

            case SetupDisposition.Cancelled:

                Console.WriteLine();
                Console.WriteLine(" *** CANCELLED: Cluster prepare was cancelled.");
                Console.WriteLine();
                Program.Exit(1);
                break;

            case SetupDisposition.Failed:

                Console.WriteLine();
                Console.WriteLine(" *** ERROR: Cluster prepare has failed.  Examine the logs here:");
                Console.WriteLine();
                Console.WriteLine($" {KubeHelper.LogFolder}");
                Console.WriteLine();
                Program.Exit(1);
                break;

            default:

                throw new NotImplementedException();
            }

            await Task.CompletedTask;
        }