/// <inheritdoc/> public override async Task RunAsync(CommandLine commandLine) { if (commandLine.HasHelpOption) { Help(); Program.Exit(0); } Console.WriteLine(); var context = KubeHelper.CurrentContext; if (context == null) { Console.Error.WriteLine($"*** ERROR: There is no current cluster."); Program.Exit(1); } using (var cluster = new ClusterProxy(context, new HostingManagerFactory())) { var clusterInfo = await cluster.GetClusterInfoAsync(); Console.WriteLine(); Console.WriteLine(NeonHelper.JsonSerialize(clusterInfo, Formatting.Indented)); Console.WriteLine(); } }
public Test_Hive(HiveFixture fixture) { if (!fixture.LoginAndInitialize(action: () => { // This adds a [HostsFixture] to the [HiveFixture] (which inherits // from [TestFixtureSet]). We'll name the HostFixture so we can use // it to setup local DNS entries for the tests. fixture.AddFixture("hosts", hosts = new HostsFixture()); })) { // This call ensures that the hive is reset to a // pristine state before each test is invoked. fixture.Reset(); // Retrieve the hosts fixture and reset it. hosts = (HostsFixture)fixture["hosts"]; hosts.Reset(); } this.hiveFixture = fixture; this.hive = fixture.Hive; }
/// <inheritdoc/> public override async Task RunAsync(CommandLine commandLine) { if (commandLine.HasHelpOption) { Help(); Program.Exit(0); } Console.WriteLine(); var context = KubeHelper.CurrentContext; if (context == null) { Console.Error.WriteLine($"*** ERROR: There is no current cluster."); Program.Exit(1); } using (var cluster = new ClusterProxy(context, new HostingManagerFactory())) { var status = await cluster.GetClusterHealthAsync(); var capabilities = cluster.Capabilities; switch (status.State) { case ClusterState.Off: case ClusterState.Paused: if ((capabilities & HostingCapabilities.Stoppable) == 0) { Console.Error.WriteLine($"*** ERROR: Cluster does not support start/stop."); Program.Exit(1); } Console.WriteLine($"Starting: {cluster.Name}..."); try { await cluster.StartAsync(); Console.WriteLine($"STARTED: {cluster.Name}"); } catch (TimeoutException) { Console.WriteLine(); Console.WriteLine($"*** ERROR: Timeout waiting for cluster."); } break; default: Console.Error.WriteLine($"*** ERROR: Cluster is already running."); Program.Exit(1); break; } } }
/// <inheritdoc/> public override async Task RunAsync(CommandLine commandLine) { if (commandLine.HasHelpOption) { Help(); Program.Exit(0); } Console.WriteLine(); var context = KubeHelper.CurrentContext; if (context == null) { Console.Error.WriteLine($"*** ERROR: There is no current cluster."); Program.Exit(1); } using (var cluster = new ClusterProxy(context, new HostingManagerFactory())) { var status = await cluster.GetClusterHealthAsync(); switch (status.State) { case ClusterState.Healthy: case ClusterState.Unhealthy: var isLocked = await cluster.IsLockedAsync(); if (!isLocked.HasValue) { Console.Error.WriteLine($"*** ERROR: [{cluster.Name}] lock status is unknown."); Program.Exit(1); } if (isLocked.Value) { Console.WriteLine($"[{cluster.Name}]: LOCKED"); Program.Exit(0); } else { Console.WriteLine($"[{cluster.Name}]: UNLOCKED"); Program.Exit(2); } break; default: Console.Error.WriteLine($"*** ERROR: Cluster is not running."); Program.Exit(1); break; } } }
/// <summary> /// Initializes the test fixture to run tests against the current cluster. This is useful /// when developing unit tests against a developer managed cluster. /// </summary> /// <param name="options"> /// Optionally specifies the options that <see cref="ClusterFixture"/> will use to /// manage the test cluster. /// </param> /// <returns>This always returns <see cref="TestFixtureStatus.AlreadyRunning"/>.</returns> /// <exception cref="NeonKubeException">Thrown when there isn't a current cluster or when it's locked.</exception> public TestFixtureStatus StartWithCurrentCluster(ClusterFixtureOptions options = null) { options ??= new ClusterFixtureOptions(); // Make a copy of the options and then disable any settings that don't apply to // running tests against the current cluster. options = options.Clone(); options.RemoveClusterOnStart = false; options.RemoveClusterOnDispose = false; this.options = options; // Verify that: // // * There is a current cluster // * That it's running // * That it's not locked Cluster = new ClusterProxy(KubeHelper.CurrentContext, new HostingManagerFactory()); try { var isLocked = Cluster.IsLockedAsync().Result; if (!isLocked.HasValue) { throw new NeonKubeException("Unable to determine the cluster lock status."); } if (isLocked.Value) { throw new NeonKubeException("Cluster is locked. Use this command to unlock it: neon cluster unlock"); } } catch (NeonKubeException) { throw; } catch (Exception e) { throw new NeonKubeException("Unable to connect cluster. Is it running?", e); } started = true; IsRunning = true; return(TestFixtureStatus.AlreadyRunning); }
/// <inheritdoc/> public override async Task RunAsync(CommandLine commandLine) { if (commandLine.HasHelpOption) { Help(); Program.Exit(0); } Console.WriteLine(); var context = KubeHelper.CurrentContext; if (context == null) { Console.Error.WriteLine($"*** ERROR: There is no current cluster."); Program.Exit(1); } using (var cluster = new ClusterProxy(context, new HostingManagerFactory())) { var status = await cluster.GetClusterHealthAsync(); var capabilities = cluster.Capabilities; switch (status.State) { case ClusterState.Healthy: case ClusterState.Unhealthy: Console.WriteLine($"Unlocking: {cluster.Name}..."); await cluster.UnlockAsync(); Console.WriteLine($"UNLOCKED: {cluster.Name}"); break; default: Console.Error.WriteLine($"*** ERROR: Cluster is not running."); Program.Exit(1); break; } } }
/// <summary> /// <para> /// Deploys a new test cluster as specified by the cluster definition passed or connects /// to a cluster previously deployed by this method when the cluster definition of the /// existing cluster and the definition passed here are the same. /// </para> /// </summary> /// <param name="clusterDefinition">The cluster definition model.</param> /// <param name="options"> /// Optionally specifies the options that <see cref="ClusterFixture"/> will use to /// manage the test cluster. /// </param> /// <returns> /// <para> /// The <see cref="TestFixtureStatus"/>: /// </para> /// <list type="table"> /// <item> /// <term><see cref="TestFixtureStatus.Disabled"/></term> /// <description> /// Returned when cluster unit testing is disabled due to the <c>NEON_CLUSTER_TESTING</c> environment /// variable not being present on the current machine which means that <see cref="TestHelper.IsClusterTestingEnabled"/> /// returns <c>false</c>. /// </description> /// </item> /// <item> /// <term><see cref="TestFixtureStatus.Started"/></term> /// <description> /// Returned when one of the <c>Start()</c> methods is called for the first time for the fixture /// instance, indicating that an existing cluster has been connected or a new cluster has been deployed. /// </description> /// </item> /// <item> /// <term><see cref="TestFixtureStatus.AlreadyRunning"/></term> /// <description> /// Returned when one of the <c>Start()</c> methods has already been called by your test /// class instance. /// </description> /// </item> /// </list> /// </returns> /// <exception cref="NeonKubeException">Thrown when the test cluster could not be deployed.</exception> /// <remarks> /// <para> /// <b>IMPORTANT:</b> Only one <see cref="ClusterFixture"/> can be run at a time on /// any one computer. This is due to the fact that cluster state like the kubeconfig, /// neonKUBE logins, logs and other files will be written to <b>~/.neonkube/spaces/$fixture/*</b> /// so multiple fixture instances will be confused when trying to manage these same files. /// </para> /// <para> /// This means that not only will running <see cref="ClusterFixture"/> based tests in parallel /// within the same instance of Visual Studio fail, but running these tests in different /// Visual Studio instances will also fail. /// </para> /// </remarks> public TestFixtureStatus StartWithClusterDefinition(ClusterDefinition clusterDefinition, ClusterFixtureOptions options = null) { Covenant.Requires <ArgumentNullException>(clusterDefinition != null, nameof(clusterDefinition)); if (clusterDefinition.IsLocked) { throw new NeonKubeException("Test clusters need to be unlocked. Please set [isLocked: false] in your cluster definition."); } if (!TestHelper.IsClusterTestingEnabled) { return(TestFixtureStatus.Disabled); } if (started) { return(TestFixtureStatus.AlreadyRunning); } options ??= new ClusterFixtureOptions(); this.options = options.Clone(); if (this.Cluster != null) { return(TestFixtureStatus.AlreadyRunning); } // Set the clusterspace mode, using any previously downloaded node image unless // the user specifies a custom image. We're going to host the fixture state // files in this fixed folder: // // ~/.neonkube/spaces/$fixture/* clusterspaceFolder = KubeHelper.SetClusterSpaceMode(string.IsNullOrEmpty(options.ImageUriOrPath) ? KubeClusterspaceMode.EnabledWithSharedCache : KubeClusterspaceMode.Enabled, KubeHelper.ClusterspacePrefix("fixture")); // Figure out whether the user passed an image URI or file path to override // the default node image. var imageUriOrPath = options.ImageUriOrPath; var imageUri = (string)null; var imagePath = (string)null; if (string.IsNullOrEmpty(imageUriOrPath)) { imageUriOrPath = KubeDownloads.GetDefaultNodeImageUri(clusterDefinition.Hosting.Environment); } if (imageUriOrPath.StartsWith("http://", StringComparison.InvariantCultureIgnoreCase) || imageUriOrPath.StartsWith("https://", StringComparison.InvariantCultureIgnoreCase)) { imageUri = imageUriOrPath; } else { imagePath = imageUriOrPath; } //------------------------------------------------------------- // We need to deal with some scenarios here: // // 1. No cluster context or login exists for the target cluster. // // A conflicting cluster may still exist though, having been deployed // by another computer or perhaps the kubecontext/logins on the current // machine may have been modified. We need to be sure to remove any // conflicting resources in this case. // // 2. Cluster context and login exist on the current machine for the target // cluster but the cluster is unhealthy or locked. We'll abort for locked // clusters and remove and redeploy for unhealth clusters. // // 3. Cluster context and login exist and the cluster is healthy. In this case, // we need to compare the deployed cluster version against the current version // and remove/redeploy when the versions don't match. // // 4. Cluster context and login exist and the cluster is healthy and cluster versions // match. In this case, We'll compare the existing cluster definition with that for // the new cluster and also compare the cluster versions and if they match and // [RemoveClusterOnStart=false] we'll just use the existing cluster. // // 5. The current cluster matches the target but [RemoveClusterOnStart=true]. // We need to remove the current cluster in this case so we'll deploy a // fresh one. // Determine whether a test cluster with the same name exists and if // its cluster definition matches the test cluster's definition. ; var clusterExists = false; var clusterContextName = KubeContextName.Parse($"root@{clusterDefinition.Name}"); var clusterContext = KubeHelper.Config.GetContext(clusterContextName); var clusterLogin = KubeHelper.GetClusterLogin(clusterContextName); if (clusterContext != null && clusterLogin != null && !clusterLogin.SetupDetails.SetupPending) { clusterExists = ClusterDefinition.AreSimilar(clusterDefinition, clusterLogin.ClusterDefinition); } if (clusterExists && !options.RemoveClusterOnStart) { // It looks like the test cluster may already exist. We'll verify // that it's running, healthy, unlocked and the cluster versions match. // When all of these conditions are true, we'll use the existing cluster, // otherwise we'll remove the cluster as well as its context/login, // and deploy a new cluster below. using (var cluster = new ClusterProxy(clusterLogin.ClusterDefinition, new HostingManagerFactory())) { KubeHelper.SetCurrentContext(clusterContextName); var isLocked = cluster.IsLockedAsync().ResultWithoutAggregate(); var clusterInfo = cluster.GetClusterInfoAsync().ResultWithoutAggregate(); var clusterHealth = cluster.GetClusterHealthAsync().ResultWithoutAggregate(); if (isLocked.HasValue && isLocked.Value) { throw new NeonKubeException($"Cluster is locked: {cluster.Name}"); } if (clusterHealth.State == ClusterState.Healthy && clusterInfo.ClusterVersion == KubeVersions.NeonKube) { // We need to reset an existing cluster to ensure it's in a known state. cluster.ResetAsync().WaitWithoutAggregate(); started = true; IsRunning = true; Cluster = new ClusterProxy(KubeHelper.CurrentContext, new HostingManagerFactory()); return(TestFixtureStatus.Started); } cluster.RemoveAsync(removeOrphans: true).WaitWithoutAggregate(); } } else { // There is no known existing cluster but there still might be a cluster // deployed by another machine or fragments of a partially deployed cluster, // so we need to do a preemptive cluster remove. using (var cluster = new ClusterProxy(clusterDefinition, new HostingManagerFactory())) { cluster.RemoveAsync(removeOrphans: true).WaitWithoutAggregate(); } } // Provision the new cluster. WriteTestOutputLine($"PREPARE CLUSTER: {clusterDefinition.Name}"); try { var controller = KubeSetup.CreateClusterPrepareController( clusterDefinition: clusterDefinition, nodeImageUri: imageUri, nodeImagePath: imagePath, maxParallel: options.MaxParallel, unredacted: options.Unredacted, neonCloudHeadendUri: options.NeonCloudHeadendUri); switch (controller.RunAsync().ResultWithoutAggregate()) { case SetupDisposition.Succeeded: WriteTestOutputLine("CLUSTER PREPARE: SUCCESS"); break; case SetupDisposition.Failed: WriteTestOutputLine("CLUSTER PREPARE: FAIL"); throw new NeonKubeException("Cluster prepare failed."); case SetupDisposition.Cancelled: default: throw new NotImplementedException(); } } finally { if (options.CaptureDeploymentLogs) { CaptureDeploymentLogs(); } } // Setup the cluster. WriteTestOutputLine($"SETUP CLUSTER: {clusterDefinition.Name}"); try { var controller = KubeSetup.CreateClusterSetupController( clusterDefinition: clusterDefinition, maxParallel: options.MaxParallel, unredacted: options.Unredacted); switch (controller.RunAsync().ResultWithoutAggregate()) { case SetupDisposition.Succeeded: WriteTestOutputLine("CLUSTER SETUP: SUCCESS"); break; case SetupDisposition.Failed: WriteTestOutputLine("CLUSTER SETUP: FAILED"); throw new NeonKubeException("Cluster setup failed."); case SetupDisposition.Cancelled: default: throw new NotImplementedException(); } } finally { if (options.CaptureDeploymentLogs) { CaptureDeploymentLogs(); } } // NOTE: We just deployed brand new cluster so there's no need to reset it. started = true; IsRunning = true; Cluster = new ClusterProxy(KubeHelper.CurrentContext, new HostingManagerFactory()); return(TestFixtureStatus.Started); }
/// <inheritdoc/> public override async Task RunAsync(CommandLine commandLine) { if (commandLine.HasHelpOption) { Help(); Program.Exit(0); } Console.WriteLine(); var context = KubeHelper.CurrentContext; if (context == null) { Console.Error.WriteLine($"*** ERROR: There is no current cluster."); Program.Exit(1); } var force = commandLine.HasOption("--force"); // Determine the individual components to be reset. We'll default to // resetting all of them when none of these options are specified. var crio = commandLine.HasOption("--crio"); var auth = commandLine.HasOption("--auth"); var harbor = commandLine.HasOption("--harbor"); var minio = commandLine.HasOption("--minio"); var monitoring = commandLine.HasOption("--monitoring"); if (!crio && !auth && !harbor && !minio && monitoring) { crio = true; auth = true; harbor = true; minio = true; monitoring = true; } // Obtain the namespaces to be retained. var keep = commandLine.GetOption("--keep-namespaces"); var keepNamespaces = new List <string>(); if (!string.IsNullOrEmpty(keep)) { foreach (var @namespace in keep.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) { keepNamespaces.Add(@namespace); } } using (var cluster = new ClusterProxy(context, new HostingManagerFactory())) { var status = await cluster.GetClusterHealthAsync(); switch (status.State) { case ClusterState.Healthy: case ClusterState.Unhealthy: if (!force) { var isLocked = await cluster.IsLockedAsync(); if (!isLocked.HasValue) { Console.Error.WriteLine($"*** ERROR: [{cluster.Name}] lock status is unknown."); Program.Exit(1); } if (isLocked.Value) { Console.Error.WriteLine($"*** ERROR: [{cluster.Name}] is locked."); Program.Exit(1); } if (!Program.PromptYesNo($"Are you sure you want to reset: {cluster.Name}?")) { Program.Exit(0); } } Console.WriteLine("Resetting cluster (this may take a while)..."); await cluster.ResetAsync( new ClusterResetOptions() { KeepNamespaces = keepNamespaces, ResetCrio = crio, ResetAuth = auth, ResetHarbor = harbor, ResetMinio = minio, ResetMonitoring = monitoring }, progressMessage => Console.WriteLine(progressMessage)); Console.WriteLine($"RESET: {cluster.Name}"); break; default: Console.Error.WriteLine($"*** ERROR: Cluster is not running."); Program.Exit(1); break; } } }
/// <inheritdoc/> public override async Task RunAsync(CommandLine commandLine) { if (commandLine.HasHelpOption) { Help(); Program.Exit(0); } Console.WriteLine(); var context = KubeHelper.CurrentContext; if (context == null) { Console.Error.WriteLine($"*** ERROR: There is no current cluster."); Program.Exit(1); } var force = commandLine.HasOption("--force"); using (var cluster = new ClusterProxy(context, new HostingManagerFactory())) { var capabilities = cluster.Capabilities; if ((capabilities & HostingCapabilities.Pausable) == 0) { Console.Error.WriteLine($"*** ERROR: Cluster does not support pause/resume."); Program.Exit(1); } var status = await cluster.GetClusterHealthAsync(); switch (status.State) { case ClusterState.Healthy: case ClusterState.Unhealthy: if (!force) { var isLocked = await cluster.IsLockedAsync(); if (!isLocked.HasValue) { Console.Error.WriteLine($"*** ERROR: [{cluster.Name}] lock status is unknown."); Program.Exit(1); } if (isLocked.Value) { Console.Error.WriteLine($"*** ERROR: [{cluster.Name}] is locked."); Program.Exit(1); } if (!Program.PromptYesNo($"Are you sure you want to pause: {cluster.Name}?")) { Program.Exit(0); } } Console.WriteLine($"Pausing: {cluster.Name}"); try { await cluster.StopAsync(StopMode.Pause); Console.WriteLine($"PAUSED: {cluster.Name}"); } catch (TimeoutException) { Console.WriteLine(); Console.WriteLine($"*** ERROR: Timeout waiting for cluster."); } break; default: Console.Error.WriteLine($"*** ERROR: Cluster is not running."); Program.Exit(1); break; } } }
/// <inheritdoc/> public override async Task RunAsync(CommandLine commandLine) { if (commandLine.HasHelpOption) { Help(); Program.Exit(0); } Console.WriteLine(); var context = KubeHelper.CurrentContext; if (context == null) { Console.Error.WriteLine($"*** ERROR: There is no current cluster."); Program.Exit(1); } var force = commandLine.HasOption("--force"); using (var cluster = new ClusterProxy(context, new HostingManagerFactory())) { var capabilities = cluster.Capabilities; if ((capabilities & HostingCapabilities.Removable) == 0) { Console.Error.WriteLine($"*** ERROR: Cluster is not removable."); Program.Exit(1); } if (!force) { var isLocked = await cluster.IsLockedAsync(); if (!isLocked.HasValue) { Console.Error.WriteLine($"*** ERROR: [{cluster.Name}] lock status is unknown."); Program.Exit(1); } if (isLocked.Value) { Console.Error.WriteLine($"*** ERROR: [{cluster.Name}] is locked."); Program.Exit(1); } if (!Program.PromptYesNo($"Are you sure you want to remove: {cluster.Name}?")) { Program.Exit(0); } } try { Console.WriteLine($"Removing: {cluster.Name}..."); await cluster.RemoveAsync(); KubeHelper.Config.RemoveContext(context); Console.WriteLine($"REMOVED: {cluster.Name}"); } catch (TimeoutException) { Console.WriteLine(); Console.WriteLine($"*** ERROR: Timeout waiting for cluster."); } } }
/// <inheritdoc/> public override async Task RunAsync(CommandLine commandLine) { if (commandLine.HasHelpOption) { Help(); Program.Exit(0); } Console.WriteLine(); var context = KubeHelper.CurrentContext; if (context == null) { Console.Error.WriteLine($"*** ERROR: There is no current cluster."); Program.Exit(1); } var turnoff = commandLine.HasOption("--turnoff"); var force = commandLine.HasOption("--force"); using (var cluster = new ClusterProxy(context, new HostingManagerFactory())) { var capabilities = cluster.Capabilities; if ((capabilities & HostingCapabilities.Stoppable) == 0) { Console.Error.WriteLine($"*** ERROR: Cluster does not support start/stop."); Program.Exit(1); } var status = await cluster.GetClusterHealthAsync(); switch (status.State) { case ClusterState.Healthy: case ClusterState.Unhealthy: if (!force) { var isLocked = await cluster.IsLockedAsync(); if (!isLocked.HasValue) { Console.Error.WriteLine($"*** ERROR: [{cluster.Name}] lock status is unknown."); Program.Exit(1); } if (isLocked.Value) { Console.Error.WriteLine($"*** ERROR: [{cluster.Name}] is locked."); Program.Exit(1); } if (!Program.PromptYesNo($"Are you sure you want to stop: {cluster.Name}?")) { Program.Exit(0); } } try { if (turnoff) { Console.WriteLine($"Turning Off: {cluster.Name}"); } else { Console.WriteLine($"Stopping: {cluster.Name}..."); } await cluster.StopAsync(turnoff?StopMode.TurnOff : StopMode.Graceful); Console.WriteLine($"STOPPED: {cluster.Name}"); } catch (TimeoutException) { Console.WriteLine(); Console.WriteLine($"*** ERROR: Timeout waiting for cluster."); } break; default: Console.Error.WriteLine($"*** ERROR: Cluster is already stopped."); Program.Exit(1); break; } } }
/// <inheritdoc/> public override void Run(CommandLine commandLine) { if (commandLine.HasHelpOption) { Help(); Program.Exit(0); } // Special-case handling of the [--remove-templates] option. if (commandLine.HasOption("--remove-templates")) { Console.WriteLine("Removing cached virtual machine templates."); foreach (var fileName in Directory.GetFiles(KubeHelper.VmTemplatesFolder, "*.*", SearchOption.TopDirectoryOnly)) { File.Delete(fileName); } Program.Exit(0); } // Implement the command. if (KubeHelper.CurrentContext != null) { Console.Error.WriteLine("*** ERROR: You are logged into a cluster. You need to logout before preparing another."); Program.Exit(1); } if (commandLine.Arguments.Length == 0) { Console.Error.WriteLine($"*** ERROR: CLUSTER-DEF expected."); Program.Exit(1); } clusterDefPath = commandLine.Arguments[0]; force = commandLine.GetFlag("--force"); ClusterDefinition.ValidateFile(clusterDefPath, strict: true); var clusterDefinition = ClusterDefinition.FromFile(clusterDefPath, strict: true); clusterDefinition.Provisioner = $"neon-cli:{Program.Version}"; // Identify this tool/version as the cluster provisioner // NOTE: // // Azure has a more restrictive password policy and our default // machine password does not meet the requirements: // // The supplied password must be between 6-72 characters long and must // satisfy at least 3 of password complexity requirements from the following: // // 1. Contains an uppercase character // 2. Contains a lowercase character // 3. Contains a numeric digit // 4. Contains a special character // 5. Control characters are not allowed // // It's also probably not a great idea to use a static password when // provisioning VMs in public clouds because it might be possible for // somebody to use this fact the SSH into nodes while the cluster is // being setup and before we set the secure password at the end. // // This is less problematic for non-cloud environments because it's // likely that the hosts won't initially be able to receive inbound // Internet traffic and besides, we need to have a known password // embedded into the VM templates. // // We're going to handle this for cloud environments by looking // at [Program.MachinePassword]. If this is set to the default // machine password then we're going to replace it with a randomlly // generated password with a few extra characters to ensure that // it meets the target cloud's password requirements. We'll use // a non-default password if the operator specified one. if (clusterDefinition.Hosting.IsCloudProvider && Program.MachinePassword == KubeConst.DefaulVmTemplatePassword) { Program.MachinePassword = NeonHelper.GetCryptoRandomPassword(20); // Append a string that guarantees that the generated password meets // cloud minimum requirements. Program.MachinePassword += ".Aa0"; } // NOTE: Cluster prepare starts new log files. cluster = new ClusterProxy(clusterDefinition, Program.CreateNodeProxy <NodeDefinition>, appendToLog: false, defaultRunOptions: RunOptions.LogOutput | RunOptions.FaultOnError); if (KubeHelper.Config.GetContext(cluster.Definition.Name) != null) { Console.Error.WriteLine($"*** ERROR: A context named [{cluster.Definition.Name}] already exists."); Program.Exit(1); } // Configure global options. if (commandLine.HasOption("--unredacted")) { cluster.SecureRunOptions = RunOptions.None; } var failed = false; try { KubeHelper.Desktop.StartOperationAsync($"Preparing [{cluster.Name}]").Wait(); //----------------------------------------------------------------- // Try to ensure that no servers are already deployed on the IP addresses defined // for cluster nodes because provisoning over an existing cluster will likely // corrupt the existing cluster and also probably prevent the new cluster from // provisioning correctly. // // Note that we're not going to perform this check for the [Machine] hosting // environment because we're expecting the bare machines to be already running // with the assigned addresses and we're also not going to do this for cloud // environments because we're assuming that the cluster will run in its own // private network so there'll ne no possibility of conflicts. if (cluster.Definition.Hosting.Environment != HostingEnvironments.Machine && !cluster.Definition.Hosting.IsCloudProvider) { Console.WriteLine(); Console.WriteLine(" Scanning for IP address conflicts..."); Console.WriteLine(); var pingOptions = new PingOptions(ttl: 32, dontFragment: true); var pingTimeout = TimeSpan.FromSeconds(2); var pingConflicts = new List <NodeDefinition>(); var pingAttempts = 2; // I'm going to use up to 20 threads at a time here for simplicity // rather then doing this as async operations. var parallelOptions = new ParallelOptions() { MaxDegreeOfParallelism = 20 }; Parallel.ForEach(cluster.Definition.NodeDefinitions.Values, parallelOptions, node => { using (var pinger = new Pinger()) { // We're going to try pinging up to [pingAttempts] times for each node // just in case the network it sketchy and we're losing reply packets. for (int i = 0; i < pingAttempts; i++) { var reply = pinger.SendPingAsync(node.PrivateAddress, (int)pingTimeout.TotalMilliseconds).Result; if (reply.Status == IPStatus.Success) { lock (pingConflicts) { pingConflicts.Add(node); } break; } } } }); if (pingConflicts.Count > 0) { Console.Error.WriteLine($"*** ERROR: Cannot provision the cluster because [{pingConflicts.Count}] other"); Console.Error.WriteLine($"*** machines conflict with the following cluster nodes:"); Console.Error.WriteLine(); foreach (var node in pingConflicts.OrderBy(n => NetHelper.AddressToUint(IPAddress.Parse(n.PrivateAddress)))) { Console.Error.WriteLine($"{node.PrivateAddress, 16}: {node.Name}"); } Program.Exit(1); } } //----------------------------------------------------------------- // Perform basic environment provisioning. This creates basic cluster components // such as virtual machines, networks, load balancers, public IP addresses, security // groups,... as required for the environment. hostingManager = new HostingManagerFactory(() => HostingLoader.Initialize()).GetMaster(cluster, Program.LogPath); if (hostingManager == null) { Console.Error.WriteLine($"*** ERROR: No hosting manager for the [{cluster.Definition.Hosting.Environment}] hosting environment could be located."); Program.Exit(1); } hostingManager.HostUsername = Program.MachineUsername; hostingManager.HostPassword = Program.MachinePassword; hostingManager.ShowStatus = !Program.Quiet; hostingManager.MaxParallel = Program.MaxParallel; hostingManager.WaitSeconds = Program.WaitSeconds; if (hostingManager.RequiresAdminPrivileges) { Program.VerifyAdminPrivileges($"Provisioning to [{cluster.Definition.Hosting.Environment}] requires elevated administrator privileges."); } if (!hostingManager.Provision(force)) { Program.Exit(1); } // Get the mounted drive prefix from the hosting manager. cluster.Definition.DrivePrefix = hostingManager.DrivePrefix; // Ensure that the nodes have valid IP addresses. cluster.Definition.ValidatePrivateNodeAddresses(); var ipAddressToServer = new Dictionary <IPAddress, SshProxy <NodeDefinition> >(); foreach (var node in cluster.Nodes.OrderBy(n => n.Name)) { SshProxy <NodeDefinition> duplicateServer; if (node.PrivateAddress == IPAddress.Any) { throw new ArgumentException($"Node [{node.Name}] has not been assigned an IP address."); } if (ipAddressToServer.TryGetValue(node.PrivateAddress, out duplicateServer)) { throw new ArgumentException($"Nodes [{duplicateServer.Name}] and [{node.Name}] have the same IP address [{node.Metadata.PrivateAddress}]."); } ipAddressToServer.Add(node.PrivateAddress, node); } // We're going to use the masters as package caches unless the user // specifies something else. packageCaches = commandLine.GetOption("--package-cache"); // This overrides the cluster definition, if specified. if (!string.IsNullOrEmpty(packageCaches)) { cluster.Definition.PackageProxy = packageCaches; } if (string.IsNullOrEmpty(cluster.Definition.PackageProxy)) { var sbProxies = new StringBuilder(); foreach (var master in cluster.Masters) { sbProxies.AppendWithSeparator($"{master.PrivateAddress}:{NetworkPorts.AppCacherNg}"); } cluster.Definition.PackageProxy = sbProxies.ToString(); } //----------------------------------------------------------------- // Prepare the cluster. // Write the operation begin marker to all cluster node logs. cluster.LogLine(logBeginMarker); var nodesText = cluster.Nodes.Count() == 1 ? "node" : "nodes"; var operation = $"Preparing [{cluster.Definition.Name}] {nodesText}"; var controller = new SetupController <NodeDefinition>(operation, cluster.Nodes) { ShowStatus = !Program.Quiet, MaxParallel = Program.MaxParallel }; controller.AddGlobalStep("setup details", () => { using (var client = new HeadendClient()) { kubeSetupInfo = client.GetSetupInfoAsync(cluster.Definition).Result; } }); // Prepare the nodes. controller.AddWaitUntilOnlineStep(timeout: TimeSpan.FromMinutes(15)); hostingManager.AddPostProvisionSteps(controller); controller.AddStep("verify OS", CommonSteps.VerifyOS); controller.AddStep("prepare", (node, stepDelay) => { Thread.Sleep(stepDelay); CommonSteps.PrepareNode(node, cluster.Definition, kubeSetupInfo, shutdown: false); }, stepStaggerSeconds: cluster.Definition.Setup.StepStaggerSeconds); if (!controller.Run()) { // Write the operation end/failed marker to all cluster node logs. cluster.LogLine(logFailedMarker); Console.Error.WriteLine("*** ERROR: One or more configuration steps failed."); Program.Exit(1); } // Persist the cluster context extension. var contextExtensionsPath = KubeHelper.GetContextExtensionPath((KubeContextName)$"{KubeConst.RootUser}@{clusterDefinition.Name}"); var contextExtension = new KubeContextExtension(contextExtensionsPath) { ClusterDefinition = clusterDefinition, SshUsername = Program.MachineUsername, SshPassword = Program.MachinePassword, SetupDetails = new KubeSetupDetails() { SetupPending = true } }; contextExtension.Save(); // Write the operation end marker to all cluster node logs. cluster.LogLine(logEndMarker); } catch { failed = true; throw; } finally { if (!failed) { KubeHelper.Desktop.EndOperationAsync($"Cluster [{cluster.Name}] has been prepared and is ready for setup.").Wait(); } else { KubeHelper.Desktop.EndOperationAsync($"Cluster [{cluster.Name}] prepare has failed.", failed: true).Wait(); } } }
/// <inheritdoc/> public override async Task RunAsync(CommandLine commandLine) { if (commandLine.HasHelpOption) { Help(); Program.Exit(0); } Console.WriteLine(); // Cluster prepare/setup uses the [ProfileClient] to retrieve secrets and profile values. // We need to inject an implementation for [PreprocessReader] so it will be able to // perform the lookups. NeonHelper.ServiceContainer.AddSingleton <IProfileClient>(new ProfileClient()); // Handle the [--remove-templates] option. if (commandLine.HasOption("--remove-templates")) { Console.WriteLine("Removing cached virtual machine templates."); foreach (var fileName in Directory.GetFiles(KubeHelper.NodeImageFolder, "*.*", SearchOption.TopDirectoryOnly)) { File.Delete(fileName); } } var nodeImageUri = commandLine.GetOption("--node-image-uri"); var nodeImagePath = commandLine.GetOption("--node-image-path"); var debug = commandLine.HasOption("--debug"); var baseImageName = commandLine.GetOption("--base-image-name"); var clusterspace = commandLine.GetOption("--clusterspace"); var headendUri = commandLine.GetOption("--headend-uri") ?? KubeConst.NeonCloudHeadendUri; var maxParallelOption = commandLine.GetOption("--max-parallel", "6"); var disablePending = commandLine.HasOption("--disable-pending"); if (!int.TryParse(maxParallelOption, out var maxParallel) || maxParallel <= 0) { Console.Error.WriteLine($"*** ERROR: [--max-parallel={maxParallelOption}] is not valid."); Program.Exit(1); } if (debug && string.IsNullOrEmpty(baseImageName)) { Console.Error.WriteLine($"*** ERROR: [--base-image-name] is required for [--debug] mode."); Program.Exit(1); } // Implement the command. if (KubeHelper.CurrentContext != null) { Console.Error.WriteLine("*** ERROR: You are logged into a cluster. You need to logout before preparing another."); Program.Exit(1); } if (commandLine.Arguments.Length == 0) { Console.Error.WriteLine($"*** ERROR: CLUSTER-DEF expected."); Program.Exit(1); } // Obtain the cluster definition. var clusterDefPath = commandLine.Arguments[0]; var clusterDefinition = (ClusterDefinition)null; ClusterDefinition.ValidateFile(clusterDefPath, strict: true); clusterDefinition = ClusterDefinition.FromFile(clusterDefPath, strict: true); // Do a quick sanity check to ensure that the hosting environment has enough // resources (memory and disk) to actually host the cluster. using (var cluster = new ClusterProxy(clusterDefinition, new HostingManagerFactory())) { var status = await cluster.GetResourceAvailabilityAsync(); if (!status.CanBeDeployed) { Console.Error.WriteLine(); Console.Error.WriteLine($"*** ERROR: Insufficent resources available to deploy cluster."); Console.Error.WriteLine(); foreach (var entity in status.Constraints.Keys .OrderBy(key => key, StringComparer.InvariantCultureIgnoreCase)) { Console.Error.WriteLine(); Console.Error.WriteLine($"{entity}:"); foreach (var constraint in status.Constraints[entity]) { Console.Error.WriteLine($" {constraint.ResourceType.ToString().ToUpperInvariant()}: {constraint.Details}"); } } Console.Error.WriteLine(); Program.Exit(1); } } if (KubeHelper.IsOnPremiseHypervisorEnvironment(clusterDefinition.Hosting.Environment)) { // Use the default node image for the hosting environment unless [--node-image-uri] // or [--node-image-path] was specified. if (string.IsNullOrEmpty(nodeImageUri) && string.IsNullOrEmpty(nodeImagePath)) { nodeImageUri = KubeDownloads.GetDefaultNodeImageUri(clusterDefinition.Hosting.Environment); } } // Parse any specified package cache endpoints. var packageCaches = commandLine.GetOption("--package-caches", null); var packageCacheEndpoints = new List <IPEndPoint>(); if (!string.IsNullOrEmpty(packageCaches)) { foreach (var item in packageCaches.Split(' ', StringSplitOptions.RemoveEmptyEntries)) { if (!NetHelper.TryParseIPv4Endpoint(item, out var endpoint)) { Console.Error.WriteLine($"*** ERROR: [{item}] is not a valid package cache IPv4 endpoint."); Program.Exit(1); } packageCacheEndpoints.Add(endpoint); } } // Create and run the cluster prepare controller. var controller = KubeSetup.CreateClusterPrepareController( clusterDefinition, nodeImageUri: nodeImageUri, nodeImagePath: nodeImagePath, maxParallel: maxParallel, packageCacheEndpoints: packageCacheEndpoints, unredacted: commandLine.HasOption("--unredacted"), debugMode: debug, baseImageName: baseImageName, clusterspace: clusterspace, neonCloudHeadendUri: headendUri); controller.DisablePendingTasks = disablePending; controller.StatusChangedEvent += status => { status.WriteToConsole(); }; switch (await controller.RunAsync()) { case SetupDisposition.Succeeded: var pendingGroups = controller.GetPendingGroups(); if (pendingGroups.Count > 0) { Console.WriteLine($"*** ERROR: [{pendingGroups.Count}] pending task groups have not been awaited:"); Console.WriteLine(); foreach (var groupName in pendingGroups) { Console.WriteLine($" {groupName}"); } Program.Exit(1); } Console.WriteLine(); Console.WriteLine($" [{clusterDefinition.Name}] cluster is prepared."); Console.WriteLine(); Program.Exit(0); break; case SetupDisposition.Cancelled: Console.WriteLine(); Console.WriteLine(" *** CANCELLED: Cluster prepare was cancelled."); Console.WriteLine(); Program.Exit(1); break; case SetupDisposition.Failed: Console.WriteLine(); Console.WriteLine(" *** ERROR: Cluster prepare has failed. Examine the logs here:"); Console.WriteLine(); Console.WriteLine($" {KubeHelper.LogFolder}"); Console.WriteLine(); Program.Exit(1); break; default: throw new NotImplementedException(); } await Task.CompletedTask; }