/// <inheritdoc/>
public override async Task RunAsync(CommandLine commandLine)
{
if (commandLine.HasHelpOption)
{
Help();
Program.Exit(0);
}
Console.WriteLine();
var context = KubeHelper.CurrentContext;
if (context == null)
{
Console.Error.WriteLine($"*** ERROR: There is no current cluster.");
Program.Exit(1);
}
using (var cluster = new ClusterProxy(context, new HostingManagerFactory()))
{
var clusterInfo = await cluster.GetClusterInfoAsync();
Console.WriteLine();
Console.WriteLine(NeonHelper.JsonSerialize(clusterInfo, Formatting.Indented));
Console.WriteLine();
}
}
public Test_Hive(HiveFixture fixture)
{
if (!fixture.LoginAndInitialize(action:
() =>
{
// This adds a [HostsFixture] to the [HiveFixture] (which inherits
// from [TestFixtureSet]). We'll name the HostFixture so we can use
// it to setup local DNS entries for the tests.
fixture.AddFixture("hosts", hosts = new HostsFixture());
}))
{
// This call ensures that the hive is reset to a
// pristine state before each test is invoked.
fixture.Reset();
// Retrieve the hosts fixture and reset it.
hosts = (HostsFixture)fixture["hosts"];
hosts.Reset();
}
this.hiveFixture = fixture;
this.hive = fixture.Hive;
}
/// <inheritdoc/>
public override async Task RunAsync(CommandLine commandLine)
{
if (commandLine.HasHelpOption)
{
Help();
Program.Exit(0);
}
Console.WriteLine();
var context = KubeHelper.CurrentContext;
if (context == null)
{
Console.Error.WriteLine($"*** ERROR: There is no current cluster.");
Program.Exit(1);
}
using (var cluster = new ClusterProxy(context, new HostingManagerFactory()))
{
var status = await cluster.GetClusterHealthAsync();
var capabilities = cluster.Capabilities;
switch (status.State)
{
case ClusterState.Off:
case ClusterState.Paused:
if ((capabilities & HostingCapabilities.Stoppable) == 0)
{
Console.Error.WriteLine($"*** ERROR: Cluster does not support start/stop.");
Program.Exit(1);
}
Console.WriteLine($"Starting: {cluster.Name}...");
try
{
await cluster.StartAsync();
Console.WriteLine($"STARTED: {cluster.Name}");
}
catch (TimeoutException)
{
Console.WriteLine();
Console.WriteLine($"*** ERROR: Timeout waiting for cluster.");
}
break;
default:
Console.Error.WriteLine($"*** ERROR: Cluster is already running.");
Program.Exit(1);
break;
}
}
}
/// <inheritdoc/>
public override async Task RunAsync(CommandLine commandLine)
{
if (commandLine.HasHelpOption)
{
Help();
Program.Exit(0);
}
Console.WriteLine();
var context = KubeHelper.CurrentContext;
if (context == null)
{
Console.Error.WriteLine($"*** ERROR: There is no current cluster.");
Program.Exit(1);
}
using (var cluster = new ClusterProxy(context, new HostingManagerFactory()))
{
var status = await cluster.GetClusterHealthAsync();
switch (status.State)
{
case ClusterState.Healthy:
case ClusterState.Unhealthy:
var isLocked = await cluster.IsLockedAsync();
if (!isLocked.HasValue)
{
Console.Error.WriteLine($"*** ERROR: [{cluster.Name}] lock status is unknown.");
Program.Exit(1);
}
if (isLocked.Value)
{
Console.WriteLine($"[{cluster.Name}]: LOCKED");
Program.Exit(0);
}
else
{
Console.WriteLine($"[{cluster.Name}]: UNLOCKED");
Program.Exit(2);
}
break;
default:
Console.Error.WriteLine($"*** ERROR: Cluster is not running.");
Program.Exit(1);
break;
}
}
}
/// <summary>
/// Initializes the test fixture to run tests against the current cluster. This is useful
/// when developing unit tests against a developer managed cluster.
/// </summary>
/// <param name="options">
/// Optionally specifies the options that <see cref="ClusterFixture"/> will use to
/// manage the test cluster.
/// </param>
/// <returns>This always returns <see cref="TestFixtureStatus.AlreadyRunning"/>.</returns>
/// <exception cref="NeonKubeException">Thrown when there isn't a current cluster or when it's locked.</exception>
public TestFixtureStatus StartWithCurrentCluster(ClusterFixtureOptions options = null)
{
options ??= new ClusterFixtureOptions();
// Make a copy of the options and then disable any settings that don't apply to
// running tests against the current cluster.
options = options.Clone();
options.RemoveClusterOnStart = false;
options.RemoveClusterOnDispose = false;
this.options = options;
// Verify that:
//
// * There is a current cluster
// * That it's running
// * That it's not locked
Cluster = new ClusterProxy(KubeHelper.CurrentContext, new HostingManagerFactory());
try
{
var isLocked = Cluster.IsLockedAsync().Result;
if (!isLocked.HasValue)
{
throw new NeonKubeException("Unable to determine the cluster lock status.");
}
if (isLocked.Value)
{
throw new NeonKubeException("Cluster is locked. Use this command to unlock it: neon cluster unlock");
}
}
catch (NeonKubeException)
{
throw;
}
catch (Exception e)
{
throw new NeonKubeException("Unable to connect cluster. Is it running?", e);
}
started = true;
IsRunning = true;
return(TestFixtureStatus.AlreadyRunning);
}
/// <inheritdoc/>
public override async Task RunAsync(CommandLine commandLine)
{
if (commandLine.HasHelpOption)
{
Help();
Program.Exit(0);
}
Console.WriteLine();
var context = KubeHelper.CurrentContext;
if (context == null)
{
Console.Error.WriteLine($"*** ERROR: There is no current cluster.");
Program.Exit(1);
}
using (var cluster = new ClusterProxy(context, new HostingManagerFactory()))
{
var status = await cluster.GetClusterHealthAsync();
var capabilities = cluster.Capabilities;
switch (status.State)
{
case ClusterState.Healthy:
case ClusterState.Unhealthy:
Console.WriteLine($"Unlocking: {cluster.Name}...");
await cluster.UnlockAsync();
Console.WriteLine($"UNLOCKED: {cluster.Name}");
break;
default:
Console.Error.WriteLine($"*** ERROR: Cluster is not running.");
Program.Exit(1);
break;
}
}
}
/// <summary>
/// <para>
/// Deploys a new test cluster as specified by the cluster definition passed or connects
/// to a cluster previously deployed by this method when the cluster definition of the
/// existing cluster and the definition passed here are the same.
/// </para>
/// </summary>
/// <param name="clusterDefinition">The cluster definition model.</param>
/// <param name="options">
/// Optionally specifies the options that <see cref="ClusterFixture"/> will use to
/// manage the test cluster.
/// </param>
/// <returns>
/// <para>
/// The <see cref="TestFixtureStatus"/>:
/// </para>
/// <list type="table">
/// <item>
/// <term><see cref="TestFixtureStatus.Disabled"/></term>
/// <description>
/// Returned when cluster unit testing is disabled due to the <c>NEON_CLUSTER_TESTING</c> environment
/// variable not being present on the current machine which means that <see cref="TestHelper.IsClusterTestingEnabled"/>
/// returns <c>false</c>.
/// </description>
/// </item>
/// <item>
/// <term><see cref="TestFixtureStatus.Started"/></term>
/// <description>
/// Returned when one of the <c>Start()</c> methods is called for the first time for the fixture
/// instance, indicating that an existing cluster has been connected or a new cluster has been deployed.
/// </description>
/// </item>
/// <item>
/// <term><see cref="TestFixtureStatus.AlreadyRunning"/></term>
/// <description>
/// Returned when one of the <c>Start()</c> methods has already been called by your test
/// class instance.
/// </description>
/// </item>
/// </list>
/// </returns>
/// <exception cref="NeonKubeException">Thrown when the test cluster could not be deployed.</exception>
/// <remarks>
/// <para>
/// <b>IMPORTANT:</b> Only one <see cref="ClusterFixture"/> can be run at a time on
/// any one computer. This is due to the fact that cluster state like the kubeconfig,
/// neonKUBE logins, logs and other files will be written to <b>~/.neonkube/spaces/$fixture/*</b>
/// so multiple fixture instances will be confused when trying to manage these same files.
/// </para>
/// <para>
/// This means that not only will running <see cref="ClusterFixture"/> based tests in parallel
/// within the same instance of Visual Studio fail, but running these tests in different
/// Visual Studio instances will also fail.
/// </para>
/// </remarks>
public TestFixtureStatus StartWithClusterDefinition(ClusterDefinition clusterDefinition, ClusterFixtureOptions options = null)
{
Covenant.Requires <ArgumentNullException>(clusterDefinition != null, nameof(clusterDefinition));
if (clusterDefinition.IsLocked)
{
throw new NeonKubeException("Test clusters need to be unlocked. Please set [isLocked: false] in your cluster definition.");
}
if (!TestHelper.IsClusterTestingEnabled)
{
return(TestFixtureStatus.Disabled);
}
if (started)
{
return(TestFixtureStatus.AlreadyRunning);
}
options ??= new ClusterFixtureOptions();
this.options = options.Clone();
if (this.Cluster != null)
{
return(TestFixtureStatus.AlreadyRunning);
}
// Set the clusterspace mode, using any previously downloaded node image unless
// the user specifies a custom image. We're going to host the fixture state
// files in this fixed folder:
//
// ~/.neonkube/spaces/$fixture/*
clusterspaceFolder = KubeHelper.SetClusterSpaceMode(string.IsNullOrEmpty(options.ImageUriOrPath) ? KubeClusterspaceMode.EnabledWithSharedCache : KubeClusterspaceMode.Enabled, KubeHelper.ClusterspacePrefix("fixture"));
// Figure out whether the user passed an image URI or file path to override
// the default node image.
var imageUriOrPath = options.ImageUriOrPath;
var imageUri = (string)null;
var imagePath = (string)null;
if (string.IsNullOrEmpty(imageUriOrPath))
{
imageUriOrPath = KubeDownloads.GetDefaultNodeImageUri(clusterDefinition.Hosting.Environment);
}
if (imageUriOrPath.StartsWith("http://", StringComparison.InvariantCultureIgnoreCase) || imageUriOrPath.StartsWith("https://", StringComparison.InvariantCultureIgnoreCase))
{
imageUri = imageUriOrPath;
}
else
{
imagePath = imageUriOrPath;
}
//-------------------------------------------------------------
// We need to deal with some scenarios here:
//
// 1. No cluster context or login exists for the target cluster.
//
// A conflicting cluster may still exist though, having been deployed
// by another computer or perhaps the kubecontext/logins on the current
// machine may have been modified. We need to be sure to remove any
// conflicting resources in this case.
//
// 2. Cluster context and login exist on the current machine for the target
// cluster but the cluster is unhealthy or locked. We'll abort for locked
// clusters and remove and redeploy for unhealth clusters.
//
// 3. Cluster context and login exist and the cluster is healthy. In this case,
// we need to compare the deployed cluster version against the current version
// and remove/redeploy when the versions don't match.
//
// 4. Cluster context and login exist and the cluster is healthy and cluster versions
// match. In this case, We'll compare the existing cluster definition with that for
// the new cluster and also compare the cluster versions and if they match and
// [RemoveClusterOnStart=false] we'll just use the existing cluster.
//
// 5. The current cluster matches the target but [RemoveClusterOnStart=true].
// We need to remove the current cluster in this case so we'll deploy a
// fresh one.
// Determine whether a test cluster with the same name exists and if
// its cluster definition matches the test cluster's definition.
;
var clusterExists = false;
var clusterContextName = KubeContextName.Parse($"root@{clusterDefinition.Name}");
var clusterContext = KubeHelper.Config.GetContext(clusterContextName);
var clusterLogin = KubeHelper.GetClusterLogin(clusterContextName);
if (clusterContext != null && clusterLogin != null && !clusterLogin.SetupDetails.SetupPending)
{
clusterExists = ClusterDefinition.AreSimilar(clusterDefinition, clusterLogin.ClusterDefinition);
}
if (clusterExists && !options.RemoveClusterOnStart)
{
// It looks like the test cluster may already exist. We'll verify
// that it's running, healthy, unlocked and the cluster versions match.
// When all of these conditions are true, we'll use the existing cluster,
// otherwise we'll remove the cluster as well as its context/login,
// and deploy a new cluster below.
using (var cluster = new ClusterProxy(clusterLogin.ClusterDefinition, new HostingManagerFactory()))
{
KubeHelper.SetCurrentContext(clusterContextName);
var isLocked = cluster.IsLockedAsync().ResultWithoutAggregate();
var clusterInfo = cluster.GetClusterInfoAsync().ResultWithoutAggregate();
var clusterHealth = cluster.GetClusterHealthAsync().ResultWithoutAggregate();
if (isLocked.HasValue && isLocked.Value)
{
throw new NeonKubeException($"Cluster is locked: {cluster.Name}");
}
if (clusterHealth.State == ClusterState.Healthy && clusterInfo.ClusterVersion == KubeVersions.NeonKube)
{
// We need to reset an existing cluster to ensure it's in a known state.
cluster.ResetAsync().WaitWithoutAggregate();
started = true;
IsRunning = true;
Cluster = new ClusterProxy(KubeHelper.CurrentContext, new HostingManagerFactory());
return(TestFixtureStatus.Started);
}
cluster.RemoveAsync(removeOrphans: true).WaitWithoutAggregate();
}
}
else
{
// There is no known existing cluster but there still might be a cluster
// deployed by another machine or fragments of a partially deployed cluster,
// so we need to do a preemptive cluster remove.
using (var cluster = new ClusterProxy(clusterDefinition, new HostingManagerFactory()))
{
cluster.RemoveAsync(removeOrphans: true).WaitWithoutAggregate();
}
}
// Provision the new cluster.
WriteTestOutputLine($"PREPARE CLUSTER: {clusterDefinition.Name}");
try
{
var controller = KubeSetup.CreateClusterPrepareController(
clusterDefinition: clusterDefinition,
nodeImageUri: imageUri,
nodeImagePath: imagePath,
maxParallel: options.MaxParallel,
unredacted: options.Unredacted,
neonCloudHeadendUri: options.NeonCloudHeadendUri);
switch (controller.RunAsync().ResultWithoutAggregate())
{
case SetupDisposition.Succeeded:
WriteTestOutputLine("CLUSTER PREPARE: SUCCESS");
break;
case SetupDisposition.Failed:
WriteTestOutputLine("CLUSTER PREPARE: FAIL");
throw new NeonKubeException("Cluster prepare failed.");
case SetupDisposition.Cancelled:
default:
throw new NotImplementedException();
}
}
finally
{
if (options.CaptureDeploymentLogs)
{
CaptureDeploymentLogs();
}
}
// Setup the cluster.
WriteTestOutputLine($"SETUP CLUSTER: {clusterDefinition.Name}");
try
{
var controller = KubeSetup.CreateClusterSetupController(
clusterDefinition: clusterDefinition,
maxParallel: options.MaxParallel,
unredacted: options.Unredacted);
switch (controller.RunAsync().ResultWithoutAggregate())
{
case SetupDisposition.Succeeded:
WriteTestOutputLine("CLUSTER SETUP: SUCCESS");
break;
case SetupDisposition.Failed:
WriteTestOutputLine("CLUSTER SETUP: FAILED");
throw new NeonKubeException("Cluster setup failed.");
case SetupDisposition.Cancelled:
default:
throw new NotImplementedException();
}
}
finally
{
if (options.CaptureDeploymentLogs)
{
CaptureDeploymentLogs();
}
}
// NOTE: We just deployed brand new cluster so there's no need to reset it.
started = true;
IsRunning = true;
Cluster = new ClusterProxy(KubeHelper.CurrentContext, new HostingManagerFactory());
return(TestFixtureStatus.Started);
}
/// <inheritdoc/>
public override async Task RunAsync(CommandLine commandLine)
{
if (commandLine.HasHelpOption)
{
Help();
Program.Exit(0);
}
Console.WriteLine();
var context = KubeHelper.CurrentContext;
if (context == null)
{
Console.Error.WriteLine($"*** ERROR: There is no current cluster.");
Program.Exit(1);
}
var force = commandLine.HasOption("--force");
// Determine the individual components to be reset. We'll default to
// resetting all of them when none of these options are specified.
var crio = commandLine.HasOption("--crio");
var auth = commandLine.HasOption("--auth");
var harbor = commandLine.HasOption("--harbor");
var minio = commandLine.HasOption("--minio");
var monitoring = commandLine.HasOption("--monitoring");
if (!crio && !auth && !harbor && !minio && monitoring)
{
crio = true;
auth = true;
harbor = true;
minio = true;
monitoring = true;
}
// Obtain the namespaces to be retained.
var keep = commandLine.GetOption("--keep-namespaces");
var keepNamespaces = new List <string>();
if (!string.IsNullOrEmpty(keep))
{
foreach (var @namespace in keep.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries))
{
keepNamespaces.Add(@namespace);
}
}
using (var cluster = new ClusterProxy(context, new HostingManagerFactory()))
{
var status = await cluster.GetClusterHealthAsync();
switch (status.State)
{
case ClusterState.Healthy:
case ClusterState.Unhealthy:
if (!force)
{
var isLocked = await cluster.IsLockedAsync();
if (!isLocked.HasValue)
{
Console.Error.WriteLine($"*** ERROR: [{cluster.Name}] lock status is unknown.");
Program.Exit(1);
}
if (isLocked.Value)
{
Console.Error.WriteLine($"*** ERROR: [{cluster.Name}] is locked.");
Program.Exit(1);
}
if (!Program.PromptYesNo($"Are you sure you want to reset: {cluster.Name}?"))
{
Program.Exit(0);
}
}
Console.WriteLine("Resetting cluster (this may take a while)...");
await cluster.ResetAsync(
new ClusterResetOptions()
{
KeepNamespaces = keepNamespaces,
ResetCrio = crio,
ResetAuth = auth,
ResetHarbor = harbor,
ResetMinio = minio,
ResetMonitoring = monitoring
},
progressMessage => Console.WriteLine(progressMessage));
Console.WriteLine($"RESET: {cluster.Name}");
break;
default:
Console.Error.WriteLine($"*** ERROR: Cluster is not running.");
Program.Exit(1);
break;
}
}
}
/// <inheritdoc/>
public override async Task RunAsync(CommandLine commandLine)
{
if (commandLine.HasHelpOption)
{
Help();
Program.Exit(0);
}
Console.WriteLine();
var context = KubeHelper.CurrentContext;
if (context == null)
{
Console.Error.WriteLine($"*** ERROR: There is no current cluster.");
Program.Exit(1);
}
var force = commandLine.HasOption("--force");
using (var cluster = new ClusterProxy(context, new HostingManagerFactory()))
{
var capabilities = cluster.Capabilities;
if ((capabilities & HostingCapabilities.Pausable) == 0)
{
Console.Error.WriteLine($"*** ERROR: Cluster does not support pause/resume.");
Program.Exit(1);
}
var status = await cluster.GetClusterHealthAsync();
switch (status.State)
{
case ClusterState.Healthy:
case ClusterState.Unhealthy:
if (!force)
{
var isLocked = await cluster.IsLockedAsync();
if (!isLocked.HasValue)
{
Console.Error.WriteLine($"*** ERROR: [{cluster.Name}] lock status is unknown.");
Program.Exit(1);
}
if (isLocked.Value)
{
Console.Error.WriteLine($"*** ERROR: [{cluster.Name}] is locked.");
Program.Exit(1);
}
if (!Program.PromptYesNo($"Are you sure you want to pause: {cluster.Name}?"))
{
Program.Exit(0);
}
}
Console.WriteLine($"Pausing: {cluster.Name}");
try
{
await cluster.StopAsync(StopMode.Pause);
Console.WriteLine($"PAUSED: {cluster.Name}");
}
catch (TimeoutException)
{
Console.WriteLine();
Console.WriteLine($"*** ERROR: Timeout waiting for cluster.");
}
break;
default:
Console.Error.WriteLine($"*** ERROR: Cluster is not running.");
Program.Exit(1);
break;
}
}
}
/// <inheritdoc/>
public override async Task RunAsync(CommandLine commandLine)
{
if (commandLine.HasHelpOption)
{
Help();
Program.Exit(0);
}
Console.WriteLine();
var context = KubeHelper.CurrentContext;
if (context == null)
{
Console.Error.WriteLine($"*** ERROR: There is no current cluster.");
Program.Exit(1);
}
var force = commandLine.HasOption("--force");
using (var cluster = new ClusterProxy(context, new HostingManagerFactory()))
{
var capabilities = cluster.Capabilities;
if ((capabilities & HostingCapabilities.Removable) == 0)
{
Console.Error.WriteLine($"*** ERROR: Cluster is not removable.");
Program.Exit(1);
}
if (!force)
{
var isLocked = await cluster.IsLockedAsync();
if (!isLocked.HasValue)
{
Console.Error.WriteLine($"*** ERROR: [{cluster.Name}] lock status is unknown.");
Program.Exit(1);
}
if (isLocked.Value)
{
Console.Error.WriteLine($"*** ERROR: [{cluster.Name}] is locked.");
Program.Exit(1);
}
if (!Program.PromptYesNo($"Are you sure you want to remove: {cluster.Name}?"))
{
Program.Exit(0);
}
}
try
{
Console.WriteLine($"Removing: {cluster.Name}...");
await cluster.RemoveAsync();
KubeHelper.Config.RemoveContext(context);
Console.WriteLine($"REMOVED: {cluster.Name}");
}
catch (TimeoutException)
{
Console.WriteLine();
Console.WriteLine($"*** ERROR: Timeout waiting for cluster.");
}
}
}
/// <inheritdoc/>
public override async Task RunAsync(CommandLine commandLine)
{
if (commandLine.HasHelpOption)
{
Help();
Program.Exit(0);
}
Console.WriteLine();
var context = KubeHelper.CurrentContext;
if (context == null)
{
Console.Error.WriteLine($"*** ERROR: There is no current cluster.");
Program.Exit(1);
}
var turnoff = commandLine.HasOption("--turnoff");
var force = commandLine.HasOption("--force");
using (var cluster = new ClusterProxy(context, new HostingManagerFactory()))
{
var capabilities = cluster.Capabilities;
if ((capabilities & HostingCapabilities.Stoppable) == 0)
{
Console.Error.WriteLine($"*** ERROR: Cluster does not support start/stop.");
Program.Exit(1);
}
var status = await cluster.GetClusterHealthAsync();
switch (status.State)
{
case ClusterState.Healthy:
case ClusterState.Unhealthy:
if (!force)
{
var isLocked = await cluster.IsLockedAsync();
if (!isLocked.HasValue)
{
Console.Error.WriteLine($"*** ERROR: [{cluster.Name}] lock status is unknown.");
Program.Exit(1);
}
if (isLocked.Value)
{
Console.Error.WriteLine($"*** ERROR: [{cluster.Name}] is locked.");
Program.Exit(1);
}
if (!Program.PromptYesNo($"Are you sure you want to stop: {cluster.Name}?"))
{
Program.Exit(0);
}
}
try
{
if (turnoff)
{
Console.WriteLine($"Turning Off: {cluster.Name}");
}
else
{
Console.WriteLine($"Stopping: {cluster.Name}...");
}
await cluster.StopAsync(turnoff?StopMode.TurnOff : StopMode.Graceful);
Console.WriteLine($"STOPPED: {cluster.Name}");
}
catch (TimeoutException)
{
Console.WriteLine();
Console.WriteLine($"*** ERROR: Timeout waiting for cluster.");
}
break;
default:
Console.Error.WriteLine($"*** ERROR: Cluster is already stopped.");
Program.Exit(1);
break;
}
}
}
/// <inheritdoc/>
public override void Run(CommandLine commandLine)
{
if (commandLine.HasHelpOption)
{
Help();
Program.Exit(0);
}
// Special-case handling of the [--remove-templates] option.
if (commandLine.HasOption("--remove-templates"))
{
Console.WriteLine("Removing cached virtual machine templates.");
foreach (var fileName in Directory.GetFiles(KubeHelper.VmTemplatesFolder, "*.*", SearchOption.TopDirectoryOnly))
{
File.Delete(fileName);
}
Program.Exit(0);
}
// Implement the command.
if (KubeHelper.CurrentContext != null)
{
Console.Error.WriteLine("*** ERROR: You are logged into a cluster. You need to logout before preparing another.");
Program.Exit(1);
}
if (commandLine.Arguments.Length == 0)
{
Console.Error.WriteLine($"*** ERROR: CLUSTER-DEF expected.");
Program.Exit(1);
}
clusterDefPath = commandLine.Arguments[0];
force = commandLine.GetFlag("--force");
ClusterDefinition.ValidateFile(clusterDefPath, strict: true);
var clusterDefinition = ClusterDefinition.FromFile(clusterDefPath, strict: true);
clusterDefinition.Provisioner = $"neon-cli:{Program.Version}"; // Identify this tool/version as the cluster provisioner
// NOTE:
//
// Azure has a more restrictive password policy and our default
// machine password does not meet the requirements:
//
// The supplied password must be between 6-72 characters long and must
// satisfy at least 3 of password complexity requirements from the following:
//
// 1. Contains an uppercase character
// 2. Contains a lowercase character
// 3. Contains a numeric digit
// 4. Contains a special character
// 5. Control characters are not allowed
//
// It's also probably not a great idea to use a static password when
// provisioning VMs in public clouds because it might be possible for
// somebody to use this fact the SSH into nodes while the cluster is
// being setup and before we set the secure password at the end.
//
// This is less problematic for non-cloud environments because it's
// likely that the hosts won't initially be able to receive inbound
// Internet traffic and besides, we need to have a known password
// embedded into the VM templates.
//
// We're going to handle this for cloud environments by looking
// at [Program.MachinePassword]. If this is set to the default
// machine password then we're going to replace it with a randomlly
// generated password with a few extra characters to ensure that
// it meets the target cloud's password requirements. We'll use
// a non-default password if the operator specified one.
if (clusterDefinition.Hosting.IsCloudProvider && Program.MachinePassword == KubeConst.DefaulVmTemplatePassword)
{
Program.MachinePassword = NeonHelper.GetCryptoRandomPassword(20);
// Append a string that guarantees that the generated password meets
// cloud minimum requirements.
Program.MachinePassword += ".Aa0";
}
// NOTE: Cluster prepare starts new log files.
cluster = new ClusterProxy(clusterDefinition, Program.CreateNodeProxy <NodeDefinition>, appendToLog: false, defaultRunOptions: RunOptions.LogOutput | RunOptions.FaultOnError);
if (KubeHelper.Config.GetContext(cluster.Definition.Name) != null)
{
Console.Error.WriteLine($"*** ERROR: A context named [{cluster.Definition.Name}] already exists.");
Program.Exit(1);
}
// Configure global options.
if (commandLine.HasOption("--unredacted"))
{
cluster.SecureRunOptions = RunOptions.None;
}
var failed = false;
try
{
KubeHelper.Desktop.StartOperationAsync($"Preparing [{cluster.Name}]").Wait();
//-----------------------------------------------------------------
// Try to ensure that no servers are already deployed on the IP addresses defined
// for cluster nodes because provisoning over an existing cluster will likely
// corrupt the existing cluster and also probably prevent the new cluster from
// provisioning correctly.
//
// Note that we're not going to perform this check for the [Machine] hosting
// environment because we're expecting the bare machines to be already running
// with the assigned addresses and we're also not going to do this for cloud
// environments because we're assuming that the cluster will run in its own
// private network so there'll ne no possibility of conflicts.
if (cluster.Definition.Hosting.Environment != HostingEnvironments.Machine &&
!cluster.Definition.Hosting.IsCloudProvider)
{
Console.WriteLine();
Console.WriteLine(" Scanning for IP address conflicts...");
Console.WriteLine();
var pingOptions = new PingOptions(ttl: 32, dontFragment: true);
var pingTimeout = TimeSpan.FromSeconds(2);
var pingConflicts = new List <NodeDefinition>();
var pingAttempts = 2;
// I'm going to use up to 20 threads at a time here for simplicity
// rather then doing this as async operations.
var parallelOptions = new ParallelOptions()
{
MaxDegreeOfParallelism = 20
};
Parallel.ForEach(cluster.Definition.NodeDefinitions.Values, parallelOptions,
node =>
{
using (var pinger = new Pinger())
{
// We're going to try pinging up to [pingAttempts] times for each node
// just in case the network it sketchy and we're losing reply packets.
for (int i = 0; i < pingAttempts; i++)
{
var reply = pinger.SendPingAsync(node.PrivateAddress, (int)pingTimeout.TotalMilliseconds).Result;
if (reply.Status == IPStatus.Success)
{
lock (pingConflicts)
{
pingConflicts.Add(node);
}
break;
}
}
}
});
if (pingConflicts.Count > 0)
{
Console.Error.WriteLine($"*** ERROR: Cannot provision the cluster because [{pingConflicts.Count}] other");
Console.Error.WriteLine($"*** machines conflict with the following cluster nodes:");
Console.Error.WriteLine();
foreach (var node in pingConflicts.OrderBy(n => NetHelper.AddressToUint(IPAddress.Parse(n.PrivateAddress))))
{
Console.Error.WriteLine($"{node.PrivateAddress, 16}: {node.Name}");
}
Program.Exit(1);
}
}
//-----------------------------------------------------------------
// Perform basic environment provisioning. This creates basic cluster components
// such as virtual machines, networks, load balancers, public IP addresses, security
// groups,... as required for the environment.
hostingManager = new HostingManagerFactory(() => HostingLoader.Initialize()).GetMaster(cluster, Program.LogPath);
if (hostingManager == null)
{
Console.Error.WriteLine($"*** ERROR: No hosting manager for the [{cluster.Definition.Hosting.Environment}] hosting environment could be located.");
Program.Exit(1);
}
hostingManager.HostUsername = Program.MachineUsername;
hostingManager.HostPassword = Program.MachinePassword;
hostingManager.ShowStatus = !Program.Quiet;
hostingManager.MaxParallel = Program.MaxParallel;
hostingManager.WaitSeconds = Program.WaitSeconds;
if (hostingManager.RequiresAdminPrivileges)
{
Program.VerifyAdminPrivileges($"Provisioning to [{cluster.Definition.Hosting.Environment}] requires elevated administrator privileges.");
}
if (!hostingManager.Provision(force))
{
Program.Exit(1);
}
// Get the mounted drive prefix from the hosting manager.
cluster.Definition.DrivePrefix = hostingManager.DrivePrefix;
// Ensure that the nodes have valid IP addresses.
cluster.Definition.ValidatePrivateNodeAddresses();
var ipAddressToServer = new Dictionary <IPAddress, SshProxy <NodeDefinition> >();
foreach (var node in cluster.Nodes.OrderBy(n => n.Name))
{
SshProxy <NodeDefinition> duplicateServer;
if (node.PrivateAddress == IPAddress.Any)
{
throw new ArgumentException($"Node [{node.Name}] has not been assigned an IP address.");
}
if (ipAddressToServer.TryGetValue(node.PrivateAddress, out duplicateServer))
{
throw new ArgumentException($"Nodes [{duplicateServer.Name}] and [{node.Name}] have the same IP address [{node.Metadata.PrivateAddress}].");
}
ipAddressToServer.Add(node.PrivateAddress, node);
}
// We're going to use the masters as package caches unless the user
// specifies something else.
packageCaches = commandLine.GetOption("--package-cache"); // This overrides the cluster definition, if specified.
if (!string.IsNullOrEmpty(packageCaches))
{
cluster.Definition.PackageProxy = packageCaches;
}
if (string.IsNullOrEmpty(cluster.Definition.PackageProxy))
{
var sbProxies = new StringBuilder();
foreach (var master in cluster.Masters)
{
sbProxies.AppendWithSeparator($"{master.PrivateAddress}:{NetworkPorts.AppCacherNg}");
}
cluster.Definition.PackageProxy = sbProxies.ToString();
}
//-----------------------------------------------------------------
// Prepare the cluster.
// Write the operation begin marker to all cluster node logs.
cluster.LogLine(logBeginMarker);
var nodesText = cluster.Nodes.Count() == 1 ? "node" : "nodes";
var operation = $"Preparing [{cluster.Definition.Name}] {nodesText}";
var controller =
new SetupController <NodeDefinition>(operation, cluster.Nodes)
{
ShowStatus = !Program.Quiet,
MaxParallel = Program.MaxParallel
};
controller.AddGlobalStep("setup details",
() =>
{
using (var client = new HeadendClient())
{
kubeSetupInfo = client.GetSetupInfoAsync(cluster.Definition).Result;
}
});
// Prepare the nodes.
controller.AddWaitUntilOnlineStep(timeout: TimeSpan.FromMinutes(15));
hostingManager.AddPostProvisionSteps(controller);
controller.AddStep("verify OS", CommonSteps.VerifyOS);
controller.AddStep("prepare",
(node, stepDelay) =>
{
Thread.Sleep(stepDelay);
CommonSteps.PrepareNode(node, cluster.Definition, kubeSetupInfo, shutdown: false);
},
stepStaggerSeconds: cluster.Definition.Setup.StepStaggerSeconds);
if (!controller.Run())
{
// Write the operation end/failed marker to all cluster node logs.
cluster.LogLine(logFailedMarker);
Console.Error.WriteLine("*** ERROR: One or more configuration steps failed.");
Program.Exit(1);
}
// Persist the cluster context extension.
var contextExtensionsPath = KubeHelper.GetContextExtensionPath((KubeContextName)$"{KubeConst.RootUser}@{clusterDefinition.Name}");
var contextExtension = new KubeContextExtension(contextExtensionsPath)
{
ClusterDefinition = clusterDefinition,
SshUsername = Program.MachineUsername,
SshPassword = Program.MachinePassword,
SetupDetails = new KubeSetupDetails()
{
SetupPending = true
}
};
contextExtension.Save();
// Write the operation end marker to all cluster node logs.
cluster.LogLine(logEndMarker);
}
catch
{
failed = true;
throw;
}
finally
{
if (!failed)
{
KubeHelper.Desktop.EndOperationAsync($"Cluster [{cluster.Name}] has been prepared and is ready for setup.").Wait();
}
else
{
KubeHelper.Desktop.EndOperationAsync($"Cluster [{cluster.Name}] prepare has failed.", failed: true).Wait();
}
}
}
/// <inheritdoc/>
public override async Task RunAsync(CommandLine commandLine)
{
if (commandLine.HasHelpOption)
{
Help();
Program.Exit(0);
}
Console.WriteLine();
// Cluster prepare/setup uses the [ProfileClient] to retrieve secrets and profile values.
// We need to inject an implementation for [PreprocessReader] so it will be able to
// perform the lookups.
NeonHelper.ServiceContainer.AddSingleton <IProfileClient>(new ProfileClient());
// Handle the [--remove-templates] option.
if (commandLine.HasOption("--remove-templates"))
{
Console.WriteLine("Removing cached virtual machine templates.");
foreach (var fileName in Directory.GetFiles(KubeHelper.NodeImageFolder, "*.*", SearchOption.TopDirectoryOnly))
{
File.Delete(fileName);
}
}
var nodeImageUri = commandLine.GetOption("--node-image-uri");
var nodeImagePath = commandLine.GetOption("--node-image-path");
var debug = commandLine.HasOption("--debug");
var baseImageName = commandLine.GetOption("--base-image-name");
var clusterspace = commandLine.GetOption("--clusterspace");
var headendUri = commandLine.GetOption("--headend-uri") ?? KubeConst.NeonCloudHeadendUri;
var maxParallelOption = commandLine.GetOption("--max-parallel", "6");
var disablePending = commandLine.HasOption("--disable-pending");
if (!int.TryParse(maxParallelOption, out var maxParallel) || maxParallel <= 0)
{
Console.Error.WriteLine($"*** ERROR: [--max-parallel={maxParallelOption}] is not valid.");
Program.Exit(1);
}
if (debug && string.IsNullOrEmpty(baseImageName))
{
Console.Error.WriteLine($"*** ERROR: [--base-image-name] is required for [--debug] mode.");
Program.Exit(1);
}
// Implement the command.
if (KubeHelper.CurrentContext != null)
{
Console.Error.WriteLine("*** ERROR: You are logged into a cluster. You need to logout before preparing another.");
Program.Exit(1);
}
if (commandLine.Arguments.Length == 0)
{
Console.Error.WriteLine($"*** ERROR: CLUSTER-DEF expected.");
Program.Exit(1);
}
// Obtain the cluster definition.
var clusterDefPath = commandLine.Arguments[0];
var clusterDefinition = (ClusterDefinition)null;
ClusterDefinition.ValidateFile(clusterDefPath, strict: true);
clusterDefinition = ClusterDefinition.FromFile(clusterDefPath, strict: true);
// Do a quick sanity check to ensure that the hosting environment has enough
// resources (memory and disk) to actually host the cluster.
using (var cluster = new ClusterProxy(clusterDefinition, new HostingManagerFactory()))
{
var status = await cluster.GetResourceAvailabilityAsync();
if (!status.CanBeDeployed)
{
Console.Error.WriteLine();
Console.Error.WriteLine($"*** ERROR: Insufficent resources available to deploy cluster.");
Console.Error.WriteLine();
foreach (var entity in status.Constraints.Keys
.OrderBy(key => key, StringComparer.InvariantCultureIgnoreCase))
{
Console.Error.WriteLine();
Console.Error.WriteLine($"{entity}:");
foreach (var constraint in status.Constraints[entity])
{
Console.Error.WriteLine($" {constraint.ResourceType.ToString().ToUpperInvariant()}: {constraint.Details}");
}
}
Console.Error.WriteLine();
Program.Exit(1);
}
}
if (KubeHelper.IsOnPremiseHypervisorEnvironment(clusterDefinition.Hosting.Environment))
{
// Use the default node image for the hosting environment unless [--node-image-uri]
// or [--node-image-path] was specified.
if (string.IsNullOrEmpty(nodeImageUri) && string.IsNullOrEmpty(nodeImagePath))
{
nodeImageUri = KubeDownloads.GetDefaultNodeImageUri(clusterDefinition.Hosting.Environment);
}
}
// Parse any specified package cache endpoints.
var packageCaches = commandLine.GetOption("--package-caches", null);
var packageCacheEndpoints = new List <IPEndPoint>();
if (!string.IsNullOrEmpty(packageCaches))
{
foreach (var item in packageCaches.Split(' ', StringSplitOptions.RemoveEmptyEntries))
{
if (!NetHelper.TryParseIPv4Endpoint(item, out var endpoint))
{
Console.Error.WriteLine($"*** ERROR: [{item}] is not a valid package cache IPv4 endpoint.");
Program.Exit(1);
}
packageCacheEndpoints.Add(endpoint);
}
}
// Create and run the cluster prepare controller.
var controller = KubeSetup.CreateClusterPrepareController(
clusterDefinition,
nodeImageUri: nodeImageUri,
nodeImagePath: nodeImagePath,
maxParallel: maxParallel,
packageCacheEndpoints: packageCacheEndpoints,
unredacted: commandLine.HasOption("--unredacted"),
debugMode: debug,
baseImageName: baseImageName,
clusterspace: clusterspace,
neonCloudHeadendUri: headendUri);
controller.DisablePendingTasks = disablePending;
controller.StatusChangedEvent +=
status =>
{
status.WriteToConsole();
};
switch (await controller.RunAsync())
{
case SetupDisposition.Succeeded:
var pendingGroups = controller.GetPendingGroups();
if (pendingGroups.Count > 0)
{
Console.WriteLine($"*** ERROR: [{pendingGroups.Count}] pending task groups have not been awaited:");
Console.WriteLine();
foreach (var groupName in pendingGroups)
{
Console.WriteLine($" {groupName}");
}
Program.Exit(1);
}
Console.WriteLine();
Console.WriteLine($" [{clusterDefinition.Name}] cluster is prepared.");
Console.WriteLine();
Program.Exit(0);
break;
case SetupDisposition.Cancelled:
Console.WriteLine();
Console.WriteLine(" *** CANCELLED: Cluster prepare was cancelled.");
Console.WriteLine();
Program.Exit(1);
break;
case SetupDisposition.Failed:
Console.WriteLine();
Console.WriteLine(" *** ERROR: Cluster prepare has failed. Examine the logs here:");
Console.WriteLine();
Console.WriteLine($" {KubeHelper.LogFolder}");
Console.WriteLine();
Program.Exit(1);
break;
default:
throw new NotImplementedException();
}
await Task.CompletedTask;
}