/// <summary>
/// Constructor.
/// </summary>
/// <exception cref="InvalidOperationException">Thrown if another test manager instance is active.</exception>
public KubeTestManager()
{
lock (syncLock)
{
if (Current != null)
{
throw new InvalidOperationException("Another test manager is active.");
}
try
{
tempFolder = new TempFolder();
Current = this;
KubeHelper.SetTestMode(tempFolder.Path);
Environment.SetEnvironmentVariable(KubeConst.TestModeFolderVar, tempFolder.Path);
}
catch
{
Environment.SetEnvironmentVariable(KubeConst.TestModeFolderVar, null);
Current = null;
throw;
}
}
}
/// <summary>
/// Exits the program returning the specified process exit code.
/// </summary>
/// <param name="exitCode">The exit code.</param>
public static void Exit(int exitCode)
{
// Ensure that all sensitive files and folders are encrypted at rest. We're
// running this after every command just to be super safe.
KubeHelper.EncryptSensitiveFiles();
throw new ProgramExitException(exitCode);
}
/// <summary>
/// Handles the <b>Logout</b> command.
/// </summary>
/// <param name="sender">The sender.</param>
/// <param name="args">The arguments.</param>
private void OnLogoutCommand(object sender, EventArgs args)
{
if (KubeHelper.CurrentContext != null)
{
ShowToast($"Logging out of: {KubeHelper.CurrentContext.Name}");
KubeHelper.SetCurrentContext((string)null);
PostUpdateUIState();
}
}
public void CreateKubernetesObject_NoConsts()
{
// Verify that [NotSupportedException] is thrown when the required constants
// are not defined by the Kubernetes object type.
Assert.Throws <NotSupportedException>(() => KubeHelper.CreateKubeObject <V1TestWithoutGroup>("test"));
Assert.Throws <NotSupportedException>(() => KubeHelper.CreateKubeObject <V1TestWithoutApiVersion>("test"));
Assert.Throws <NotSupportedException>(() => KubeHelper.CreateKubeObject <V1TestWithoutKind>("test"));
}
/// <inheritdoc/>
public void Dispose()
{
if (tempFolder != null)
{
KubeHelper.ResetTestMode();
tempFolder.Dispose();
tempFolder = null;
Current = null;
}
}
/// <inheritdoc/>
public void Dispose()
{
KubeHelper.ResetClusterspaceMode();
if (tempFolder != null)
{
tempFolder.Dispose();
tempFolder = null;
Current = null;
}
}
/// <summary>
/// Static constructor.
/// </summary>
static Pod()
{
// Initializes these properties from environment variables when we're running
// in a cluster, otherwise configure test values when running on workstation.
if (NeonHelper.IsDevWorkstation)
{
Namespace = "default";
Name = KubeHelper.GetEmulatedPodName("neon-cluster-operator");
}
else
{
Namespace = Environment.GetEnvironmentVariable("POD_NAMESPACE");
Name = Environment.GetEnvironmentVariable("POD_NAME");
}
}
/// <inheritdoc/>
public override void Run(CommandLine commandLine)
{
if (commandLine.HasHelpOption || commandLine.Arguments.Length == 0)
{
Console.WriteLine(usage);
Program.Exit(0);
}
Console.Error.WriteLine();
var currentContext = KubeHelper.CurrentContext;
var newContextName = KubeContextName.Parse(commandLine.Arguments.First());
// Ensure that the new context exists.
if (KubeHelper.Config.GetContext(newContextName) == null)
{
Console.Error.WriteLine($"*** Context [{newContextName}] not found.");
Program.Exit(1);
}
// Check whether we're already logged into the cluster.
if (KubeHelper.CurrentContext != null && newContextName == KubeContextName.Parse(KubeHelper.CurrentContext.Name))
{
Console.Error.WriteLine($"*** You are already logged into: {newContextName}");
Program.Exit(0);
}
// Logout of the current cluster.
if (currentContext != null)
{
Console.Error.WriteLine($"Logging out of [{currentContext.Name}].");
KubeHelper.SetCurrentContext((string)null);
}
// ...and log into the new context.
KubeHelper.SetCurrentContext(newContextName);
Console.WriteLine($"*** Logged into [{newContextName}].");
// Notify the desktop application.
KubeHelper.Desktop.Login().Wait();
}
/// <inheritdoc/>
public override async Task RunAsync(CommandLine commandLine)
{
Console.WriteLine();
// Actually logout.
if (KubeHelper.CurrentContext == null)
{
Console.WriteLine($"You are not logged into a neonKUBE cluster.");
return;
}
Console.WriteLine($"Logout: {KubeHelper.CurrentContext.Name}");
KubeHelper.SetCurrentContext((string)null);
await Task.CompletedTask;
}
/// <summary>
/// <para>
/// Synchronizes the UI state with the current cluster configuration.
/// </para>
/// <note>
/// This method is somewhat special in that it must be executed on the UI
/// thread and if it's called on another thread, then the method will post
/// a message to itself to invoke itself shortly on the UI thread.
/// </note>
/// </summary>
public async Task UpdateUIStateAsync()
{
if (InvokeRequired)
{
PostUpdateUIState();
return;
}
KubeHelper.LoadConfig();
await UpdateProxiesAsync();
if (InErrorState)
{
return;
}
if (!operationInProgress)
{
notifyIcon.Icon = IsConnected ? connectedIcon : disconnectedIcon;
if (notifyStack.Count > 0 && !string.IsNullOrEmpty(notifyStack.Peek().BalloonText))
{
SetBalloonText(notifyStack.Peek().BalloonText);
}
else if (IsConnected)
{
SetBalloonText($"{Text}: {KubeHelper.CurrentContextName}");
}
else
{
SetBalloonText($"{Text}: disconnected");
}
}
else if (remoteOperation != null && NeonHelper.GetProcessById(remoteOperation.ProcessId) == null)
{
// The original [neon-cli] process is no longer running;
// it must have terminated before signalling the end
// of the operation. We're going to clear the operation
// status.
//
// This is an important fail-safe.
StopOperation();
return;
}
}
/// <inheritdoc/>
public override async Task RunAsync(CommandLine commandLine)
{
Console.WriteLine();
var currentContextName = KubeHelper.CurrentContextName;
if (currentContextName == null)
{
Console.Error.WriteLine("*** ERROR: No cluster selected.");
Program.Exit(1);
}
var currentLogin = KubeHelper.GetClusterLogin(currentContextName);
NeonHelper.OpenBrowser($"https://{currentLogin.ClusterDefinition.Domain}");
await Task.CompletedTask;
}
/// <inheritdoc/>
public override void Run(CommandLine commandLine)
{
Console.WriteLine("");
// Actually logout.
if (KubeHelper.CurrentContext == null)
{
Console.WriteLine($"You are not logged into a cluster.");
return;
}
Console.WriteLine($"Logging out of: {KubeHelper.CurrentContext.Name}");
KubeHelper.SetCurrentContext((string)null);
Console.WriteLine("");
// Notify the desktop application.
KubeHelper.Desktop.Logout().Wait();
}
public void CreateIso()
{
// Verify that we can create a new ISO file from a folder.
using (var tempFolder = new TempFolder())
{
using (var tempIso = new TempFile(suffix: ".iso"))
{
for (int i = 0; i < 10; i++)
{
File.WriteAllText(Path.Combine(tempFolder.Path, $"{i}.txt"), $"{i}");
}
KubeHelper.CreateIsoFile(tempFolder.Path, tempIso.Path, "TEST");
using (var file = new FileStream(tempIso.Path, FileMode.Open, FileAccess.Read))
{
Assert.True(file.Length > 0);
}
}
}
}
public void CreateKubernetesObject()
{
// Create a few global Kubernetes objects and verify that their ApiVersion
// and Kind properties are initialized properly.
var configmap = KubeHelper.CreateKubeObject <V1ConfigMap>("test");
Assert.Equal(V1ConfigMap.KubeApiVersion, configmap.ApiVersion);
Assert.Equal(V1ConfigMap.KubeKind, configmap.Kind);
var deployment = KubeHelper.CreateKubeObject <V1Deployment>("test");
Assert.Equal($"{V1Deployment.KubeGroup}/{V1Deployment.KubeApiVersion}", deployment.ApiVersion);
Assert.Equal(V1Deployment.KubeKind, deployment.Kind);
// Verify that a custom object can be created.
var test = KubeHelper.CreateKubeObject <V1Test>("test");
Assert.Equal($"{V1Test.KubeGroup}/{V1Test.KubeApiVersion}", test.ApiVersion);
Assert.Equal(V1Test.KubeKind, test.Kind);
}
/// <summary>
/// Handles cluster context commands.
/// </summary>
/// <param name="sender">The sender.</param>
/// <param name="args">The arguments.</param>
private void OnClusterContext(object sender, EventArgs args)
{
// The cluster context name is the text of the sending menu item.
var menuItem = (MenuItem)sender;
var contextName = menuItem.Text;
StartOperation(connectingAnimation);
try
{
KubeHelper.SetCurrentContext(contextName);
ShowToast($"Logged into: {contextName}");
}
catch
{
StopFailedOperation($"Cannot log into: {contextName}");
}
finally
{
StopOperation();
}
}
/// <summary>
/// Constructor.
/// </summary>
public MainForm()
{
MainForm.Current = this;
InitializeComponent();
Load += MainForm_Load;
Shown += (s, a) => Visible = false; // The main form should always be hidden
// Ensure that temporary files are written to the users temporary folder because
// there's a decent chance that this folder will be encrypted at rest.
TempFile.Root = KubeHelper.TempFolder;
TempFolder.Root = KubeHelper.TempFolder;
// Preload the notification icons and animations for better performance.
appIcon = new Icon(@"Images\app.ico");
connectedIcon = new Icon(@"Images\connected.ico");
disconnectedIcon = new Icon(@"Images\disconnected.ico");
errorIcon = new Icon(@"Images\error.ico");
connectingAnimation = AnimatedIcon.Load("Images", "connecting", animationFrameRate);
workingAnimation = AnimatedIcon.Load("Images", "working", animationFrameRate);
errorAnimation = AnimatedIcon.Load("Images", "error", animationFrameRate);
notifyStack = new Stack <NotifyState>();
// Initialize the cluster hosting provider components.
HostingLoader.Initialize();
// Initialize the client state.
proxies = new List <ReverseProxy>();
portForwards = new List <PortForward>();
Headend = new HeadendClient();
KubeHelper.LoadClientConfig();
}
/// <summary>
/// Constructor.
/// </summary>
/// <param name="mode">Optionally specifies the test mode. This defaults to <see cref="KubeClusterspaceMode.EnabledWithSharedCache"/>.</param>
/// <exception cref="InvalidOperationException">Thrown if another test manager instance is active.</exception>
public KubeTestManager(KubeClusterspaceMode mode = KubeClusterspaceMode.EnabledWithSharedCache)
{
lock (syncLock)
{
if (Current != null)
{
throw new InvalidOperationException("Another test manager is already active.");
}
try
{
tempFolder = new TempFolder();
Current = this;
KubeHelper.SetClusterSpaceMode(mode, tempFolder.Path);
}
catch
{
KubeHelper.ResetClusterspaceMode();
Current = null;
throw;
}
}
}
/// <inheritdoc/>
public override async Task RunAsync(CommandLine commandLine)
{
if (commandLine.Arguments.Length != 2)
{
Help();
Program.Exit(1);
}
var sourceFolder = commandLine.Arguments.ElementAtOrDefault(0);
var isoPath = commandLine.Arguments.ElementAtOrDefault(1);
var linux = commandLine.GetFlag("--linux");
var label = commandLine.GetOption("--label");
if (string.IsNullOrEmpty(sourceFolder) || string.IsNullOrEmpty(isoPath))
{
Help();
Program.Exit(1);
}
if (linux)
{
foreach (var file in Directory.EnumerateFiles(sourceFolder))
{
var text = File.ReadAllText(file);
text = NeonHelper.ToLinuxLineEndings(text);
File.WriteAllText(file, text);
}
}
KubeHelper.CreateIsoFile(sourceFolder, isoPath, label);
Program.Exit(0);
await Task.CompletedTask;
}
/// <inheritdoc/>
public override async Task RunAsync(CommandLine commandLine)
{
KubeContextName contextName = null;
var path = commandLine.Arguments.FirstOrDefault();
var rawName = commandLine.GetOption("--context");
if (rawName != null)
{
contextName = KubeContextName.Parse(rawName);
if (!contextName.IsNeonKube)
{
Console.Error.WriteLine($"*** ERROR: [{contextName}] is not a neonKUBE context.");
Program.Exit(1);
}
}
else
{
contextName = KubeHelper.CurrentContextName;
if (contextName == null)
{
Console.Error.WriteLine($"*** ERROR: You are not logged into a neonKUBE cluster.");
Program.Exit(1);
}
}
var context = KubeHelper.Config.GetContext(contextName);
if (context == null)
{
Console.Error.WriteLine($"*** ERROR: Context [{contextName}] not found.");
Program.Exit(1);
}
var cluster = KubeHelper.Config.GetCluster(context.Properties.Cluster);
var user = KubeHelper.Config.GetUser(context.Properties.User);
if (context == null)
{
Console.Error.WriteLine($"*** ERROR: Context [{contextName}] not found.");
Program.Exit(1);
}
if (user == null)
{
Console.Error.WriteLine($"*** ERROR: User [{context.Properties.User}] not found.");
Program.Exit(1);
}
var login = new ClusterLoginExport()
{
Cluster = cluster,
Context = context,
Extensions = KubeHelper.GetClusterLogin(contextName),
User = user
};
var yaml = NeonHelper.YamlSerialize(login);
if (path == null)
{
Console.WriteLine(yaml);
}
else
{
File.WriteAllText(path, yaml);
}
await Task.CompletedTask;
}
public RCONController(KubeHelper client)
{
_client = client;
}
/// <summary>
/// Updates the running proxies to match the current cluster
/// (if there is one). This may only be called on the UI thread.
/// </summary>
private async Task UpdateProxiesAsync()
{
if (InvokeRequired)
{
throw new InvalidOperationException($"[{nameof(UpdateProxiesAsync)}()] may only be called on the UI thread.");
}
if (KubeHelper.CurrentContext == null)
{
StopProxies();
StopPortForwards();
}
else
{
var cluster = Program.GetCluster();
// We're going to use the current Kubenetes context name and cluster ID
// to determine whether we're still connected to the same cluster.
if (proxiedContext != null)
{
if (proxiedContext.Name == KubeHelper.CurrentContext.Name &&
proxiedContext.Extension.ClusterId == KubeHelper.CurrentContext.Extension.ClusterId)
{
// We're still proxying the same cluster so no changes
// are required.
return;
}
}
StopProxies();
StopPortForwards();
if (KubeHelper.CurrentContext == null)
{
// Wr're not logged into a cluster so don't start any proxies.
return;
}
try
{
// Start the connecting animation if we're not already in the error state.
if (!InErrorState)
{
StartNotifyAnimation(connectingAnimation, $"{KubeHelper.CurrentContextName}: Connecting...", isTransient: true);
}
//-------------------------------------------------------------
// The Kubernetes dashboard reverse proxy.
// Setup a callback that transparently adds an [Authentication] header
// to all requests with the correct bearer token. We'll need to
// obtain the token secret via two steps:
//
// 1. Identify the dashboard token secret by listing all secrets
// in the [kube-system] namespace looking for one named like
// [root-user-token-*].
//
// 2. Reading that secret and extracting the value.
var response = (ExecuteResponse)null;
var secretName = string.Empty;
var dashboardToken = string.Empty;
await Task.Run(
async() =>
{
response = KubeHelper.Kubectl("--namespace", "kube-system", "get", "secrets", "-o=name");
await Task.CompletedTask;
});
if (response.ExitCode != 0)
{
try
{
response.EnsureSuccess();
}
catch (Exception e)
{
Program.LogError(e);
SetErrorState($"{KubeHelper.CurrentContextName}: Kubernetes API failure");
return;
}
}
// Step 1: Determine the secret name.
using (var reader = new StringReader(response.OutputText))
{
const string secretPrefix = "secret/";
secretName = reader.Lines().FirstOrDefault(line => line.StartsWith($"{secretPrefix}root-user-token-"));
Covenant.Assert(!string.IsNullOrEmpty(secretName));
secretName = secretName.Substring(secretPrefix.Length);
}
// Step 2: Describe the secret and extract the token value. This
// is a bit of a hack because I'm making assumptions about
// the output format.
await Task.Run(
async() =>
{
response = KubeHelper.Kubectl("--namespace", "kube-system", "describe", "secret", secretName);
await Task.CompletedTask;
});
if (response.ExitCode != 0)
{
try
{
response.EnsureSuccess();
}
catch (Exception e)
{
Program.LogError(e);
SetErrorState($"{KubeHelper.CurrentContextName}: Kubernetes API failure");
return;
}
}
using (var reader = new StringReader(response.OutputText))
{
var tokenLine = reader.Lines().FirstOrDefault(line => line.StartsWith("token:"));
Covenant.Assert(!string.IsNullOrEmpty(tokenLine));
dashboardToken = tokenLine.Split(new char[] { ' ' }, 2).Skip(1).First().Trim();
}
Action <RequestContext> dashboardRequestHandler =
context =>
{
context.Request.Headers.Add("Authorization", $"Bearer {dashboardToken}");
};
// Start the proxy.
var userContext = KubeHelper.Config.GetUser(KubeHelper.CurrentContext.Properties.User);
var certPem = Encoding.UTF8.GetString(Convert.FromBase64String(userContext.Properties.ClientCertificateData));
var keyPem = Encoding.UTF8.GetString(Convert.FromBase64String(userContext.Properties.ClientKeyData));
var dashboardCert = TlsCertificate.Parse(KubeHelper.CurrentContext.Extension.KubernetesDashboardCertificate).ToX509(publicOnly: true);
var kubeDashboardProxy =
new ReverseProxy(
localPort: KubeHelper.ClientConfig.KubeDashboardProxyPort,
remotePort: KubeHostPorts.KubeDashboard,
remoteHost: cluster.GetReachableMaster().PrivateAddress.ToString(),
validCertificate: dashboardCert,
requestHandler: dashboardRequestHandler);
proxies.Add(kubeDashboardProxy);
var kibanaDashboardProxy =
new PortForward(
serviceName: "kibana-kibana",
localPort: KubeConst.KibanaDashboardProxyPort,
remotePort: KubeConst.KibanaDashboardProxyPort,
@namespace: "logging");
portForwards.Add(kibanaDashboardProxy);
var prometheusDashboardProxy =
new PortForward(
serviceName: "prometheus",
localPort: KubeConst.PrometheusDashboardProxyPort,
remotePort: KubeConst.PrometheusDashboardProxyPort,
@namespace: "istio-system");
portForwards.Add(prometheusDashboardProxy);
var kialiDashboardProxy =
new PortForward(
serviceName: "kiali",
localPort: KubeConst.KialiDashboardProxyPort,
remotePort: KubeConst.KialiDashboardProxyPort,
@namespace: "istio-system");
portForwards.Add(kialiDashboardProxy);
var grafanaDashboardProxy =
new PortForward(
serviceName: "grafana",
localPort: KubeConst.GrafanaDashboardProxyPort,
remotePort: KubeConst.GrafanaDashboardProxyPort,
@namespace: "istio-system");
portForwards.Add(grafanaDashboardProxy);
//-------------------------------------------------------------
// Remember which cluster context we're proxying.
proxiedContext = KubeHelper.CurrentContext;
}
catch (Exception e)
{
Console.WriteLine(e);
}
finally
{
// Stop any non-error transient animation on the top of the notify stack.
if (notifyStack.Count > 0 && notifyStack.Peek().IsTransient&& !notifyStack.Peek().IsError)
{
StopNotifyAnimation();
}
}
}
}
/// <inheritdoc/>
public override void Run(CommandLine commandLine)
{
if (commandLine.HasHelpOption)
{
Help();
Program.Exit(0);
}
// Special-case handling of the [--remove-templates] option.
if (commandLine.HasOption("--remove-templates"))
{
Console.WriteLine("Removing cached virtual machine templates.");
foreach (var fileName in Directory.GetFiles(KubeHelper.VmTemplatesFolder, "*.*", SearchOption.TopDirectoryOnly))
{
File.Delete(fileName);
}
Program.Exit(0);
}
// Implement the command.
if (KubeHelper.CurrentContext != null)
{
Console.Error.WriteLine("*** ERROR: You are logged into a cluster. You need to logout before preparing another.");
Program.Exit(1);
}
if (commandLine.Arguments.Length == 0)
{
Console.Error.WriteLine($"*** ERROR: CLUSTER-DEF expected.");
Program.Exit(1);
}
clusterDefPath = commandLine.Arguments[0];
force = commandLine.GetFlag("--force");
ClusterDefinition.ValidateFile(clusterDefPath, strict: true);
var clusterDefinition = ClusterDefinition.FromFile(clusterDefPath, strict: true);
clusterDefinition.Provisioner = $"neon-cli:{Program.Version}"; // Identify this tool/version as the cluster provisioner
// NOTE:
//
// Azure has a more restrictive password policy and our default
// machine password does not meet the requirements:
//
// The supplied password must be between 6-72 characters long and must
// satisfy at least 3 of password complexity requirements from the following:
//
// 1. Contains an uppercase character
// 2. Contains a lowercase character
// 3. Contains a numeric digit
// 4. Contains a special character
// 5. Control characters are not allowed
//
// It's also probably not a great idea to use a static password when
// provisioning VMs in public clouds because it might be possible for
// somebody to use this fact the SSH into nodes while the cluster is
// being setup and before we set the secure password at the end.
//
// This is less problematic for non-cloud environments because it's
// likely that the hosts won't initially be able to receive inbound
// Internet traffic and besides, we need to have a known password
// embedded into the VM templates.
//
// We're going to handle this for cloud environments by looking
// at [Program.MachinePassword]. If this is set to the default
// machine password then we're going to replace it with a randomlly
// generated password with a few extra characters to ensure that
// it meets the target cloud's password requirements. We'll use
// a non-default password if the operator specified one.
if (clusterDefinition.Hosting.IsCloudProvider && Program.MachinePassword == KubeConst.DefaulVmTemplatePassword)
{
Program.MachinePassword = NeonHelper.GetCryptoRandomPassword(20);
// Append a string that guarantees that the generated password meets
// cloud minimum requirements.
Program.MachinePassword += ".Aa0";
}
// NOTE: Cluster prepare starts new log files.
cluster = new ClusterProxy(clusterDefinition, Program.CreateNodeProxy <NodeDefinition>, appendToLog: false, defaultRunOptions: RunOptions.LogOutput | RunOptions.FaultOnError);
if (KubeHelper.Config.GetContext(cluster.Definition.Name) != null)
{
Console.Error.WriteLine($"*** ERROR: A context named [{cluster.Definition.Name}] already exists.");
Program.Exit(1);
}
// Configure global options.
if (commandLine.HasOption("--unredacted"))
{
cluster.SecureRunOptions = RunOptions.None;
}
var failed = false;
try
{
KubeHelper.Desktop.StartOperationAsync($"Preparing [{cluster.Name}]").Wait();
//-----------------------------------------------------------------
// Try to ensure that no servers are already deployed on the IP addresses defined
// for cluster nodes because provisoning over an existing cluster will likely
// corrupt the existing cluster and also probably prevent the new cluster from
// provisioning correctly.
//
// Note that we're not going to perform this check for the [Machine] hosting
// environment because we're expecting the bare machines to be already running
// with the assigned addresses and we're also not going to do this for cloud
// environments because we're assuming that the cluster will run in its own
// private network so there'll ne no possibility of conflicts.
if (cluster.Definition.Hosting.Environment != HostingEnvironments.Machine &&
!cluster.Definition.Hosting.IsCloudProvider)
{
Console.WriteLine();
Console.WriteLine(" Scanning for IP address conflicts...");
Console.WriteLine();
var pingOptions = new PingOptions(ttl: 32, dontFragment: true);
var pingTimeout = TimeSpan.FromSeconds(2);
var pingConflicts = new List <NodeDefinition>();
var pingAttempts = 2;
// I'm going to use up to 20 threads at a time here for simplicity
// rather then doing this as async operations.
var parallelOptions = new ParallelOptions()
{
MaxDegreeOfParallelism = 20
};
Parallel.ForEach(cluster.Definition.NodeDefinitions.Values, parallelOptions,
node =>
{
using (var pinger = new Pinger())
{
// We're going to try pinging up to [pingAttempts] times for each node
// just in case the network it sketchy and we're losing reply packets.
for (int i = 0; i < pingAttempts; i++)
{
var reply = pinger.SendPingAsync(node.PrivateAddress, (int)pingTimeout.TotalMilliseconds).Result;
if (reply.Status == IPStatus.Success)
{
lock (pingConflicts)
{
pingConflicts.Add(node);
}
break;
}
}
}
});
if (pingConflicts.Count > 0)
{
Console.Error.WriteLine($"*** ERROR: Cannot provision the cluster because [{pingConflicts.Count}] other");
Console.Error.WriteLine($"*** machines conflict with the following cluster nodes:");
Console.Error.WriteLine();
foreach (var node in pingConflicts.OrderBy(n => NetHelper.AddressToUint(IPAddress.Parse(n.PrivateAddress))))
{
Console.Error.WriteLine($"{node.PrivateAddress, 16}: {node.Name}");
}
Program.Exit(1);
}
}
//-----------------------------------------------------------------
// Perform basic environment provisioning. This creates basic cluster components
// such as virtual machines, networks, load balancers, public IP addresses, security
// groups,... as required for the environment.
hostingManager = new HostingManagerFactory(() => HostingLoader.Initialize()).GetMaster(cluster, Program.LogPath);
if (hostingManager == null)
{
Console.Error.WriteLine($"*** ERROR: No hosting manager for the [{cluster.Definition.Hosting.Environment}] hosting environment could be located.");
Program.Exit(1);
}
hostingManager.HostUsername = Program.MachineUsername;
hostingManager.HostPassword = Program.MachinePassword;
hostingManager.ShowStatus = !Program.Quiet;
hostingManager.MaxParallel = Program.MaxParallel;
hostingManager.WaitSeconds = Program.WaitSeconds;
if (hostingManager.RequiresAdminPrivileges)
{
Program.VerifyAdminPrivileges($"Provisioning to [{cluster.Definition.Hosting.Environment}] requires elevated administrator privileges.");
}
if (!hostingManager.Provision(force))
{
Program.Exit(1);
}
// Get the mounted drive prefix from the hosting manager.
cluster.Definition.DrivePrefix = hostingManager.DrivePrefix;
// Ensure that the nodes have valid IP addresses.
cluster.Definition.ValidatePrivateNodeAddresses();
var ipAddressToServer = new Dictionary <IPAddress, SshProxy <NodeDefinition> >();
foreach (var node in cluster.Nodes.OrderBy(n => n.Name))
{
SshProxy <NodeDefinition> duplicateServer;
if (node.PrivateAddress == IPAddress.Any)
{
throw new ArgumentException($"Node [{node.Name}] has not been assigned an IP address.");
}
if (ipAddressToServer.TryGetValue(node.PrivateAddress, out duplicateServer))
{
throw new ArgumentException($"Nodes [{duplicateServer.Name}] and [{node.Name}] have the same IP address [{node.Metadata.PrivateAddress}].");
}
ipAddressToServer.Add(node.PrivateAddress, node);
}
// We're going to use the masters as package caches unless the user
// specifies something else.
packageCaches = commandLine.GetOption("--package-cache"); // This overrides the cluster definition, if specified.
if (!string.IsNullOrEmpty(packageCaches))
{
cluster.Definition.PackageProxy = packageCaches;
}
if (string.IsNullOrEmpty(cluster.Definition.PackageProxy))
{
var sbProxies = new StringBuilder();
foreach (var master in cluster.Masters)
{
sbProxies.AppendWithSeparator($"{master.PrivateAddress}:{NetworkPorts.AppCacherNg}");
}
cluster.Definition.PackageProxy = sbProxies.ToString();
}
//-----------------------------------------------------------------
// Prepare the cluster.
// Write the operation begin marker to all cluster node logs.
cluster.LogLine(logBeginMarker);
var nodesText = cluster.Nodes.Count() == 1 ? "node" : "nodes";
var operation = $"Preparing [{cluster.Definition.Name}] {nodesText}";
var controller =
new SetupController <NodeDefinition>(operation, cluster.Nodes)
{
ShowStatus = !Program.Quiet,
MaxParallel = Program.MaxParallel
};
controller.AddGlobalStep("setup details",
() =>
{
using (var client = new HeadendClient())
{
kubeSetupInfo = client.GetSetupInfoAsync(cluster.Definition).Result;
}
});
// Prepare the nodes.
controller.AddWaitUntilOnlineStep(timeout: TimeSpan.FromMinutes(15));
hostingManager.AddPostProvisionSteps(controller);
controller.AddStep("verify OS", CommonSteps.VerifyOS);
controller.AddStep("prepare",
(node, stepDelay) =>
{
Thread.Sleep(stepDelay);
CommonSteps.PrepareNode(node, cluster.Definition, kubeSetupInfo, shutdown: false);
},
stepStaggerSeconds: cluster.Definition.Setup.StepStaggerSeconds);
if (!controller.Run())
{
// Write the operation end/failed marker to all cluster node logs.
cluster.LogLine(logFailedMarker);
Console.Error.WriteLine("*** ERROR: One or more configuration steps failed.");
Program.Exit(1);
}
// Persist the cluster context extension.
var contextExtensionsPath = KubeHelper.GetContextExtensionPath((KubeContextName)$"{KubeConst.RootUser}@{clusterDefinition.Name}");
var contextExtension = new KubeContextExtension(contextExtensionsPath)
{
ClusterDefinition = clusterDefinition,
SshUsername = Program.MachineUsername,
SshPassword = Program.MachinePassword,
SetupDetails = new KubeSetupDetails()
{
SetupPending = true
}
};
contextExtension.Save();
// Write the operation end marker to all cluster node logs.
cluster.LogLine(logEndMarker);
}
catch
{
failed = true;
throw;
}
finally
{
if (!failed)
{
KubeHelper.Desktop.EndOperationAsync($"Cluster [{cluster.Name}] has been prepared and is ready for setup.").Wait();
}
else
{
KubeHelper.Desktop.EndOperationAsync($"Cluster [{cluster.Name}] prepare has failed.", failed: true).Wait();
}
}
}
public PodsController(KubeHelper client)
{
_client = client;
}
/// <inheritdoc/>
public override async Task RunAsync(CommandLine commandLine)
{
if (commandLine.Arguments.Length > 0)
{
Console.Error.WriteLine("*** ERROR: Unexpected argument.");
Program.Exit(1);
}
Console.WriteLine();
var contextName = KubernetesClientConfiguration.BuildDefaultConfig().CurrentContext;
if (string.IsNullOrEmpty(contextName))
{
Console.Error.WriteLine($"*** ERROR: There is no current cluster.");
Program.Exit(1);
}
var clusterLogin = KubeHelper.GetClusterLogin(KubeContextName.Parse(contextName));
if (clusterLogin == null)
{
Console.Error.WriteLine($"*** ERROR: There is no current cluster or the current cluster is not a neonKUBE cluster.");
Program.Exit(1);
}
// Handle the command line options.
var all = commandLine.HasOption("--all");
var containerImages = commandLine.HasOption("--container-images");
var priorityClass = commandLine.HasOption("--priority-class");
var resources = commandLine.HasOption("--resources");
var details = commandLine.HasOption("--details");
if (all || (!containerImages && !priorityClass && !resources))
{
containerImages = true;
priorityClass = true;
resources = true;
}
// Perform the requested checks.
var k8s = new Kubernetes(KubernetesClientConfiguration.BuildConfigFromConfigFile(KubeHelper.KubeConfigPath));
var error = false;
if (containerImages && !await ClusterChecker.CheckNodeContainerImagesAsync(clusterLogin, k8s, details: details))
{
error = true;
}
if (priorityClass && !await ClusterChecker.CheckPodPrioritiesAsync(clusterLogin, k8s, details: details))
{
error = true;
}
if (resources && !await ClusterChecker.CheckResourcesAsync(clusterLogin, k8s, details: details))
{
error = true;
}
if (error)
{
Console.Error.WriteLine();
Console.Error.WriteLine("*** ERROR: Cluster check failed with one or more errors.");
Program.Exit(1);
}
await Task.CompletedTask;
}
public void VirtualMachines(bool isAdmin)
{
try
{
using (CreateService(isAdmin))
{
var hyperVProxy = new HyperVProxy(isAdmin, socketPath);
// List VMs before we create any below. We had an issue once where we'd see
// a [NullReferenceException] when there were no VMs.
var vms = hyperVProxy.ListVms();
// Create a VM and verify.
hyperVProxy.AddVm(
machineName: TestMachineName1,
memorySize: "1 GiB",
processorCount: 2,
drivePath: test1VhdxPath,
checkpointDrives: false,
templateDrivePath: templatePath,
switchName: "External");
var vm = hyperVProxy.GetVm(machineName: TestMachineName1);
Assert.NotNull(vm);
Assert.Equal(TestMachineName1, vm.Name);
Assert.Equal(VirtualMachineState.Off, vm.State);
Assert.Equal("External", vm.SwitchName);
// Start the VM and verify.
hyperVProxy.StartVm(machineName: TestMachineName1);
vm = hyperVProxy.GetVm(machineName: TestMachineName1);
Assert.NotNull(vm);
Assert.Equal(VirtualMachineState.Running, vm.State);
// Fetch the VM network adapters.
var adapters = hyperVProxy.GetVmNetworkAdapters(TestMachineName1);
Assert.NotNull(adapters);
Assert.NotEmpty(adapters);
// Save the VM and verify.
hyperVProxy.SaveVm(machineName: TestMachineName1);
vm = hyperVProxy.GetVm(machineName: TestMachineName1);
Assert.NotNull(vm);
Assert.Equal(VirtualMachineState.Saved, vm.State);
// Create and start another VM and verify.
hyperVProxy.AddVm(
machineName: TestMachineName2,
memorySize: "1 GiB",
processorCount: 2,
drivePath: test2VhdxPath,
checkpointDrives: false,
templateDrivePath: templatePath,
switchName: "External");
vm = hyperVProxy.GetVm(machineName: TestMachineName2);
Assert.NotNull(vm);
Assert.Equal(TestMachineName2, vm.Name);
Assert.Equal(VirtualMachineState.Off, vm.State);
Assert.Equal("External", vm.SwitchName);
hyperVProxy.StartVm(machineName: TestMachineName2);
vm = hyperVProxy.GetVm(machineName: TestMachineName2);
Assert.Equal(VirtualMachineState.Running, vm.State);
// List and check the VM existence.
var list = hyperVProxy.ListVms();
Assert.Contains(list, item => item.Name == TestMachineName1);
Assert.Contains(list, item => item.Name == TestMachineName2);
Assert.True(hyperVProxy.VmExists(TestMachineName1));
Assert.True(hyperVProxy.VmExists(TestMachineName2));
Assert.False(hyperVProxy.VmExists(Guid.NewGuid().ToString("d")));
// Test DVD/CD insert and eject operations.
using (var tempFolder = new TempFolder())
{
var isoFolder = Path.Combine(tempFolder.Path, "iso-contents");
var isoPath = Path.Combine(tempFolder.Path, "data.iso");
Directory.CreateDirectory(isoFolder);
File.WriteAllText(Path.Combine(isoFolder, "hello.txt"), "HELLO WORLD!");
KubeHelper.CreateIsoFile(isoFolder, isoPath);
// $todo(jefflill): Eject is failing:
//
// https://github.com/nforgeio/neonKUBE/issues/1456
// hyperVProxy.InsertVmDvd(TestMachineName2, isoPath);
// hyperVProxy.EjectVmDvd(TestMachineName2);
}
// Stop the second VM and verify.
hyperVProxy.StopVm(machineName: TestMachineName2, turnOff: true);
vm = hyperVProxy.GetVm(machineName: TestMachineName2);
Assert.Equal(VirtualMachineState.Off, vm.State);
// Add a drive to the second VM and verify.
hyperVProxy.AddVmDrive(TestMachineName2,
new VirtualDrive()
{
Path = extraVhdxPath,
Size = 1 * ByteUnits.GibiBytes,
IsDynamic = true
});
var drives = hyperVProxy.GetVmDrives(machineName: TestMachineName1);
// $todo(jefflill): We should be seeing two drives here:
//
// https://github.com/nforgeio/neonKUBE/issues/1455
Assert.NotEmpty(drives);
// Assert.Equal(2, drives.Count);
// Compact the extra drive we added to the second VM.
hyperVProxy.CompactDrive(extraVhdxPath);
// Remove the VMs and verify.
hyperVProxy.RemoveVm(machineName: TestMachineName1, keepDrives: false);
Assert.Null(hyperVProxy.GetVm(machineName: TestMachineName1));
}
}
finally
{
ClearState();
}
}
/// <summary>
/// <para>
/// Deploys a new test cluster as specified by the cluster definition passed or connects
/// to a cluster previously deployed by this method when the cluster definition of the
/// existing cluster and the definition passed here are the same.
/// </para>
/// </summary>
/// <param name="clusterDefinition">The cluster definition model.</param>
/// <param name="options">
/// Optionally specifies the options that <see cref="ClusterFixture"/> will use to
/// manage the test cluster.
/// </param>
/// <returns>
/// <para>
/// The <see cref="TestFixtureStatus"/>:
/// </para>
/// <list type="table">
/// <item>
/// <term><see cref="TestFixtureStatus.Disabled"/></term>
/// <description>
/// Returned when cluster unit testing is disabled due to the <c>NEON_CLUSTER_TESTING</c> environment
/// variable not being present on the current machine which means that <see cref="TestHelper.IsClusterTestingEnabled"/>
/// returns <c>false</c>.
/// </description>
/// </item>
/// <item>
/// <term><see cref="TestFixtureStatus.Started"/></term>
/// <description>
/// Returned when one of the <c>Start()</c> methods is called for the first time for the fixture
/// instance, indicating that an existing cluster has been connected or a new cluster has been deployed.
/// </description>
/// </item>
/// <item>
/// <term><see cref="TestFixtureStatus.AlreadyRunning"/></term>
/// <description>
/// Returned when one of the <c>Start()</c> methods has already been called by your test
/// class instance.
/// </description>
/// </item>
/// </list>
/// </returns>
/// <exception cref="NeonKubeException">Thrown when the test cluster could not be deployed.</exception>
/// <remarks>
/// <para>
/// <b>IMPORTANT:</b> Only one <see cref="ClusterFixture"/> can be run at a time on
/// any one computer. This is due to the fact that cluster state like the kubeconfig,
/// neonKUBE logins, logs and other files will be written to <b>~/.neonkube/spaces/$fixture/*</b>
/// so multiple fixture instances will be confused when trying to manage these same files.
/// </para>
/// <para>
/// This means that not only will running <see cref="ClusterFixture"/> based tests in parallel
/// within the same instance of Visual Studio fail, but running these tests in different
/// Visual Studio instances will also fail.
/// </para>
/// </remarks>
public TestFixtureStatus StartWithClusterDefinition(ClusterDefinition clusterDefinition, ClusterFixtureOptions options = null)
{
Covenant.Requires <ArgumentNullException>(clusterDefinition != null, nameof(clusterDefinition));
if (clusterDefinition.IsLocked)
{
throw new NeonKubeException("Test clusters need to be unlocked. Please set [isLocked: false] in your cluster definition.");
}
if (!TestHelper.IsClusterTestingEnabled)
{
return(TestFixtureStatus.Disabled);
}
if (started)
{
return(TestFixtureStatus.AlreadyRunning);
}
options ??= new ClusterFixtureOptions();
this.options = options.Clone();
if (this.Cluster != null)
{
return(TestFixtureStatus.AlreadyRunning);
}
// Set the clusterspace mode, using any previously downloaded node image unless
// the user specifies a custom image. We're going to host the fixture state
// files in this fixed folder:
//
// ~/.neonkube/spaces/$fixture/*
clusterspaceFolder = KubeHelper.SetClusterSpaceMode(string.IsNullOrEmpty(options.ImageUriOrPath) ? KubeClusterspaceMode.EnabledWithSharedCache : KubeClusterspaceMode.Enabled, KubeHelper.ClusterspacePrefix("fixture"));
// Figure out whether the user passed an image URI or file path to override
// the default node image.
var imageUriOrPath = options.ImageUriOrPath;
var imageUri = (string)null;
var imagePath = (string)null;
if (string.IsNullOrEmpty(imageUriOrPath))
{
imageUriOrPath = KubeDownloads.GetDefaultNodeImageUri(clusterDefinition.Hosting.Environment);
}
if (imageUriOrPath.StartsWith("http://", StringComparison.InvariantCultureIgnoreCase) || imageUriOrPath.StartsWith("https://", StringComparison.InvariantCultureIgnoreCase))
{
imageUri = imageUriOrPath;
}
else
{
imagePath = imageUriOrPath;
}
//-------------------------------------------------------------
// We need to deal with some scenarios here:
//
// 1. No cluster context or login exists for the target cluster.
//
// A conflicting cluster may still exist though, having been deployed
// by another computer or perhaps the kubecontext/logins on the current
// machine may have been modified. We need to be sure to remove any
// conflicting resources in this case.
//
// 2. Cluster context and login exist on the current machine for the target
// cluster but the cluster is unhealthy or locked. We'll abort for locked
// clusters and remove and redeploy for unhealth clusters.
//
// 3. Cluster context and login exist and the cluster is healthy. In this case,
// we need to compare the deployed cluster version against the current version
// and remove/redeploy when the versions don't match.
//
// 4. Cluster context and login exist and the cluster is healthy and cluster versions
// match. In this case, We'll compare the existing cluster definition with that for
// the new cluster and also compare the cluster versions and if they match and
// [RemoveClusterOnStart=false] we'll just use the existing cluster.
//
// 5. The current cluster matches the target but [RemoveClusterOnStart=true].
// We need to remove the current cluster in this case so we'll deploy a
// fresh one.
// Determine whether a test cluster with the same name exists and if
// its cluster definition matches the test cluster's definition.
;
var clusterExists = false;
var clusterContextName = KubeContextName.Parse($"root@{clusterDefinition.Name}");
var clusterContext = KubeHelper.Config.GetContext(clusterContextName);
var clusterLogin = KubeHelper.GetClusterLogin(clusterContextName);
if (clusterContext != null && clusterLogin != null && !clusterLogin.SetupDetails.SetupPending)
{
clusterExists = ClusterDefinition.AreSimilar(clusterDefinition, clusterLogin.ClusterDefinition);
}
if (clusterExists && !options.RemoveClusterOnStart)
{
// It looks like the test cluster may already exist. We'll verify
// that it's running, healthy, unlocked and the cluster versions match.
// When all of these conditions are true, we'll use the existing cluster,
// otherwise we'll remove the cluster as well as its context/login,
// and deploy a new cluster below.
using (var cluster = new ClusterProxy(clusterLogin.ClusterDefinition, new HostingManagerFactory()))
{
KubeHelper.SetCurrentContext(clusterContextName);
var isLocked = cluster.IsLockedAsync().ResultWithoutAggregate();
var clusterInfo = cluster.GetClusterInfoAsync().ResultWithoutAggregate();
var clusterHealth = cluster.GetClusterHealthAsync().ResultWithoutAggregate();
if (isLocked.HasValue && isLocked.Value)
{
throw new NeonKubeException($"Cluster is locked: {cluster.Name}");
}
if (clusterHealth.State == ClusterState.Healthy && clusterInfo.ClusterVersion == KubeVersions.NeonKube)
{
// We need to reset an existing cluster to ensure it's in a known state.
cluster.ResetAsync().WaitWithoutAggregate();
started = true;
IsRunning = true;
Cluster = new ClusterProxy(KubeHelper.CurrentContext, new HostingManagerFactory());
return(TestFixtureStatus.Started);
}
cluster.RemoveAsync(removeOrphans: true).WaitWithoutAggregate();
}
}
else
{
// There is no known existing cluster but there still might be a cluster
// deployed by another machine or fragments of a partially deployed cluster,
// so we need to do a preemptive cluster remove.
using (var cluster = new ClusterProxy(clusterDefinition, new HostingManagerFactory()))
{
cluster.RemoveAsync(removeOrphans: true).WaitWithoutAggregate();
}
}
// Provision the new cluster.
WriteTestOutputLine($"PREPARE CLUSTER: {clusterDefinition.Name}");
try
{
var controller = KubeSetup.CreateClusterPrepareController(
clusterDefinition: clusterDefinition,
nodeImageUri: imageUri,
nodeImagePath: imagePath,
maxParallel: options.MaxParallel,
unredacted: options.Unredacted,
neonCloudHeadendUri: options.NeonCloudHeadendUri);
switch (controller.RunAsync().ResultWithoutAggregate())
{
case SetupDisposition.Succeeded:
WriteTestOutputLine("CLUSTER PREPARE: SUCCESS");
break;
case SetupDisposition.Failed:
WriteTestOutputLine("CLUSTER PREPARE: FAIL");
throw new NeonKubeException("Cluster prepare failed.");
case SetupDisposition.Cancelled:
default:
throw new NotImplementedException();
}
}
finally
{
if (options.CaptureDeploymentLogs)
{
CaptureDeploymentLogs();
}
}
// Setup the cluster.
WriteTestOutputLine($"SETUP CLUSTER: {clusterDefinition.Name}");
try
{
var controller = KubeSetup.CreateClusterSetupController(
clusterDefinition: clusterDefinition,
maxParallel: options.MaxParallel,
unredacted: options.Unredacted);
switch (controller.RunAsync().ResultWithoutAggregate())
{
case SetupDisposition.Succeeded:
WriteTestOutputLine("CLUSTER SETUP: SUCCESS");
break;
case SetupDisposition.Failed:
WriteTestOutputLine("CLUSTER SETUP: FAILED");
throw new NeonKubeException("Cluster setup failed.");
case SetupDisposition.Cancelled:
default:
throw new NotImplementedException();
}
}
finally
{
if (options.CaptureDeploymentLogs)
{
CaptureDeploymentLogs();
}
}
// NOTE: We just deployed brand new cluster so there's no need to reset it.
started = true;
IsRunning = true;
Cluster = new ClusterProxy(KubeHelper.CurrentContext, new HostingManagerFactory());
return(TestFixtureStatus.Started);
}
/// <inheritdoc/>
public override async Task RunAsync(CommandLine commandLine)
{
if (commandLine.Arguments.Length < 1)
{
Console.Error.WriteLine("*** ERROR: [root@CLUSTER-NAME] argument is required.");
Program.Exit(1);
}
Console.WriteLine();
// Cluster prepare/setup uses the [ProfileClient] to retrieve secrets and profile values.
// We need to inject an implementation for [PreprocessReader] so it will be able to
// perform the lookups.
NeonHelper.ServiceContainer.AddSingleton <IProfileClient>(new ProfileClient());
var contextName = KubeContextName.Parse(commandLine.Arguments[0]);
var kubeCluster = KubeHelper.Config.GetCluster(contextName.Cluster);
var unredacted = commandLine.HasOption("--unredacted");
var debug = commandLine.HasOption("--debug");
var check = commandLine.HasOption("--check");
var uploadCharts = commandLine.HasOption("--upload-charts") || debug;
var clusterspace = commandLine.GetOption("--clusterspace");
var maxParallelOption = commandLine.GetOption("--max-parallel", "6");
var disablePending = commandLine.HasOption("--disable-pending");
if (!int.TryParse(maxParallelOption, out var maxParallel) || maxParallel <= 0)
{
Console.Error.WriteLine($"*** ERROR: [--max-parallel={maxParallelOption}] is not valid.");
Program.Exit(1);
}
clusterLogin = KubeHelper.GetClusterLogin(contextName);
if (clusterLogin == null)
{
Console.Error.WriteLine($"*** ERROR: Be sure to prepare the cluster first via: neon cluster prepare...");
Program.Exit(1);
}
if (string.IsNullOrEmpty(clusterLogin.SshPassword))
{
Console.Error.WriteLine($"*** ERROR: No cluster node SSH password found.");
Program.Exit(1);
}
if (kubeCluster != null && !clusterLogin.SetupDetails.SetupPending)
{
if (commandLine.GetOption("--force") == null && !Program.PromptYesNo($"One or more logins reference [{kubeCluster.Name}]. Do you wish to delete these?"))
{
Program.Exit(0);
}
// Remove the cluster from the kubeconfig and remove any
// contexts that reference it.
KubeHelper.Config.Clusters.Remove(kubeCluster);
var delList = new List <KubeConfigContext>();
foreach (var context in KubeHelper.Config.Contexts)
{
if (context.Properties.Cluster == kubeCluster.Name)
{
delList.Add(context);
}
}
foreach (var context in delList)
{
KubeHelper.Config.Contexts.Remove(context);
}
if (KubeHelper.CurrentContext != null && KubeHelper.CurrentContext.Properties.Cluster == kubeCluster.Name)
{
KubeHelper.Config.CurrentContext = null;
}
KubeHelper.Config.Save();
}
kubeContext = new KubeConfigContext(contextName);
KubeHelper.InitContext(kubeContext);
// Create and run the cluster setup controller.
var clusterDefinition = clusterLogin.ClusterDefinition;
var controller = KubeSetup.CreateClusterSetupController(
clusterDefinition,
maxParallel: maxParallel,
unredacted: unredacted,
debugMode: debug,
uploadCharts: uploadCharts,
clusterspace: clusterspace);
controller.DisablePendingTasks = disablePending;
controller.StatusChangedEvent +=
status =>
{
status.WriteToConsole();
};
switch (await controller.RunAsync())
{
case SetupDisposition.Succeeded:
var pendingGroups = controller.GetPendingGroups();
if (pendingGroups.Count > 0)
{
Console.WriteLine($"*** ERROR: [{pendingGroups.Count}] pending task groups have not been awaited:");
Console.WriteLine();
foreach (var groupName in pendingGroups)
{
Console.WriteLine($" {groupName}");
}
Program.Exit(1);
}
Console.WriteLine();
Console.WriteLine($" [{clusterDefinition.Name}] cluster is ready.");
Console.WriteLine();
if (check && !debug)
{
var k8s = new Kubernetes(KubernetesClientConfiguration.BuildConfigFromConfigFile(KubeHelper.KubeConfigPath));
if (!await ClusterChecker.CheckAsync(clusterLogin, k8s))
{
Program.Exit(1);
}
}
Program.Exit(0);
break;
case SetupDisposition.Cancelled:
Console.WriteLine(" *** CANCELLED: Cluster setup was cancelled.");
Console.WriteLine();
Console.WriteLine();
Program.Exit(1);
break;
case SetupDisposition.Failed:
Console.WriteLine();
Console.WriteLine(" *** ERROR: Cluster setup failed. Examine the logs here:");
Console.WriteLine();
Console.WriteLine($" {KubeHelper.LogFolder}");
Console.WriteLine();
Program.Exit(1);
break;
default:
throw new NotImplementedException();
}
await Task.CompletedTask;
}
/// <inheritdoc/>
public override async Task RunAsync(CommandLine commandLine)
{
if (commandLine.HasHelpOption)
{
Help();
Program.Exit(0);
}
Console.WriteLine();
// Cluster prepare/setup uses the [ProfileClient] to retrieve secrets and profile values.
// We need to inject an implementation for [PreprocessReader] so it will be able to
// perform the lookups.
NeonHelper.ServiceContainer.AddSingleton <IProfileClient>(new ProfileClient());
// Handle the [--remove-templates] option.
if (commandLine.HasOption("--remove-templates"))
{
Console.WriteLine("Removing cached virtual machine templates.");
foreach (var fileName in Directory.GetFiles(KubeHelper.NodeImageFolder, "*.*", SearchOption.TopDirectoryOnly))
{
File.Delete(fileName);
}
}
var nodeImageUri = commandLine.GetOption("--node-image-uri");
var nodeImagePath = commandLine.GetOption("--node-image-path");
var debug = commandLine.HasOption("--debug");
var baseImageName = commandLine.GetOption("--base-image-name");
var clusterspace = commandLine.GetOption("--clusterspace");
var headendUri = commandLine.GetOption("--headend-uri") ?? KubeConst.NeonCloudHeadendUri;
var maxParallelOption = commandLine.GetOption("--max-parallel", "6");
var disablePending = commandLine.HasOption("--disable-pending");
if (!int.TryParse(maxParallelOption, out var maxParallel) || maxParallel <= 0)
{
Console.Error.WriteLine($"*** ERROR: [--max-parallel={maxParallelOption}] is not valid.");
Program.Exit(1);
}
if (debug && string.IsNullOrEmpty(baseImageName))
{
Console.Error.WriteLine($"*** ERROR: [--base-image-name] is required for [--debug] mode.");
Program.Exit(1);
}
// Implement the command.
if (KubeHelper.CurrentContext != null)
{
Console.Error.WriteLine("*** ERROR: You are logged into a cluster. You need to logout before preparing another.");
Program.Exit(1);
}
if (commandLine.Arguments.Length == 0)
{
Console.Error.WriteLine($"*** ERROR: CLUSTER-DEF expected.");
Program.Exit(1);
}
// Obtain the cluster definition.
var clusterDefPath = commandLine.Arguments[0];
var clusterDefinition = (ClusterDefinition)null;
ClusterDefinition.ValidateFile(clusterDefPath, strict: true);
clusterDefinition = ClusterDefinition.FromFile(clusterDefPath, strict: true);
// Do a quick sanity check to ensure that the hosting environment has enough
// resources (memory and disk) to actually host the cluster.
using (var cluster = new ClusterProxy(clusterDefinition, new HostingManagerFactory()))
{
var status = await cluster.GetResourceAvailabilityAsync();
if (!status.CanBeDeployed)
{
Console.Error.WriteLine();
Console.Error.WriteLine($"*** ERROR: Insufficent resources available to deploy cluster.");
Console.Error.WriteLine();
foreach (var entity in status.Constraints.Keys
.OrderBy(key => key, StringComparer.InvariantCultureIgnoreCase))
{
Console.Error.WriteLine();
Console.Error.WriteLine($"{entity}:");
foreach (var constraint in status.Constraints[entity])
{
Console.Error.WriteLine($" {constraint.ResourceType.ToString().ToUpperInvariant()}: {constraint.Details}");
}
}
Console.Error.WriteLine();
Program.Exit(1);
}
}
if (KubeHelper.IsOnPremiseHypervisorEnvironment(clusterDefinition.Hosting.Environment))
{
// Use the default node image for the hosting environment unless [--node-image-uri]
// or [--node-image-path] was specified.
if (string.IsNullOrEmpty(nodeImageUri) && string.IsNullOrEmpty(nodeImagePath))
{
nodeImageUri = KubeDownloads.GetDefaultNodeImageUri(clusterDefinition.Hosting.Environment);
}
}
// Parse any specified package cache endpoints.
var packageCaches = commandLine.GetOption("--package-caches", null);
var packageCacheEndpoints = new List <IPEndPoint>();
if (!string.IsNullOrEmpty(packageCaches))
{
foreach (var item in packageCaches.Split(' ', StringSplitOptions.RemoveEmptyEntries))
{
if (!NetHelper.TryParseIPv4Endpoint(item, out var endpoint))
{
Console.Error.WriteLine($"*** ERROR: [{item}] is not a valid package cache IPv4 endpoint.");
Program.Exit(1);
}
packageCacheEndpoints.Add(endpoint);
}
}
// Create and run the cluster prepare controller.
var controller = KubeSetup.CreateClusterPrepareController(
clusterDefinition,
nodeImageUri: nodeImageUri,
nodeImagePath: nodeImagePath,
maxParallel: maxParallel,
packageCacheEndpoints: packageCacheEndpoints,
unredacted: commandLine.HasOption("--unredacted"),
debugMode: debug,
baseImageName: baseImageName,
clusterspace: clusterspace,
neonCloudHeadendUri: headendUri);
controller.DisablePendingTasks = disablePending;
controller.StatusChangedEvent +=
status =>
{
status.WriteToConsole();
};
switch (await controller.RunAsync())
{
case SetupDisposition.Succeeded:
var pendingGroups = controller.GetPendingGroups();
if (pendingGroups.Count > 0)
{
Console.WriteLine($"*** ERROR: [{pendingGroups.Count}] pending task groups have not been awaited:");
Console.WriteLine();
foreach (var groupName in pendingGroups)
{
Console.WriteLine($" {groupName}");
}
Program.Exit(1);
}
Console.WriteLine();
Console.WriteLine($" [{clusterDefinition.Name}] cluster is prepared.");
Console.WriteLine();
Program.Exit(0);
break;
case SetupDisposition.Cancelled:
Console.WriteLine();
Console.WriteLine(" *** CANCELLED: Cluster prepare was cancelled.");
Console.WriteLine();
Program.Exit(1);
break;
case SetupDisposition.Failed:
Console.WriteLine();
Console.WriteLine(" *** ERROR: Cluster prepare has failed. Examine the logs here:");
Console.WriteLine();
Console.WriteLine($" {KubeHelper.LogFolder}");
Console.WriteLine();
Program.Exit(1);
break;
default:
throw new NotImplementedException();
}
await Task.CompletedTask;
}
/// <inheritdoc/>
public override async Task RunAsync(CommandLine commandLine)
{
if (commandLine.HasHelpOption || commandLine.Arguments.Length == 0)
{
Console.WriteLine(usage);
Program.Exit(0);
}
Console.Error.WriteLine();
var currentContext = KubeHelper.CurrentContext;
var newContextName = KubeContextName.Parse(commandLine.Arguments.First());
// Ensure that the new context exists.
if (KubeHelper.Config.GetContext(newContextName) == null)
{
Console.Error.WriteLine($"*** Context [{newContextName}] not found.");
Program.Exit(1);
}
// Check whether we're already logged into the cluster.
if (KubeHelper.CurrentContext != null && newContextName == KubeContextName.Parse(KubeHelper.CurrentContext.Name))
{
Console.Error.WriteLine($"*** You are already logged into: {newContextName}");
Program.Exit(0);
}
// Logout of the current cluster.
if (currentContext != null)
{
Console.Error.WriteLine($"Logout: {currentContext.Name}...");
KubeHelper.SetCurrentContext((string)null);
}
// Log into the new context and then send a simple command to ensure
// that cluster is ready.
var orgContext = KubeHelper.CurrentContext;
KubeHelper.SetCurrentContext(newContextName);
Console.WriteLine($"Login: {newContextName}...");
try
{
using (var k8s = new Kubernetes(KubernetesClientConfiguration.BuildConfigFromConfigFile(KubeHelper.KubeConfigPath)))
{
await k8s.ListNamespaceAsync();
}
if (!string.IsNullOrEmpty(NeonHelper.DockerCli))
{
Console.WriteLine($"Login: Docker to Harbor...");
var login = KubeHelper.GetClusterLogin(KubeHelper.CurrentContextName);
NeonHelper.Execute(NeonHelper.DockerCli,
new object[]
{
"login",
$"{ClusterDomain.HarborRegistry}.{login.ClusterDefinition.Domain}",
"--username",
"root",
"--password-stdin"
},
input: new StringReader(login.SsoPassword));
}
}
catch (Exception e)
{
KubeHelper.SetCurrentContext(orgContext?.Name);
Console.WriteLine("*** ERROR: Cluster is not responding.");
Console.WriteLine();
Console.WriteLine(NeonHelper.ExceptionError(e));
Console.WriteLine();
Program.Exit(1);
}
Console.WriteLine();
Console.WriteLine($"Now logged into: {newContextName}");
await Task.CompletedTask;
}
