public void Conversions() { Assert.True(NetHelper.AddressEquals(IPAddress.Parse("0.0.0.0"), NetHelper.UintToAddress(0))); Assert.True(NetHelper.AddressEquals(IPAddress.Parse("255.0.0.0"), NetHelper.UintToAddress(0xFF000000))); Assert.True(NetHelper.AddressEquals(IPAddress.Parse("1.2.3.4"), NetHelper.UintToAddress(0x01020304))); Assert.Equal(0x00000000L, NetHelper.AddressToUint(IPAddress.Parse("0.0.0.0"))); Assert.Equal(0xFF000000L, NetHelper.AddressToUint(IPAddress.Parse("255.0.0.0"))); Assert.Equal(0x01020304L, NetHelper.AddressToUint(IPAddress.Parse("1.2.3.4"))); }
/// <summary> /// Configures OpenVPN on a manager node. /// </summary> /// <param name="manager">The manager.</param> private void ConfigManagerVpn(SshProxy <NodeDefinition> manager) { // Upload the setup and configuration files. // // NOTE: // // These steps are redundant and will be repeated during the // common node configuration, but we need some of the scripts // here, before that happens. manager.CreateHiveHostFolders(); manager.UploadConfigFiles(hive.Definition); manager.UploadResources(hive.Definition); // Install OpenVPN. manager.Status = "vpn install"; manager.SudoCommand("safe-apt-get update"); manager.SudoCommand("safe-apt-get install -yq openvpn"); // Configure OpenVPN. var nodesSubnet = NetworkCidr.Parse(hive.Definition.Network.NodesSubnet); var vpnSubnet = NetworkCidr.Parse(manager.Metadata.VpnPoolSubnet); var duplicateCN = hive.Definition.Vpn.AllowSharedCredentials ? "duplicate-cn" : ";duplicate-cn"; var vpnServerAddress = NetHelper.UintToAddress(NetHelper.AddressToUint(vpnSubnet.Address) + 1); var serverConf = $@"#------------------------------------------------------------------------------ # OpenVPN config file customized for the [{manager.Name}] neonHIVE manager node. # OpenVPN listening port. port {NetworkPorts.OpenVPN} # Enable TCP and/or UDP transports. proto tcp ;proto udp # Set packet tunneling mode. dev tun # SSL/TLS root certificate (ca), certificate # (cert), and private key (key). Each client # and the server must have their own cert and # key file. The server and all clients will # use the same ca file. # # See the [easy-rsa] directory for a series # of scripts for generating RSA certificates # and private keys. Remember to use # a unique Common Name for the server # and each of the client certificates. # # Any X509 key management system can be used. # OpenVPN can also use a PKCS #12 formatted key file # (see [pkcs12] directive in man page). ca ca.crt cert server.crt key server.key # This file should be kept secret # Diffie hellman parameters (2048-bit) generated via: # # openssl dhparam -out dhparam.pem 2048 # dh dhparam.pem # The currently recommended topology. topology subnet # Configure server mode and supply a VPN subnet # for OpenVPN to draw client addresses from. # The server will take {vpnServerAddress} for itself, # the rest will be made available to clients. # Each client will be able to reach the server # on {vpnServerAddress}. Comment this line out if you are # ethernet bridging. See the man page for more info. server {vpnSubnet.Address} {vpnSubnet.Mask} # Maintain a record of client virtual IP address # associations in this file. If OpenVPN goes down or # is restarted, reconnecting clients can be assigned # the same virtual IP address from the pool that was # previously assigned. ;ifconfig-pool-persist ipp.txt # Push routes to the client to allow it # to reach other private subnets behind # the server. Remember that these # private subnets will also need # to know to route the OpenVPN client # address pool ({vpnSubnet.Address}) # back to this specific OpenVPN server. push ""route {nodesSubnet.Address} {nodesSubnet.Mask}"" # Uncomment this directive if multiple clients # might connect with the same certificate/key # files or common names. This is recommended # only for testing purposes. For production use, # each client should have its own certificate/key # pair. {duplicateCN} # The keepalive directive causes ping-like # messages to be sent back and forth over # the link so that each side knows when # the other side has gone down. # Ping every 10 seconds, assume that remote # peer is down if no ping received during # a 120 second time period. keepalive 10 120 # For extra security beyond that provided # by SSL/TLS, create an [HMAC firewall] # to help block DoS attacks and UDP port flooding. # # Generate with: # openvpn --genkey --secret ta.key # # The server and each client must have # a copy of this key. # The second parameter should be '0' # on the server and '1' on the clients. tls-auth ta.key 0 # This file is secret # Select a cryptographic cipher. # This config item must be copied to # the client config file as well. cipher AES-256-CBC # Enable compression on the VPN link. # Don't enable this unless it is also # enabled in the client config file. # # We're not enabling this due to the # VORACLE security vulnerablity: # # https://community.openvpn.net/openvpn/wiki/VORACLE # # The maximum number of concurrently connected # clients we want to allow. max-clients {VpnOptions.ServerAddressCount - 2} # This macro sets the TCP_NODELAY socket flag on # the server as well as pushes it to connecting # clients. The TCP_NODELAY flag disables the Nagle # algorithm on TCP sockets causing packets to be # transmitted immediately with low latency, rather # than waiting a short period of time in order to # aggregate several packets into a larger containing # packet. In VPN applications over TCP, TCP_NODELAY # is generally a good latency optimization. tcp-nodelay # It's a good idea to reduce the OpenVPN # daemon's privileges after initialization. # # You can uncomment this out on # non-Windows systems. ;user nobody ;group nobody # The persist options will try to avoid # accessing certain resources on restart # that may no longer be accessible because # of the privilege downgrade. persist-key persist-tun # Output a short status file showing # current connections, truncated # and rewritten every minute. status openvpn-status.log # By default, log messages will go to the syslog (ork # on Windows, if running as a service, they will go to # the [\Program Files\OpenVPN\log] directory). # Use log or log-append to override this default. # [log] will truncate the log file on OpenVPN startup, # while [log-append] will append to it. Use one # or the other (but not both). log /var/log/openvpn.log ;log-append openvpn.log # Set the appropriate level of log # file verbosity. # # 0 is silent, except for fatal errors # 4 is reasonable for general usage # 5 and 6 can help to debug connection problems # 9 is extremely verbose verb 4 # Silence repeating messages. At most 20 # sequential messages of the same message # category will be output to the log. ;mute 20 "; manager.Status = "vpn config"; manager.SudoCommand("mkdir -p /etc/openvpn"); manager.UploadText("/etc/openvpn/server.conf", serverConf); manager.UploadText("/etc/openvpn/ca.crt", vpnCaFiles.GetCert("ca")); manager.UploadText("/etc/openvpn/server.crt", vpnCaFiles.GetCert("server")); manager.UploadText("/etc/openvpn/server.key", vpnCaFiles.GetKey("server")); manager.SudoCommand("chmod 600", "/etc/openvpn/server.key"); // This is a secret! manager.UploadText("/etc/openvpn/ta.key", vpnCaFiles.GetTaKey()); manager.SudoCommand("chmod 600", "/etc/openvpn/ta.key"); // This is a secret too! manager.UploadText("/etc/openvpn/dhparam.pem", vpnCaFiles.GetDHParam()); // Initialize the [root] user's credentials. vpnCredentials = new VpnCredentials() { CaCert = vpnCaFiles.GetCert("ca"), UserCert = vpnCaFiles.GetCert(HiveConst.RootUser), UserKey = vpnCaFiles.GetKey(HiveConst.RootUser), TaKey = vpnCaFiles.GetTaKey(), CaZipKey = VpnCaFiles.GenerateKey(), CaZip = vpnCaFiles.ToZipBytes() }; // Upload the initial (empty) Certificate Revocation List (CRL) file and then // upload a OpenVPN systemd unit drop-in so that it will recognize revoked certificates. manager.UploadText("/etc/openvpn/crl.pem", vpnCaFiles.GetFile("crl.pem")); manager.SudoCommand("chmod 664", "/etc/openvpn/crl.pem"); // OpenVPN needs to be able to read this after having its privileges downgraded. var openVpnUnit = @"[Unit] Description=OpenVPN connection to %i PartOf=openvpn.service ReloadPropagatedFrom=openvpn.service Before=systemd-user-sessions.service Documentation=man:openvpn(8) Documentation=https://community.openvpn.net/openvpn/wiki/Openvpn23ManPage Documentation=https://community.openvpn.net/openvpn/wiki/HOWTO [Service] PrivateTmp=true KillMode=mixed Type=forking ExecStart=/usr/sbin/openvpn --daemon ovpn-%i --status /run/openvpn/%i.status 10 --cd /etc/openvpn --script-security 2 --config /etc/openvpn/%i.conf --writepid /run/openvpn/%i.pid --crl-verify /etc/openvpn/crl.pem PIDFile=/run/openvpn/%i.pid ExecReload=/bin/kill -HUP $MAINPID WorkingDirectory=/etc/openvpn ProtectSystem=yes CapabilityBoundingSet=CAP_IPC_LOCK CAP_NET_ADMIN CAP_NET_BIND_SERVICE CAP_NET_RAW CAP_SETGID CAP_SETUID CAP_SYS_CHROOT CAP_DAC_READ_SEARCH CAP_AUDIT_WRITE LimitNPROC=10 DeviceAllow=/dev/null rw DeviceAllow=/dev/net/tun rw [Install] WantedBy=multi-user.target "; manager.UploadText("/etc/systemd/system/[email protected]", openVpnUnit); manager.SudoCommand("chmod 644 /etc/systemd/system/[email protected]"); // Do a daemon-reload so systemd will be aware of the new drop-in. manager.SudoCommand("systemctl disable openvpn"); manager.SudoCommand("systemctl daemon-reload"); // Enable and restart OpenVPN. manager.SudoCommand("systemctl enable openvpn"); manager.SudoCommand("systemctl restart openvpn"); //----------------------------------------------------------------- // SPECIAL NOTE: // // I figured out that I need this lovely bit of code after banging my head on the desk for // 12 freaking days. The problem was getting OpenVPN to work in Windows Azure (this will // also probably impact other cloud environments). // // Azure implements VNETs as layer 3 overlays. This means that the host network interfaces // are not actually on an ethernet segment and the VPN default gateway is actually handling // all of the ARP packets, routing between the VNET subnets, load balancers, and the Internet. // This is problematic for OpenVPN traffic because the VPN client IP address space is not // part of the VNET which means the VNET gateway is not able to route packets from hive // hosts back to the manager's OpenVPN client addresses by default. // // The solution is to configure the managers with secondary NIC cards in a different subnet // and provision special Azure user-defined routes that direct VPN return packets to the // correct manager. // // I figured this part out the second day. The problem was though that it simply didn't work. // From an external VPN client, I would try to ping a worker node through OpenVPN running on // a manager. I'd see the ping traffic: // // 1. manager/tun0: request // 2. manager/eth1: request // 3. worker/eth0: request // 4. worker/eth0: reply // 5. manager/eth0: reply // 6: NOTHING! EXPECTED: manager/tun0: reply // // So the problem was that I could see the ICMP ping request hit the various interfaces // on the manager and be received by the worker. I'd then see the worker send the reply, // and be routed via the user-defined Azure rult back to the manager. The problem was // that the packet was simply dropped there. It never made it back to tun0 so OpenVPN // could forward it back to the client. // // After days and days of trying to learn about Linux routing, iptables and policy rules, // I finally ran across this posting for the second time: // // https://unix.stackexchange.com/questions/21093/output-traffic-on-different-interfaces-based-on-destination-port // // This was the key. I ran across this a few days ago and didn't read it closely enough. // It made more sense after learning more about this stuff. // // Linux has a built-in IP address spoofing filter enabled by default. This filter has the // kernel discard any packets whose source address doesn't match the IP address/route implied // by the remote interface that transmitted the packet. This is exactly what's happening // when Azure forwards the VPN return packets via the user-defined route. I'd see return // packets hit eth0 on the manager, be processed by the low-level RAW and MANGLE iptables // and then they'd disappear. // // The solution is simply to disable the spoofing filter. I'm going to go ahead and do this // for all interfaces which should be fine for hives hosted in cloud environments, because the // VNET/Load Balancer/Security Groups will be used to lock things down. Local hives will // need to be manually placed behind a suitable router/firewall as well. // // For robustness, I'm going to deploy this as a service daemon that polls the filter state // for each interface every 5 seconds, and disables any enabled filters. This will ensure // that the filters will always be disabled, even as interfaces are bought up and down. var disableSpoofUnit = $@"[Unit] Description=Disable Network Anti-Spoofing Filters Documentation= After= Requires= Before= [Service] Type=simple ExecStart={HiveHostFolders.Bin}/disable-spoof-filters.sh [Install] WantedBy=multi-user.target "; var disableSpoofScript = @"#!/bin/bash #------------------------------------------------------------------------------ # This script is a deployed as a service to ensure that the Linux anti-spoofing # filters are disabled for the network interfaces on manager nodes hosting # OpenVPN. This is required to allow VPN return traffic from other nodes to # routed back to tun0 and ultimately, connected VPN clients. # # Note that it appears that we need to disable the filter for all interfaces # for this to actually work. while : do flush=false for f in /proc/sys/net/ipv4/conf/*/rp_filter do filter_enabled=$(cat $f) if [ ""$filter_enabled"" == ""1"" ] ; then echo 0 > $f flush=true fi done if [ ""$flush"" == ""true"" ] ; then echo 1 > /proc/sys/net/ipv4/route/flush fi sleep 5 done"; manager.UploadText("/lib/systemd/system/disable-spoof-filters.service", disableSpoofUnit); manager.SudoCommand("chmod 644 /lib/systemd/system/disable-spoof-filters.service"); manager.UploadText($"{HiveHostFolders.Bin}/disable-spoof-filters.sh", disableSpoofScript); manager.SudoCommand($"chmod 770 {HiveHostFolders.Bin}/disable-spoof-filters.sh"); manager.SudoCommand("systemctl enable disable-spoof-filters"); manager.SudoCommand("systemctl restart disable-spoof-filters"); }
/// <summary> /// Used by cloud and potentially other hosting manager implementations to verify the /// node address assignments and/or to automatically assign these addresses. /// </summary> /// <param name="clusterDefinition">The cluster definition.</param> /// <remarks> /// <note> /// This method verifies that node addresses for on-premise environments are located /// within the premise subnet. The method will not attempt to assign node addresses /// for on-premise node and requires all nodes have explicit addresses. /// </note> /// </remarks> protected void AssignNodeAddresses(ClusterDefinition clusterDefinition) { Covenant.Requires <ArgumentNullException>(clusterDefinition != null, nameof(clusterDefinition)); var networkOptions = clusterDefinition.Network; // Verify that explicit address assignments are not duplicated // across any nodes. var addressToNode = new Dictionary <IPAddress, NodeDefinition>(); foreach (var node in clusterDefinition.SortedNodes) { if (string.IsNullOrEmpty(node.Address)) { continue; } var address = NetHelper.ParseIPv4Address(node.Address); if (addressToNode.TryGetValue(address, out var conflictNode)) { throw new ClusterDefinitionException($"Nodes [{conflictNode.Name}] and [{node.Name}] both specify the same address [{address}]. Node addresses must be unique."); } addressToNode.Add(address, node); } if (KubeHelper.IsOnPremiseEnvironment(clusterDefinition.Hosting.Environment)) { // Verify that all nodes have explicit addresses for on-premise environments. foreach (var node in clusterDefinition.SortedNodes) { if (string.IsNullOrEmpty(node.Address)) { throw new ClusterDefinitionException($"Node [{node.Name}] is not assigned an address. All nodes must have explicit IP addresses for on-premise hosting environments."); } } return; } // Ensure that any explicit node IP address assignments are located // within the subnet where the nodes will be provisioned and do not // conflict with any of the addresses reserved by the cloud provider // or neonKUBE. var nodeSubnetInfo = clusterDefinition.NodeSubnet; var nodeSubnet = NetworkCidr.Parse(nodeSubnetInfo.Subnet); if (clusterDefinition.Nodes.Count() > nodeSubnet.AddressCount - nodeSubnetInfo.ReservedAddresses) { throw new ClusterDefinitionException($"The cluster includes [{clusterDefinition.Nodes.Count()}] nodes which will not fit within the [{nodeSubnet}] target subnet after accounting for [{nodeSubnetInfo.ReservedAddresses}] reserved addresses."); } var firstValidAddressUint = NetHelper.AddressToUint(nodeSubnet.FirstAddress) + KubeConst.CloudSubnetStartReservedIPs; var firstValidAddress = NetHelper.UintToAddress(firstValidAddressUint); var lastValidAddressUint = NetHelper.AddressToUint(nodeSubnet.LastAddress) - KubeConst.CloudSubnetEndReservedIPs; var lastValidAddress = NetHelper.UintToAddress(lastValidAddressUint); foreach (var node in clusterDefinition.SortedNodes.OrderBy(node => node.Name)) { if (string.IsNullOrEmpty(node.Address)) { // Ignore nodes with unassigned addresses. continue; } var address = NetHelper.ParseIPv4Address(node.Address); if (!nodeSubnet.Contains(address)) { throw new ClusterDefinitionException($"Node [{node.Name}] is assigned [{node.Address}={node.Address}] which is outside of the [{nodeSubnet}]."); } var addressUint = NetHelper.AddressToUint(address); if (addressUint < firstValidAddressUint) { throw new ClusterDefinitionException($"Node [{node.Name}] defines IP address [{node.Address}={node.Address}] which is reserved. The first valid node address for subnet [{nodeSubnet}] is [{firstValidAddress}]."); } if (addressUint > lastValidAddressUint) { throw new ClusterDefinitionException($"Node [{node.Name}] defines IP address [{node.Address}={node.Address}] which is reserved. The last valid node address for subnet [{nodeSubnet}] is [{lastValidAddress}]."); } } //----------------------------------------------------------------- // Automatically assign unused IP addresses within the subnet to nodes that // were not explicitly assigned an address in the cluster definition. var assignedAddresses = new HashSet <uint>(); foreach (var node in clusterDefinition.SortedNodes) { if (string.IsNullOrEmpty(node.Address)) { continue; } var address = NetHelper.ParseIPv4Address(node.Address); var addressUint = NetHelper.AddressToUint(address); if (!assignedAddresses.Contains(addressUint)) { assignedAddresses.Add(addressUint); } } foreach (var azureNode in clusterDefinition.SortedControlThenWorkerNodes) { if (!string.IsNullOrEmpty(azureNode.Address)) { continue; } for (var addressUint = firstValidAddressUint; addressUint <= lastValidAddressUint; addressUint++) { if (!assignedAddresses.Contains(addressUint)) { azureNode.Address = NetHelper.UintToAddress(addressUint).ToString(); assignedAddresses.Add(addressUint); break; } } } }
/// <inheritdoc/> public override void Run(CommandLine commandLine) { if (commandLine.HasHelpOption) { Help(); Program.Exit(0); } // Special-case handling of the [--remove-templates] option. if (commandLine.HasOption("--remove-templates")) { Console.WriteLine("Removing cached virtual machine templates."); foreach (var fileName in Directory.GetFiles(HiveHelper.GetVmTemplatesFolder(), "*.*", SearchOption.TopDirectoryOnly)) { File.Delete(fileName); } Program.Exit(0); } // Implement the command. packageCacheUri = commandLine.GetOption("--package-cache"); // This overrides the hive definition, if specified. if (Program.HiveLogin != null) { Console.Error.WriteLine("*** ERROR: You are logged into a hive. You need to logout before preparing another."); Program.Exit(1); } if (commandLine.Arguments.Length == 0) { Console.Error.WriteLine($"*** ERROR: HIVE-DEF expected."); Program.Exit(1); } hiveDefPath = commandLine.Arguments[0]; force = commandLine.GetFlag("--force"); HiveDefinition.ValidateFile(hiveDefPath, strict: true); var hiveDefinition = HiveDefinition.FromFile(hiveDefPath, strict: true); hiveDefinition.Provisioner = $"neon-cli:{Program.Version}"; // Identify this tool/version as the hive provisioner // NOTE: // // Azure has implemented a more restrictive password policy and our // default machine password does not meet the requirements: // // The supplied password must be between 6-72 characters long and must // satisfy at least 3 of password complexity requirements from the following: // // 1. Contains an uppercase character // 2. Contains a lowercase character // 3. Contains a numeric digit // 4. Contains a special character // 5. Control characters are not allowed // // It's also probably not a great idea to use a static password when // provisioning VMs in public clouds because it might be possible for // somebody to use this fact the SSH into nodes while the hive is being // setup and before we set the secure password at the end. // // This is less problematic for non-cloud environments because it's // likely that the hosts won't initially be able to receive inbound // Internet traffic and besides, we need to have a known password // embedded into the VM templates. // // We're going to handle this for cloud environments by looking // at [Program.MachinePassword]. If this is set to the default // machine password then we're going to replace it with a randomlly // generated password with a few extra characters to ensure that // it meets the target cloud's password requirements. We'll use // a non-default password if the operator specified one. if (hiveDefinition.Hosting.IsCloudProvider && Program.MachinePassword == HiveConst.DefaulVmTemplatePassword) { Program.MachinePassword = NeonHelper.GetRandomPassword(20); // Append a string that guarantees that the generated password meets // cloud minimum requirements. Program.MachinePassword += ".Aa0"; } // Note that hive prepare starts new log files. hive = new HiveProxy(hiveDefinition, Program.CreateNodeProxy <NodeDefinition>, appendLog: false, useBootstrap: true, defaultRunOptions: RunOptions.LogOutput | RunOptions.FaultOnError); if (File.Exists(Program.GetHiveLoginPath(HiveConst.RootUser, hive.Definition.Name))) { Console.Error.WriteLine($"*** ERROR: A hive login named [{HiveConst.RootUser}@{hive.Definition.Name}] already exists."); Program.Exit(1); } Program.OSProperties = OSProperties.For(hiveDefinition.HiveNode.OperatingSystem); // Configure global options. if (commandLine.HasOption("--unredacted")) { hive.SecureRunOptions = RunOptions.None; } //----------------------------------------------------------------- // $todo(jeff.lill): // // We're temporarily disabling redaction to make it easier to investigate // Vault setup issues. Remove this line before final launch. // // https://github.com/jefflill/NeonForge/issues/225 hive.SecureRunOptions = RunOptions.None; //----------------------------------------------------------------- // Assign the VPN client return subnets to the manager nodes if VPN is enabled. if (hive.Definition.Vpn.Enabled) { var vpnSubnet = NetworkCidr.Parse(hive.Definition.Network.VpnPoolSubnet); var prefixLength = 25; var nextVpnSubnetAddress = vpnSubnet.Address; // Note that we're not going to assign the first block of addresses in the // VPN subnet to any managers to prevent conflicts with addresses reserved // by some cloud platforms at the beginning of a subnet. Azure for example // reserves 4 IP addresses for DNS servers and platform provided VPNs. foreach (var manager in hive.Definition.SortedManagers) { var managerVpnSubnet = new NetworkCidr(NetHelper.AddressIncrement(nextVpnSubnetAddress, VpnOptions.ServerAddressCount), prefixLength); manager.VpnPoolSubnet = managerVpnSubnet.ToString(); nextVpnSubnetAddress = managerVpnSubnet.NextAddress; } } //----------------------------------------------------------------- // Try to ensure that no servers are already deployed on the IP addresses defined // for hive nodes because provisoning over an existing hive will likely // corrupt the existing hive and also probably prevent the new hive from // provisioning correctly. // // Note that we're not going to perform this check for the [Machine] hosting // environment because we're expecting the bare machines to be already running // with the assigned addresses and we're also not going to do this for cloud // environments because we're assuming that the hive will run in its own private // network so there'll ne no possibility of conflicts. if (hive.Definition.Hosting.Environment != HostingEnvironments.Machine && !hive.Definition.Hosting.IsCloudProvider) { Console.WriteLine(); Console.WriteLine("Scanning for IP address conflicts..."); Console.WriteLine(); var pingOptions = new PingOptions(ttl: 32, dontFragment: true); var pingTimeout = TimeSpan.FromSeconds(2); var pingConflicts = new List <NodeDefinition>(); var pingAttempts = 2; // I'm going to use up to 20 threads at a time here for simplicity // rather then doing this as async operations. var parallelOptions = new ParallelOptions() { MaxDegreeOfParallelism = 20 }; Parallel.ForEach(hive.Definition.NodeDefinitions.Values, parallelOptions, node => { using (var ping = new Ping()) { // We're going to try pinging up to [pingAttempts] times for each node // just in case the network it sketchy and we're losing reply packets. for (int i = 0; i < pingAttempts; i++) { var reply = ping.Send(node.PrivateAddress, (int)pingTimeout.TotalMilliseconds); if (reply.Status == IPStatus.Success) { lock (pingConflicts) { pingConflicts.Add(node); } break; } } } }); if (pingConflicts.Count > 0) { Console.Error.WriteLine($"*** ERROR: Cannot provision the hive because [{pingConflicts.Count}] other"); Console.Error.WriteLine($"*** machines conflict with the following hive nodes:"); Console.Error.WriteLine(); foreach (var node in pingConflicts.OrderBy(n => NetHelper.AddressToUint(IPAddress.Parse(n.PrivateAddress)))) { Console.Error.WriteLine($"{node.PrivateAddress, 16}: {node.Name}"); } Program.Exit(1); } } //----------------------------------------------------------------- // Perform basic environment provisioning. This creates basic hive components // such as virtual machines, networks, load balancers, public IP addresses, security // groups,... as required for the environment. hostingManager = new HostingManagerFactory(() => HostingLoader.Initialize()).GetManager(hive, Program.LogPath); if (hostingManager == null) { Console.Error.WriteLine($"*** ERROR: No hosting manager for the [{hive.Definition.Hosting.Environment}] hosting environment could be located."); Program.Exit(1); } hostingManager.HostUsername = Program.MachineUsername; hostingManager.HostPassword = Program.MachinePassword; hostingManager.ShowStatus = !Program.Quiet; hostingManager.MaxParallel = Program.MaxParallel; hostingManager.WaitSeconds = Program.WaitSeconds; if (hostingManager.RequiresAdminPrivileges) { Program.VerifyAdminPrivileges($"Provisioning to [{hive.Definition.Hosting.Environment}] requires elevated administrator privileges."); } if (!hostingManager.Provision(force)) { Program.Exit(1); } // Get the mounted drive prefix from the hosting manager. hive.Definition.DrivePrefix = hostingManager.DrivePrefix; // Ensure that the nodes have valid IP addresses. hive.Definition.ValidatePrivateNodeAddresses(); var ipAddressToServer = new Dictionary <IPAddress, SshProxy <NodeDefinition> >(); foreach (var node in hive.Nodes.OrderBy(n => n.Name)) { SshProxy <NodeDefinition> duplicateServer; if (node.PrivateAddress == IPAddress.Any) { throw new ArgumentException($"Node [{node.Name}] has not been assigned an IP address."); } if (ipAddressToServer.TryGetValue(node.PrivateAddress, out duplicateServer)) { throw new ArgumentException($"Nodes [{duplicateServer.Name}] and [{node.Name}] have the same IP address [{node.Metadata.PrivateAddress}]."); } ipAddressToServer.Add(node.PrivateAddress, node); } //----------------------------------------------------------------- // Perform basic node provisioning including operating system updates & configuration, // and configure OpenVPN on the manager nodes so that hive setup will be // able to reach the nodes on all ports. // Write the operation begin marker to all hive node logs. hive.LogLine(logBeginMarker); var operation = $"Preparing [{hive.Definition.Name}] nodes"; var controller = new SetupController <NodeDefinition>(operation, hive.Nodes) { ShowStatus = !Program.Quiet, MaxParallel = Program.MaxParallel }; if (!string.IsNullOrEmpty(packageCacheUri)) { hive.Definition.PackageProxy = packageCacheUri; } // Prepare the nodes. controller.AddWaitUntilOnlineStep(timeout: TimeSpan.FromMinutes(15)); hostingManager.AddPostProvisionSteps(controller); controller.AddStep("verify OS", (node, stepDelay) => { Thread.Sleep(stepDelay); CommonSteps.VerifyOS(node); }); controller.AddStep("prepare", (node, stepDelay) => { Thread.Sleep(stepDelay); CommonSteps.PrepareNode(node, hive.Definition, shutdown: false); }, stepStaggerSeconds: hive.Definition.Setup.StepStaggerSeconds); // Add any VPN configuration steps. if (hive.Definition.Vpn.Enabled) { controller.AddGlobalStep("vpn credentials", () => CreateVpnCredentials()); controller.AddStep("vpn server", (node, stepDelay) => { Thread.Sleep(stepDelay); ConfigManagerVpn(node); }, node => node.Metadata.IsManager); // Add a step to establish a VPN connection if we're provisioning to a cloud. // We specifically don't want to do this if we're provisioning to a on-premise // datacenter because we're assuming that we're already directly connected to // the LAN while preparing and setting up the hive. if (hive.Definition.Hosting.IsCloudProvider) { controller.AddStep("vpn connect", (manager, stepDelay) => { Thread.Sleep(stepDelay); // Create a hive login with just enough credentials to connect the VPN. // Note that this isn't really a node specific command but I wanted to // be able to display the connection status somewhere. var vpnLogin = new HiveLogin() { Definition = hive.Definition, VpnCredentials = vpnCredentials }; // Ensure that we don't have an old VPN client for the hive running. HiveHelper.VpnClose(vpnLogin.Definition.Name); // ...and then start a new one. HiveHelper.VpnOpen(vpnLogin, onStatus: message => manager.Status = $"{message}", onError: message => manager.Status = $"ERROR: {message}"); }, n => n == hive.FirstManager); } // Perform any post-VPN setup provisioning required by the hosting provider. hostingManager.AddPostVpnSteps(controller); } if (!controller.Run()) { // Write the operation end/failed marker to all hive node logs. hive.LogLine(logFailedMarker); Console.Error.WriteLine("*** ERROR: One or more configuration steps failed."); Program.Exit(1); } // Write the hive login file. var hiveLoginPath = Program.GetHiveLoginPath(HiveConst.RootUser, hive.Definition.Name); var hiveLogin = new HiveLogin() { Path = hiveLoginPath, Username = HiveConst.RootUser, Definition = hive.Definition, SshUsername = Program.MachineUsername, SshPassword = Program.MachinePassword, SshProvisionPassword = Program.MachinePassword, SetupPending = true }; if (hive.Definition.Vpn.Enabled) { hiveLogin.VpnCredentials = vpnCredentials; } // Generate the hive certificates. const int bitCount = 2048; const int validDays = 365000; // About 1,000 years. if (hiveLogin.HiveCertificate == null) { var hostnames = new string[] { $"{hive.Name}.nhive.io", $"*.{hive.Name}.nhive.io", $"*.neon-vault.{hive.Name}.nhive.io", $"*.neon-registry-cache.{hive.Name}.nhive.io", $"*.neon-hivemq.{hive.Name}.nhive.io" }; hiveLogin.HiveCertificate = TlsCertificate.CreateSelfSigned(hostnames, bitCount, validDays, issuedBy: "neonHIVE", issuedTo: $"neonHIVE: {hiveDefinition.Name}"); hiveLogin.HiveCertificate.FriendlyName = $"neonHIVE: {hiveLogin.Definition.Name}"; } // Persist the certificates into the hive login. hiveLogin.Save(); // Write the operation end marker to all hive node logs. hive.LogLine(logEndMarker); }
/// <inheritdoc/> public override void Run(CommandLine commandLine) { if (commandLine.HasHelpOption) { Help(); Program.Exit(0); } // Special-case handling of the [--remove-templates] option. if (commandLine.HasOption("--remove-templates")) { Console.WriteLine("Removing cached virtual machine templates."); foreach (var fileName in Directory.GetFiles(KubeHelper.VmTemplatesFolder, "*.*", SearchOption.TopDirectoryOnly)) { File.Delete(fileName); } Program.Exit(0); } // Implement the command. if (KubeHelper.CurrentContext != null) { Console.Error.WriteLine("*** ERROR: You are logged into a cluster. You need to logout before preparing another."); Program.Exit(1); } if (commandLine.Arguments.Length == 0) { Console.Error.WriteLine($"*** ERROR: CLUSTER-DEF expected."); Program.Exit(1); } clusterDefPath = commandLine.Arguments[0]; force = commandLine.GetFlag("--force"); ClusterDefinition.ValidateFile(clusterDefPath, strict: true); var clusterDefinition = ClusterDefinition.FromFile(clusterDefPath, strict: true); clusterDefinition.Provisioner = $"neon-cli:{Program.Version}"; // Identify this tool/version as the cluster provisioner // NOTE: // // Azure has a more restrictive password policy and our default // machine password does not meet the requirements: // // The supplied password must be between 6-72 characters long and must // satisfy at least 3 of password complexity requirements from the following: // // 1. Contains an uppercase character // 2. Contains a lowercase character // 3. Contains a numeric digit // 4. Contains a special character // 5. Control characters are not allowed // // It's also probably not a great idea to use a static password when // provisioning VMs in public clouds because it might be possible for // somebody to use this fact the SSH into nodes while the cluster is // being setup and before we set the secure password at the end. // // This is less problematic for non-cloud environments because it's // likely that the hosts won't initially be able to receive inbound // Internet traffic and besides, we need to have a known password // embedded into the VM templates. // // We're going to handle this for cloud environments by looking // at [Program.MachinePassword]. If this is set to the default // machine password then we're going to replace it with a randomlly // generated password with a few extra characters to ensure that // it meets the target cloud's password requirements. We'll use // a non-default password if the operator specified one. if (clusterDefinition.Hosting.IsCloudProvider && Program.MachinePassword == KubeConst.DefaulVmTemplatePassword) { Program.MachinePassword = NeonHelper.GetCryptoRandomPassword(20); // Append a string that guarantees that the generated password meets // cloud minimum requirements. Program.MachinePassword += ".Aa0"; } // NOTE: Cluster prepare starts new log files. cluster = new ClusterProxy(clusterDefinition, Program.CreateNodeProxy <NodeDefinition>, appendToLog: false, defaultRunOptions: RunOptions.LogOutput | RunOptions.FaultOnError); if (KubeHelper.Config.GetContext(cluster.Definition.Name) != null) { Console.Error.WriteLine($"*** ERROR: A context named [{cluster.Definition.Name}] already exists."); Program.Exit(1); } // Configure global options. if (commandLine.HasOption("--unredacted")) { cluster.SecureRunOptions = RunOptions.None; } var failed = false; try { KubeHelper.Desktop.StartOperationAsync($"Preparing [{cluster.Name}]").Wait(); //----------------------------------------------------------------- // Try to ensure that no servers are already deployed on the IP addresses defined // for cluster nodes because provisoning over an existing cluster will likely // corrupt the existing cluster and also probably prevent the new cluster from // provisioning correctly. // // Note that we're not going to perform this check for the [Machine] hosting // environment because we're expecting the bare machines to be already running // with the assigned addresses and we're also not going to do this for cloud // environments because we're assuming that the cluster will run in its own // private network so there'll ne no possibility of conflicts. if (cluster.Definition.Hosting.Environment != HostingEnvironments.Machine && !cluster.Definition.Hosting.IsCloudProvider) { Console.WriteLine(); Console.WriteLine(" Scanning for IP address conflicts..."); Console.WriteLine(); var pingOptions = new PingOptions(ttl: 32, dontFragment: true); var pingTimeout = TimeSpan.FromSeconds(2); var pingConflicts = new List <NodeDefinition>(); var pingAttempts = 2; // I'm going to use up to 20 threads at a time here for simplicity // rather then doing this as async operations. var parallelOptions = new ParallelOptions() { MaxDegreeOfParallelism = 20 }; Parallel.ForEach(cluster.Definition.NodeDefinitions.Values, parallelOptions, node => { using (var pinger = new Pinger()) { // We're going to try pinging up to [pingAttempts] times for each node // just in case the network it sketchy and we're losing reply packets. for (int i = 0; i < pingAttempts; i++) { var reply = pinger.SendPingAsync(node.PrivateAddress, (int)pingTimeout.TotalMilliseconds).Result; if (reply.Status == IPStatus.Success) { lock (pingConflicts) { pingConflicts.Add(node); } break; } } } }); if (pingConflicts.Count > 0) { Console.Error.WriteLine($"*** ERROR: Cannot provision the cluster because [{pingConflicts.Count}] other"); Console.Error.WriteLine($"*** machines conflict with the following cluster nodes:"); Console.Error.WriteLine(); foreach (var node in pingConflicts.OrderBy(n => NetHelper.AddressToUint(IPAddress.Parse(n.PrivateAddress)))) { Console.Error.WriteLine($"{node.PrivateAddress, 16}: {node.Name}"); } Program.Exit(1); } } //----------------------------------------------------------------- // Perform basic environment provisioning. This creates basic cluster components // such as virtual machines, networks, load balancers, public IP addresses, security // groups,... as required for the environment. hostingManager = new HostingManagerFactory(() => HostingLoader.Initialize()).GetMaster(cluster, Program.LogPath); if (hostingManager == null) { Console.Error.WriteLine($"*** ERROR: No hosting manager for the [{cluster.Definition.Hosting.Environment}] hosting environment could be located."); Program.Exit(1); } hostingManager.HostUsername = Program.MachineUsername; hostingManager.HostPassword = Program.MachinePassword; hostingManager.ShowStatus = !Program.Quiet; hostingManager.MaxParallel = Program.MaxParallel; hostingManager.WaitSeconds = Program.WaitSeconds; if (hostingManager.RequiresAdminPrivileges) { Program.VerifyAdminPrivileges($"Provisioning to [{cluster.Definition.Hosting.Environment}] requires elevated administrator privileges."); } if (!hostingManager.Provision(force)) { Program.Exit(1); } // Get the mounted drive prefix from the hosting manager. cluster.Definition.DrivePrefix = hostingManager.DrivePrefix; // Ensure that the nodes have valid IP addresses. cluster.Definition.ValidatePrivateNodeAddresses(); var ipAddressToServer = new Dictionary <IPAddress, SshProxy <NodeDefinition> >(); foreach (var node in cluster.Nodes.OrderBy(n => n.Name)) { SshProxy <NodeDefinition> duplicateServer; if (node.PrivateAddress == IPAddress.Any) { throw new ArgumentException($"Node [{node.Name}] has not been assigned an IP address."); } if (ipAddressToServer.TryGetValue(node.PrivateAddress, out duplicateServer)) { throw new ArgumentException($"Nodes [{duplicateServer.Name}] and [{node.Name}] have the same IP address [{node.Metadata.PrivateAddress}]."); } ipAddressToServer.Add(node.PrivateAddress, node); } // We're going to use the masters as package caches unless the user // specifies something else. packageCaches = commandLine.GetOption("--package-cache"); // This overrides the cluster definition, if specified. if (!string.IsNullOrEmpty(packageCaches)) { cluster.Definition.PackageProxy = packageCaches; } if (string.IsNullOrEmpty(cluster.Definition.PackageProxy)) { var sbProxies = new StringBuilder(); foreach (var master in cluster.Masters) { sbProxies.AppendWithSeparator($"{master.PrivateAddress}:{NetworkPorts.AppCacherNg}"); } cluster.Definition.PackageProxy = sbProxies.ToString(); } //----------------------------------------------------------------- // Prepare the cluster. // Write the operation begin marker to all cluster node logs. cluster.LogLine(logBeginMarker); var nodesText = cluster.Nodes.Count() == 1 ? "node" : "nodes"; var operation = $"Preparing [{cluster.Definition.Name}] {nodesText}"; var controller = new SetupController <NodeDefinition>(operation, cluster.Nodes) { ShowStatus = !Program.Quiet, MaxParallel = Program.MaxParallel }; controller.AddGlobalStep("setup details", () => { using (var client = new HeadendClient()) { kubeSetupInfo = client.GetSetupInfoAsync(cluster.Definition).Result; } }); // Prepare the nodes. controller.AddWaitUntilOnlineStep(timeout: TimeSpan.FromMinutes(15)); hostingManager.AddPostProvisionSteps(controller); controller.AddStep("verify OS", CommonSteps.VerifyOS); controller.AddStep("prepare", (node, stepDelay) => { Thread.Sleep(stepDelay); CommonSteps.PrepareNode(node, cluster.Definition, kubeSetupInfo, shutdown: false); }, stepStaggerSeconds: cluster.Definition.Setup.StepStaggerSeconds); if (!controller.Run()) { // Write the operation end/failed marker to all cluster node logs. cluster.LogLine(logFailedMarker); Console.Error.WriteLine("*** ERROR: One or more configuration steps failed."); Program.Exit(1); } // Persist the cluster context extension. var contextExtensionsPath = KubeHelper.GetContextExtensionPath((KubeContextName)$"{KubeConst.RootUser}@{clusterDefinition.Name}"); var contextExtension = new KubeContextExtension(contextExtensionsPath) { ClusterDefinition = clusterDefinition, SshUsername = Program.MachineUsername, SshPassword = Program.MachinePassword, SetupDetails = new KubeSetupDetails() { SetupPending = true } }; contextExtension.Save(); // Write the operation end marker to all cluster node logs. cluster.LogLine(logEndMarker); } catch { failed = true; throw; } finally { if (!failed) { KubeHelper.Desktop.EndOperationAsync($"Cluster [{cluster.Name}] has been prepared and is ready for setup.").Wait(); } else { KubeHelper.Desktop.EndOperationAsync($"Cluster [{cluster.Name}] prepare has failed.", failed: true).Wait(); } } }