/// <inheritdoc/>
public override void Run(CommandLine commandLine)
{
if (commandLine.Arguments.Length < 1)
{
Console.Error.WriteLine("*** ERROR: HIVE-DEF is required.");
Program.Exit(1);
}
// Parse and validate the hive definition.
HiveDefinition.FromFile(commandLine.Arguments[0], strict: true);
Console.WriteLine("");
Console.WriteLine("*** The hive definition is OK.");
}
/// <inheritdoc/>
public override void Run(CommandLine commandLine)
{
if (commandLine.HasHelpOption)
{
Help();
Program.Exit(0);
}
// Special-case handling of the [--remove-templates] option.
if (commandLine.HasOption("--remove-templates"))
{
Console.WriteLine("Removing cached virtual machine templates.");
foreach (var fileName in Directory.GetFiles(HiveHelper.GetVmTemplatesFolder(), "*.*", SearchOption.TopDirectoryOnly))
{
File.Delete(fileName);
}
Program.Exit(0);
}
// Implement the command.
packageCacheUri = commandLine.GetOption("--package-cache"); // This overrides the hive definition, if specified.
if (Program.HiveLogin != null)
{
Console.Error.WriteLine("*** ERROR: You are logged into a hive. You need to logout before preparing another.");
Program.Exit(1);
}
if (commandLine.Arguments.Length == 0)
{
Console.Error.WriteLine($"*** ERROR: HIVE-DEF expected.");
Program.Exit(1);
}
hiveDefPath = commandLine.Arguments[0];
force = commandLine.GetFlag("--force");
HiveDefinition.ValidateFile(hiveDefPath, strict: true);
var hiveDefinition = HiveDefinition.FromFile(hiveDefPath, strict: true);
hiveDefinition.Provisioner = $"neon-cli:{Program.Version}"; // Identify this tool/version as the hive provisioner
// NOTE:
//
// Azure has implemented a more restrictive password policy and our
// default machine password does not meet the requirements:
//
// The supplied password must be between 6-72 characters long and must
// satisfy at least 3 of password complexity requirements from the following:
//
// 1. Contains an uppercase character
// 2. Contains a lowercase character
// 3. Contains a numeric digit
// 4. Contains a special character
// 5. Control characters are not allowed
//
// It's also probably not a great idea to use a static password when
// provisioning VMs in public clouds because it might be possible for
// somebody to use this fact the SSH into nodes while the hive is being
// setup and before we set the secure password at the end.
//
// This is less problematic for non-cloud environments because it's
// likely that the hosts won't initially be able to receive inbound
// Internet traffic and besides, we need to have a known password
// embedded into the VM templates.
//
// We're going to handle this for cloud environments by looking
// at [Program.MachinePassword]. If this is set to the default
// machine password then we're going to replace it with a randomlly
// generated password with a few extra characters to ensure that
// it meets the target cloud's password requirements. We'll use
// a non-default password if the operator specified one.
if (hiveDefinition.Hosting.IsCloudProvider && Program.MachinePassword == HiveConst.DefaulVmTemplatePassword)
{
Program.MachinePassword = NeonHelper.GetRandomPassword(20);
// Append a string that guarantees that the generated password meets
// cloud minimum requirements.
Program.MachinePassword += ".Aa0";
}
// Note that hive prepare starts new log files.
hive = new HiveProxy(hiveDefinition, Program.CreateNodeProxy <NodeDefinition>, appendLog: false, useBootstrap: true, defaultRunOptions: RunOptions.LogOutput | RunOptions.FaultOnError);
if (File.Exists(Program.GetHiveLoginPath(HiveConst.RootUser, hive.Definition.Name)))
{
Console.Error.WriteLine($"*** ERROR: A hive login named [{HiveConst.RootUser}@{hive.Definition.Name}] already exists.");
Program.Exit(1);
}
Program.OSProperties = OSProperties.For(hiveDefinition.HiveNode.OperatingSystem);
// Configure global options.
if (commandLine.HasOption("--unredacted"))
{
hive.SecureRunOptions = RunOptions.None;
}
//-----------------------------------------------------------------
// $todo(jeff.lill):
//
// We're temporarily disabling redaction to make it easier to investigate
// Vault setup issues. Remove this line before final launch.
//
// https://github.com/jefflill/NeonForge/issues/225
hive.SecureRunOptions = RunOptions.None;
//-----------------------------------------------------------------
// Assign the VPN client return subnets to the manager nodes if VPN is enabled.
if (hive.Definition.Vpn.Enabled)
{
var vpnSubnet = NetworkCidr.Parse(hive.Definition.Network.VpnPoolSubnet);
var prefixLength = 25;
var nextVpnSubnetAddress = vpnSubnet.Address;
// Note that we're not going to assign the first block of addresses in the
// VPN subnet to any managers to prevent conflicts with addresses reserved
// by some cloud platforms at the beginning of a subnet. Azure for example
// reserves 4 IP addresses for DNS servers and platform provided VPNs.
foreach (var manager in hive.Definition.SortedManagers)
{
var managerVpnSubnet = new NetworkCidr(NetHelper.AddressIncrement(nextVpnSubnetAddress, VpnOptions.ServerAddressCount), prefixLength);
manager.VpnPoolSubnet = managerVpnSubnet.ToString();
nextVpnSubnetAddress = managerVpnSubnet.NextAddress;
}
}
//-----------------------------------------------------------------
// Try to ensure that no servers are already deployed on the IP addresses defined
// for hive nodes because provisoning over an existing hive will likely
// corrupt the existing hive and also probably prevent the new hive from
// provisioning correctly.
//
// Note that we're not going to perform this check for the [Machine] hosting
// environment because we're expecting the bare machines to be already running
// with the assigned addresses and we're also not going to do this for cloud
// environments because we're assuming that the hive will run in its own private
// network so there'll ne no possibility of conflicts.
if (hive.Definition.Hosting.Environment != HostingEnvironments.Machine &&
!hive.Definition.Hosting.IsCloudProvider)
{
Console.WriteLine();
Console.WriteLine("Scanning for IP address conflicts...");
Console.WriteLine();
var pingOptions = new PingOptions(ttl: 32, dontFragment: true);
var pingTimeout = TimeSpan.FromSeconds(2);
var pingConflicts = new List <NodeDefinition>();
var pingAttempts = 2;
// I'm going to use up to 20 threads at a time here for simplicity
// rather then doing this as async operations.
var parallelOptions = new ParallelOptions()
{
MaxDegreeOfParallelism = 20
};
Parallel.ForEach(hive.Definition.NodeDefinitions.Values, parallelOptions,
node =>
{
using (var ping = new Ping())
{
// We're going to try pinging up to [pingAttempts] times for each node
// just in case the network it sketchy and we're losing reply packets.
for (int i = 0; i < pingAttempts; i++)
{
var reply = ping.Send(node.PrivateAddress, (int)pingTimeout.TotalMilliseconds);
if (reply.Status == IPStatus.Success)
{
lock (pingConflicts)
{
pingConflicts.Add(node);
}
break;
}
}
}
});
if (pingConflicts.Count > 0)
{
Console.Error.WriteLine($"*** ERROR: Cannot provision the hive because [{pingConflicts.Count}] other");
Console.Error.WriteLine($"*** machines conflict with the following hive nodes:");
Console.Error.WriteLine();
foreach (var node in pingConflicts.OrderBy(n => NetHelper.AddressToUint(IPAddress.Parse(n.PrivateAddress))))
{
Console.Error.WriteLine($"{node.PrivateAddress, 16}: {node.Name}");
}
Program.Exit(1);
}
}
//-----------------------------------------------------------------
// Perform basic environment provisioning. This creates basic hive components
// such as virtual machines, networks, load balancers, public IP addresses, security
// groups,... as required for the environment.
hostingManager = new HostingManagerFactory(() => HostingLoader.Initialize()).GetManager(hive, Program.LogPath);
if (hostingManager == null)
{
Console.Error.WriteLine($"*** ERROR: No hosting manager for the [{hive.Definition.Hosting.Environment}] hosting environment could be located.");
Program.Exit(1);
}
hostingManager.HostUsername = Program.MachineUsername;
hostingManager.HostPassword = Program.MachinePassword;
hostingManager.ShowStatus = !Program.Quiet;
hostingManager.MaxParallel = Program.MaxParallel;
hostingManager.WaitSeconds = Program.WaitSeconds;
if (hostingManager.RequiresAdminPrivileges)
{
Program.VerifyAdminPrivileges($"Provisioning to [{hive.Definition.Hosting.Environment}] requires elevated administrator privileges.");
}
if (!hostingManager.Provision(force))
{
Program.Exit(1);
}
// Get the mounted drive prefix from the hosting manager.
hive.Definition.DrivePrefix = hostingManager.DrivePrefix;
// Ensure that the nodes have valid IP addresses.
hive.Definition.ValidatePrivateNodeAddresses();
var ipAddressToServer = new Dictionary <IPAddress, SshProxy <NodeDefinition> >();
foreach (var node in hive.Nodes.OrderBy(n => n.Name))
{
SshProxy <NodeDefinition> duplicateServer;
if (node.PrivateAddress == IPAddress.Any)
{
throw new ArgumentException($"Node [{node.Name}] has not been assigned an IP address.");
}
if (ipAddressToServer.TryGetValue(node.PrivateAddress, out duplicateServer))
{
throw new ArgumentException($"Nodes [{duplicateServer.Name}] and [{node.Name}] have the same IP address [{node.Metadata.PrivateAddress}].");
}
ipAddressToServer.Add(node.PrivateAddress, node);
}
//-----------------------------------------------------------------
// Perform basic node provisioning including operating system updates & configuration,
// and configure OpenVPN on the manager nodes so that hive setup will be
// able to reach the nodes on all ports.
// Write the operation begin marker to all hive node logs.
hive.LogLine(logBeginMarker);
var operation = $"Preparing [{hive.Definition.Name}] nodes";
var controller =
new SetupController <NodeDefinition>(operation, hive.Nodes)
{
ShowStatus = !Program.Quiet,
MaxParallel = Program.MaxParallel
};
if (!string.IsNullOrEmpty(packageCacheUri))
{
hive.Definition.PackageProxy = packageCacheUri;
}
// Prepare the nodes.
controller.AddWaitUntilOnlineStep(timeout: TimeSpan.FromMinutes(15));
hostingManager.AddPostProvisionSteps(controller);
controller.AddStep("verify OS",
(node, stepDelay) =>
{
Thread.Sleep(stepDelay);
CommonSteps.VerifyOS(node);
});
controller.AddStep("prepare",
(node, stepDelay) =>
{
Thread.Sleep(stepDelay);
CommonSteps.PrepareNode(node, hive.Definition, shutdown: false);
},
stepStaggerSeconds: hive.Definition.Setup.StepStaggerSeconds);
// Add any VPN configuration steps.
if (hive.Definition.Vpn.Enabled)
{
controller.AddGlobalStep("vpn credentials", () => CreateVpnCredentials());
controller.AddStep("vpn server",
(node, stepDelay) =>
{
Thread.Sleep(stepDelay);
ConfigManagerVpn(node);
},
node => node.Metadata.IsManager);
// Add a step to establish a VPN connection if we're provisioning to a cloud.
// We specifically don't want to do this if we're provisioning to a on-premise
// datacenter because we're assuming that we're already directly connected to
// the LAN while preparing and setting up the hive.
if (hive.Definition.Hosting.IsCloudProvider)
{
controller.AddStep("vpn connect",
(manager, stepDelay) =>
{
Thread.Sleep(stepDelay);
// Create a hive login with just enough credentials to connect the VPN.
// Note that this isn't really a node specific command but I wanted to
// be able to display the connection status somewhere.
var vpnLogin =
new HiveLogin()
{
Definition = hive.Definition,
VpnCredentials = vpnCredentials
};
// Ensure that we don't have an old VPN client for the hive running.
HiveHelper.VpnClose(vpnLogin.Definition.Name);
// ...and then start a new one.
HiveHelper.VpnOpen(vpnLogin,
onStatus: message => manager.Status = $"{message}",
onError: message => manager.Status = $"ERROR: {message}");
},
n => n == hive.FirstManager);
}
// Perform any post-VPN setup provisioning required by the hosting provider.
hostingManager.AddPostVpnSteps(controller);
}
if (!controller.Run())
{
// Write the operation end/failed marker to all hive node logs.
hive.LogLine(logFailedMarker);
Console.Error.WriteLine("*** ERROR: One or more configuration steps failed.");
Program.Exit(1);
}
// Write the hive login file.
var hiveLoginPath = Program.GetHiveLoginPath(HiveConst.RootUser, hive.Definition.Name);
var hiveLogin = new HiveLogin()
{
Path = hiveLoginPath,
Username = HiveConst.RootUser,
Definition = hive.Definition,
SshUsername = Program.MachineUsername,
SshPassword = Program.MachinePassword,
SshProvisionPassword = Program.MachinePassword,
SetupPending = true
};
if (hive.Definition.Vpn.Enabled)
{
hiveLogin.VpnCredentials = vpnCredentials;
}
// Generate the hive certificates.
const int bitCount = 2048;
const int validDays = 365000; // About 1,000 years.
if (hiveLogin.HiveCertificate == null)
{
var hostnames = new string[]
{
$"{hive.Name}.nhive.io",
$"*.{hive.Name}.nhive.io",
$"*.neon-vault.{hive.Name}.nhive.io",
$"*.neon-registry-cache.{hive.Name}.nhive.io",
$"*.neon-hivemq.{hive.Name}.nhive.io"
};
hiveLogin.HiveCertificate = TlsCertificate.CreateSelfSigned(hostnames, bitCount, validDays,
issuedBy: "neonHIVE",
issuedTo: $"neonHIVE: {hiveDefinition.Name}");
hiveLogin.HiveCertificate.FriendlyName = $"neonHIVE: {hiveLogin.Definition.Name}";
}
// Persist the certificates into the hive login.
hiveLogin.Save();
// Write the operation end marker to all hive node logs.
hive.LogLine(logEndMarker);
}
/// <summary>
/// Initializes the VPN certificate authority, and creates the OpenVPN server and
/// root client certificates.
/// </summary>
/// <param name="defPath">Path to the hive definition file.</param>
/// <param name="targetFolder">The output folder.</param>
private void InitializeCA(string defPath, string targetFolder)
{
DirectNotAllowed();
var hiveDefinition = HiveDefinition.FromFile(defPath, strict: true);
// This implements the steps described here:
//
// http://www.macfreek.nl/memory/Create_a_OpenVPN_Certificate_Authority
// Initialize
// ----------
Directory.CreateDirectory(caFolder);
// Initialize the file paths.
//
// IMPORTANT:
//
// Do not change these file names because the [VpnCaFiles] class
// depends on this naming convention.
var indexPath = Path.Combine(caFolder, "index.txt");
var caSignCnfPath = Path.Combine(caFolder, "ca-sign.cnf");
var caCnfPath = Path.Combine(caFolder, "ca.cnf");
var caKeyPath = Path.Combine(caFolder, "ca.key");
var caReqPath = Path.Combine(caFolder, "ca.req");
var caCrtPath = Path.Combine(caFolder, "ca.crt");
var dhParamPath = Path.Combine(caFolder, "dhparam.pem");
var serverCnfPath = Path.Combine(caFolder, "server.cnf");
var serverKeyPath = Path.Combine(caFolder, "server.key");
var serverReqPath = Path.Combine(caFolder, "server.req");
var serverCrtPath = Path.Combine(caFolder, "server.crt");
var rootCnfPath = Path.Combine(caFolder, $"{HiveConst.RootUser}.cnf");
var rootReqPath = Path.Combine(caFolder, $"{HiveConst.RootUser}.req");
var rootKeyPath = Path.Combine(caFolder, $"{HiveConst.RootUser}.key");
var rootCrtPath = Path.Combine(caFolder, $"{HiveConst.RootUser}.crt");
var taKeyPath = Path.Combine(caFolder, "ta.key");
var crlnumberPath = Path.Combine(caFolder, "crlnumber");
var crlPath = Path.Combine(caFolder, "crl.pem");
// Create an empty certificate index file.
File.WriteAllText(indexPath, string.Empty);
// CA Configuration Files
// ----------------------
// Create configuration files. In our setup, [ca-sign.cnf] contains the configuration for signing certificates.
// We only use it in conjunction with the [openssl ca command]. It described the folder structure within the [ca]
// directory, the location of support files for the CA, as well as properties of the signed certificates (duration,
// restricted usage) as well as the policy for the name ("distinguished name") of signed certificates. Finally,
// it lists the policy for certification revocation lists. For this small-scale CA, there is no public URL to
// download the CRL; I plan to distribute it manually.
var caSignCnf =
$@"# ca-sign.cnf
# This configuration file is used by the 'ca' command, to create signed certificates.
[ ca ]
default_ca = CA_default # The default ca section
[ CA_default ]
dir = {$"{this.caFolder}"} # Where everything is kept
certs = $dir/ # Where the issued certs are kept
crl_dir = $dir/ # Where the issued crl are kept
new_certs_dir = $dir/ # default place for new certs
private_key = $dir/ca.key # The private key
certificate = $dir/ca.crt # The CA root certificate
database = $dir/index.txt # List of signed certificates
serial = $dir/serial # The current serial number
crlnumber = $dir/crlnumber # the current crl number
crl = $dir/crl.pem # The current CRL
RANDFILE = $dir/.rand # private random number file
unique_subject = no # allow multiple certificates with same subject.
default_md = sha256 # Use hash algorithm specified in the request
default_days = 365000 # client certificates last about 1000 years
default_crl_days = 30 # How often clients should download the CRL
#x509_extensions = X509_ca # The x509 extensions for the root certificate
#x509_extensions = X509_server # The x509 extensions for a server certificate
x509_extensions = X509_client # The x509 extensions for a client certificate
# These options control what fields from the distinguished name to show before signing.
# They are required to make sure all fields are shown.
name_opt = ca_default # Subject Name options
cert_opt = ca_default # Certificate field options
copy_extensions = copy # Accept requested extensions
policy = policy_dn
[ X509_ca ]
# X509v3 extensions for the root certificate
basicConstraints = CA:TRUE
nsCertType = sslCA # restrict the usage
keyUsage = keyCertSign, cRLSign # restrict the usage
subjectKeyIdentifier = hash
authorityKeyIdentifier = keyid:always,issuer:always
#subjectAltName = email:move # Move email address from DN to extensions
#crlDistributionPoints = URI:http://www.example.com/example_ca.crl
[ X509_server ]
# X509v3 extensions for server certificates
basicConstraints = CA:FALSE
nsCertType = server # restrict the usage
keyUsage = digitalSignature, keyEncipherment
extendedKeyUsage = serverAuth # restrict the usage
subjectKeyIdentifier = hash
authorityKeyIdentifier = keyid,issuer
#subjectAltName = email:move # Move email address from DN to extensions
#crlDistributionPoints = URI:http://www.example.com/example_ca.crl
[ X509_client ]
# X509v3 extensions for client certificates
basicConstraints = CA:FALSE
nsCertType = client # restrict the usage
keyUsage = digitalSignature # restrict the usage
extendedKeyUsage = clientAuth # restrict the usage
subjectKeyIdentifier = hash
authorityKeyIdentifier = keyid,issuer
#subjectAltName = email:move # Move email address from DN to extensions
#crlDistributionPoints = URI:http://www.example.com/example_ca.crl
[ policy_dn ]
countryName = supplied # required parameter, any value allowed
stateOrProvinceName = optional
localityName = optional
organizationName = match # required, and must match root certificate
organizationalUnitName = optional
commonName = supplied # required parameter, any value allowed
emailAddress = optional # email in DN is deprecated, use subjectAltName
";
File.WriteAllText(caSignCnfPath, caSignCnf);
// The x509_extensions sections are not really required by openssl or openvpn, but
// adds extra security by telling OpenVPN that clients may connect to servers
// only. nsCertType is required for the OpenVPN option ns-cert-type server|client;
// keyUsage and extendedKeyUsage are required for remote-cert-tls server|client.
// [ca.cnf] defines the distinguished name for the certificate authority. It also
// contains the key length (2048 is recommended nowadays, over the default of 1024),
// and if the key should be encrypted.
var caCnf =
$@"# ca.cnf
# This configuration file is used by the 'req' command when the root certificates is created.
[ req ]
default_bits = 2048 # default strength of client certificates
default_md = sha256
encrypt_key = no # ""no"" is equivalent to -nodes
prompt = no
string_mask = utf8only
distinguished_name = ca_distinguished_name # root certificate name
req_extensions = req_cert_extensions
#attributes = req_attributes
[ ca_distinguished_name ]
# root certificate name
countryName = {hiveDefinition.Vpn.CertCountryCode}
#stateOrProvinceName = Utrecht
#localityName = Hometown
organizationName = {hiveDefinition.Vpn.CertOrganization}
#organizationalUnitName = My Department Name
commonName = ca
#emailAddress = [email protected] # email in DN is deprecated, use subjectAltName
[ req_cert_extensions ]
nsCertType = server
#subjectAltName = email:[email protected]
";
File.WriteAllText(caCnfPath, caCnf);
// Note that in the above examples, the email address is specified in the [subjectAltName],
// instead of in the distinguished name. This is in accordance with PKIX standards.
// Build CA certificate
// --------------------
// If your CA should be valid after the year 2038, be sure to use openssl 0.9.9 or higher.
//
// First create a request with the correct name, and then self-sign a certificate and create
// a serial number file.
Program.Execute("openssl", "req", "-new",
"-config", caCnfPath,
"-keyout", caKeyPath,
"-out", caReqPath);
Program.Execute("openssl", "ca", "-batch",
"-config", caSignCnfPath,
"-extensions", "X509_ca",
"-days", 365000,
"-create_serial",
"-selfsign",
"-keyfile", caKeyPath,
"-in", caReqPath,
"-out", caCrtPath);
// Generate Prime Numbers (the Diffie Hellman parameters)
// ------------------------------------------------------
Program.Execute("openssl", "dhparam", "-out", dhParamPath, "2048");
// Build server certificate
// ------------------------
// First, create a configuration for the server, similar to [ca.cnf]:
var serverCnf =
$@"# server.cnf
# This configuration file is used by the 'req' command when the server certificate is created.
[ req ]
default_bits = 2048
default_md = sha256
encrypt_key = no
prompt = no
string_mask = utf8only
distinguished_name = server_distinguished_name
req_extensions = req_cert_extensions
#attributes = req_attributes
[ server_distinguished_name ]
countryName = {hiveDefinition.Vpn.CertCountryCode}
#stateOrProvinceName =
#localityName =
organizationName = {hiveDefinition.Vpn.CertOrganization}
#organizationalUnitName = My Department Name
commonName = server
#emailAddress =
[ req_cert_extensions ]
nsCertType = server
#subjectAltName = email:[email protected]
";
File.WriteAllText(serverCnfPath, serverCnf);
// Create the server request and private key.
Program.Execute("openssl", "req", "-new",
"-config", serverCnfPath,
"-keyout", serverKeyPath,
"-out", serverReqPath);
// Create the server certificate.
Program.Execute("openssl", "ca", "-batch",
"-config", caSignCnfPath,
"-extensions", "X509_server",
"-in", serverReqPath,
"-out", serverCrtPath);
// Build the [root] client certificate.
try
{
File.WriteAllText(rootCnfPath, GetClientConfig(hiveDefinition, HiveConst.RootUser, rootPrivileges: true));
Program.Execute("openssl", "req", "-new",
"-config", rootCnfPath,
"-keyout", rootKeyPath,
"-out", rootReqPath);
Program.Execute("openssl", "ca", "-batch",
"-config", caSignCnfPath,
"-out", rootCrtPath,
"-in", rootReqPath);
}
finally
{
if (File.Exists(rootCnfPath))
{
File.Delete(rootCnfPath);
}
}
// Initialize the Certificate Revocation List (CLR) number file
// and then generate the initial (empty) CRL.
File.WriteAllText(crlnumberPath, "00");
Program.Execute("openssl", "ca",
"-config", caSignCnfPath,
"-gencrl",
"-out", crlPath);
// As one final additional step, we're going to generate a shared
// key that OpenVPN can use to quickly reject packets that didn't
// come from a client with the key. This provides a decent amount
// of DOS protection, especially for VPNs that only use the UDP
// transport.
Program.Execute("openvpn", "--genkey", "--secret", taKeyPath);
// Copy all of the CA files to the target folder.
Directory.CreateDirectory(targetFolder);
foreach (var file in Directory.GetFiles(caFolder, "*.*", SearchOption.TopDirectoryOnly))
{
File.Copy(file, Path.Combine(targetFolder, Path.GetFileName(file)));
}
}