Beispiel #1
0
        /// <summary>
        /// Creates a Hyper-V virtual machine for a hive node.
        /// </summary>
        /// <param name="node">The target node.</param>
        private void ProvisionVM(SshProxy <NodeDefinition> node)
        {
            // $todo(jeff.lill):
            //
            // This code currently assumes that the VM will use DHCP to obtain
            // its initial network configuration so the code can SSH into the
            // node to configure a static IP.
            //
            // It appears that it is possible to inject an IP address, but
            // I wasn't able to get this to work (perhaps Windows Server is
            // required.  Here's a link discussing this:
            //
            //  http://www.itprotoday.com/virtualization/modify-ip-configuration-vm-hyper-v-host
            //
            // An alternative technique might be to update [/etc/network/interfaces]
            // remotely via PowerShell as described here:
            //
            //  https://www.altaro.com/hyper-v/transfer-files-linux-hyper-v-guest/

            using (var hyperv = new HyperVClient())
            {
                var vmName = GetVmName(node.Metadata);

                // Extract the template file contents to the virtual machine's
                // virtual hard drive file.

                var drivePath = Path.Combine(vmDriveFolder, $"{vmName}-[0].vhdx");

                using (var zip = new ZipFile(driveTemplatePath))
                {
                    if (zip.Count != 1)
                    {
                        throw new ArgumentException($"[{driveTemplatePath}] ZIP archive includes more than one file.");
                    }

                    ZipEntry entry = null;

                    foreach (ZipEntry item in zip)
                    {
                        entry = item;
                        break;
                    }

                    if (!entry.IsFile)
                    {
                        throw new ArgumentException($"[{driveTemplatePath}] ZIP archive includes entry [{entry.Name}] that is not a file.");
                    }

                    if (!entry.Name.EndsWith(".vhdx", StringComparison.InvariantCultureIgnoreCase))
                    {
                        throw new ArgumentException($"[{driveTemplatePath}] ZIP archive includes a file that's not named like [*.vhdx].");
                    }

                    node.Status = $"create disk";

                    // $hack(jeff.lill): Update console at 2 sec intervals to avoid annoying flicker

                    var updateInterval = TimeSpan.FromSeconds(2);
                    var stopwatch      = new Stopwatch();

                    stopwatch.Start();

                    using (var input = zip.GetInputStream(entry))
                    {
                        using (var output = new FileStream(drivePath, FileMode.Create, FileAccess.ReadWrite))
                        {
                            var buffer = new byte[64 * 1024];
                            int cb;

                            while (true)
                            {
                                cb = input.Read(buffer, 0, buffer.Length);

                                if (cb == 0)
                                {
                                    break;
                                }

                                output.Write(buffer, 0, cb);

                                var percentComplete = (int)(((double)output.Length / (double)entry.Size) * 100.0);

                                if (stopwatch.Elapsed >= updateInterval || percentComplete >= 100.0)
                                {
                                    node.Status = $"[{percentComplete}%] create disk";
                                    stopwatch.Restart();
                                }
                            }
                        }
                    }
                }

                // Stop and delete the virtual machine if one exists.

                if (hyperv.VMExists(vmName))
                {
                    hyperv.StopVM(vmName);
                    hyperv.RemoveVM(vmName);
                }

                // We need to create a raw drive if the node hosts a Ceph OSD.

                var extraDrives = new List <VirtualDrive>();

                if (node.Metadata.Labels.CephOSD)
                {
                    extraDrives.Add(
                        new VirtualDrive()
                    {
                        IsDynamic = true,
                        Size      = node.Metadata.GetCephOSDDriveSize(hive.Definition),
                        Path      = Path.Combine(vmDriveFolder, $"{vmName}-[1].vhdx")
                    });
                }

                // Create the virtual machine if it doesn't already exist.

                var processors     = node.Metadata.GetVmProcessors(hive.Definition);
                var memoryBytes    = node.Metadata.GetVmMemory(hive.Definition);
                var minMemoryBytes = node.Metadata.GetVmMinimumMemory(hive.Definition);
                var diskBytes      = node.Metadata.GetVmDisk(hive.Definition);

                node.Status = $"create virtual machine";
                hyperv.AddVM(
                    vmName,
                    processorCount: processors,
                    diskSize: diskBytes.ToString(),
                    memorySize: memoryBytes.ToString(),
                    minimumMemorySize: minMemoryBytes.ToString(),
                    drivePath: drivePath,
                    switchName: switchName,
                    extraDrives: extraDrives);

                node.Status = $"start virtual machine";

                hyperv.StartVM(vmName);

                // Retrieve the virtual machine's network adapters (there should only be one)
                // to obtain the IP address we'll use to SSH into the machine and configure
                // it's static IP.

                node.Status = $"fetch ip address";

                var adapters  = hyperv.ListVMNetworkAdapters(vmName, waitForAddresses: true);
                var adapter   = adapters.FirstOrDefault();
                var address   = adapter.Addresses.First();
                var subnet    = NetworkCidr.Parse(hive.Definition.Network.PremiseSubnet);
                var gateway   = hive.Definition.Network.Gateway;
                var broadcast = hive.Definition.Network.Broadcast;

                if (adapter == null)
                {
                    throw new HyperVException($"Virtual machine [{vmName}] has no network adapters.");
                }

                // We're going to temporarily set the node to the current VM address
                // so we can connect via SSH.

                var savedNodeAddress = node.PrivateAddress;

                try
                {
                    node.PrivateAddress = address;

                    using (var nodeProxy = hive.GetNode(node.Name))
                    {
                        node.Status = $"connecting";
                        nodeProxy.WaitForBoot();

                        // We need to ensure that the host folders exist.

                        nodeProxy.CreateHiveHostFolders();

                        // Replace the [/etc/network/interfaces] file to configure the static
                        // IP and then reboot to reinitialize networking subsystem.

                        var primaryInterface = node.GetNetworkInterface(address);

                        node.Status = $"set static ip [{savedNodeAddress}]";

                        var interfacesText =
                            $@"# This file describes the network interfaces available on your system
# and how to activate them. For more information, see interfaces(5).

source /etc/network/interfaces.d/*

# The loopback network interface
auto lo
iface lo inet loopback

# The primary network interface
auto {primaryInterface}
iface {primaryInterface} inet static
address {savedNodeAddress}
netmask {subnet.Mask}
gateway {gateway}
broadcast {broadcast}
";
                        nodeProxy.UploadText("/etc/network/interfaces", interfacesText);

                        // Temporarily configure the public Google DNS servers as
                        // the name servers so DNS will work after we reboot with
                        // the static IP.  Note that hive setup will eventually
                        // configure the name servers specified in the hive
                        // definition.

                        // $todo(jeff.lill):
                        //
                        // Is there a good reason why we're not just configuring the
                        // DNS servers from the hive definition here???
                        //
                        // Using the Google DNS seems like it could break some hive
                        // network configurations (e.g. for hives that don't have
                        // access to the public Internet).  Totally private hives
                        // aren't really a supported scenario right now though because
                        // we assume we can use [apt-get]... to pull down packages.

                        var resolvBaseText =
                            $@"nameserver 8.8.8.8
nameserver 8.8.4.4
";
                        nodeProxy.UploadText("/etc/resolvconf/resolv.conf.d/base", resolvBaseText);

                        // Extend the primary partition and file system to fill
                        // the virtual the drive.

                        node.Status = $"resize primary partition";

                        // $hack(jeff.lill):
                        //
                        // I've seen a transient error here but can't reproduce it.  I'm going
                        // to assume for now that the file system might not be quite ready for
                        // this operation directly after the VM has been rebooted, so we're going
                        // to delay for a few seconds before performing the operations.

                        Thread.Sleep(TimeSpan.FromSeconds(5));
                        nodeProxy.SudoCommand("growpart /dev/sda 1");
                        nodeProxy.SudoCommand("resize2fs /dev/sda1");

                        // Reboot to pick up the changes.

                        node.Status = $"rebooting";
                        nodeProxy.Reboot(wait: false);
                    }
                }
                finally
                {
                    // Restore the node's IP address.

                    node.PrivateAddress = savedNodeAddress;
                }
            }
        }
        /// <summary>
        /// Creates a Hyper-V virtual machine for a cluster node.
        /// </summary>
        /// <param name="node">The target node.</param>
        private void ProvisionVM(SshProxy <NodeDefinition> node)
        {
            // $todo(jeff.lill):
            //
            // This code currently assumes that the VM will use DHCP to obtain
            // its initial network configuration so the code can SSH into the
            // node to configure a static IP.
            //
            // It appears that it is possible to inject an IP address, but
            // I wasn't able to get this to work (perhaps Windows Server is
            // required).  Here's a link discussing this:
            //
            //  http://www.itprotoday.com/virtualization/modify-ip-configuration-vm-hyper-v-host
            //
            // An alternative technique might be to update [/etc/network/interfaces]
            // remotely via PowerShell as described here:
            //
            //  https://www.altaro.com/hyper-v/transfer-files-linux-hyper-v-guest/

            using (var hyperv = new HyperVClient())
            {
                var vmName = GetVmName(node.Metadata);

                // Copy the VHDX template file to the virtual machine's
                // virtual hard drive file.

                var driveTemplateInfoPath = driveTemplatePath + ".info";
                var driveTemplateInfo     = NeonHelper.JsonDeserialize <DriveTemplateInfo>(File.ReadAllText(driveTemplateInfoPath));
                var drivePath             = Path.Combine(vmDriveFolder, $"{vmName}.vhdx");

                node.Status = $"create: disk";

                // $hack(jeff.lill): Update console at 2 sec intervals to mitigate annoying flicker

                var updateInterval = TimeSpan.FromSeconds(2);
                var stopwatch      = new Stopwatch();

                stopwatch.Start();

                using (var input = new FileStream(driveTemplatePath, FileMode.Open, FileAccess.Read))
                {
                    if (driveTemplateInfo.Compressed)
                    {
                        using (var output = new FileStream(drivePath, FileMode.Create, FileAccess.ReadWrite))
                        {
                            using (var decompressor = new GZipStream(input, CompressionMode.Decompress))
                            {
                                var  buffer = new byte[64 * 1024];
                                long cbRead = 0;
                                int  cb;

                                while (true)
                                {
                                    cb = decompressor.Read(buffer, 0, buffer.Length);

                                    if (cb == 0)
                                    {
                                        break;
                                    }

                                    output.Write(buffer, 0, cb);

                                    cbRead += cb;

                                    var percentComplete = (int)(((double)output.Length / (double)cbRead) * 100.0);

                                    if (stopwatch.Elapsed >= updateInterval || percentComplete >= 100.0)
                                    {
                                        node.Status = $"create: disk [{percentComplete}%]";
                                        stopwatch.Restart();
                                    }
                                }
                            }
                        }
                    }
                    else
                    {
                        using (var output = new FileStream(drivePath, FileMode.Create, FileAccess.ReadWrite))
                        {
                            var buffer = new byte[64 * 1024];
                            int cb;

                            while (true)
                            {
                                cb = input.Read(buffer, 0, buffer.Length);

                                if (cb == 0)
                                {
                                    break;
                                }

                                output.Write(buffer, 0, cb);

                                var percentComplete = (int)(((double)output.Length / (double)input.Length) * 100.0);

                                if (stopwatch.Elapsed >= updateInterval || percentComplete >= 100.0)
                                {
                                    node.Status = $"create: disk [{percentComplete}%]";
                                    stopwatch.Restart();
                                }
                            }
                        }
                    }
                }

                // Stop and delete the virtual machine if one exists.

                if (hyperv.VMExists(vmName))
                {
                    hyperv.StopVM(vmName);
                    hyperv.RemoveVM(vmName);
                }

                // Create the virtual machine if it doesn't already exist.

                // We need to create a raw drive if the node hosts a Ceph OSD.

                var extraDrives = new List <VirtualDrive>();

                if (node.Metadata.Labels.CephOSD)
                {
                    extraDrives.Add(
                        new VirtualDrive()
                    {
                        IsDynamic = true,
                        Size      = node.Metadata.GetCephOSDDriveSize(cluster.Definition),
                        Path      = Path.Combine(vmDriveFolder, $"{vmName}-[1].vhdx")
                    });
                }

                var processors  = node.Metadata.GetVmProcessors(cluster.Definition);
                var memoryBytes = node.Metadata.GetVmMemory(cluster.Definition);
                var diskBytes   = node.Metadata.GetVmDisk(cluster.Definition);

                node.Status = $"create: virtual machine";
                hyperv.AddVM(
                    vmName,
                    processorCount: processors,
                    diskSize:       diskBytes.ToString(),
                    memorySize:     memoryBytes.ToString(),
                    drivePath:      drivePath,
                    switchName:     switchName,
                    extraDrives:    extraDrives);

                node.Status = $"start: virtual machine";

                hyperv.StartVM(vmName);

                // Retrieve the virtual machine's network adapters (there should only be one)
                // to obtain the IP address we'll use to SSH into the machine and configure
                // it's static IP.

                node.Status = $"discover: ip address";

                var adapters = hyperv.ListVMNetworkAdapters(vmName, waitForAddresses: true);
                var adapter  = adapters.FirstOrDefault();
                var address  = adapter.Addresses.First();

                if (adapter == null)
                {
                    throw new HyperVException($"Virtual machine [{vmName}] has no network adapters.");
                }

                // We're going to temporarily set the node to the current VM address
                // so we can connect via SSH.

                var nodePrivateAddress = node.PrivateAddress;

                try
                {
                    node.PrivateAddress = address;

                    using (var nodeProxy = cluster.GetNode(node.Name))
                    {
                        node.Status = $"connecting...";
                        nodeProxy.WaitForBoot();

                        // We need to ensure that the host folders exist.

                        nodeProxy.CreateHostFolders();

                        // Configure the node's network stack to the static IP address
                        // and upstream nameservers.

                        node.Status = $"config: network [IP={node.PrivateAddress}]";

                        var primaryInterface = node.GetNetworkInterface(address);

                        node.ConfigureNetwork(
                            networkInterface:   primaryInterface,
                            address:            nodePrivateAddress,
                            gateway:            IPAddress.Parse(cluster.Definition.Network.Gateway),
                            subnet:             NetworkCidr.Parse(cluster.Definition.Network.PremiseSubnet),
                            nameservers:        cluster.Definition.Network.Nameservers.Select(ns => IPAddress.Parse(ns)));

                        // Extend the primary partition and file system to fill
                        // the virtual the drive.

                        node.Status = $"resize: primary drive";

                        // $hack(jeff.lill):
                        //
                        // I've seen a transient error here but can't reproduce it.  I'm going
                        // to assume for now that the file system might not be quite ready for
                        // this operation directly after the VM has been rebooted, so we're going
                        // to delay for a few seconds before performing the operations.

                        Thread.Sleep(TimeSpan.FromSeconds(5));
                        nodeProxy.SudoCommand("growpart /dev/sda 2");
                        nodeProxy.SudoCommand("resize2fs /dev/sda2");

                        // Reboot to pick up the changes.

                        node.Status = $"restarting...";
                        nodeProxy.Reboot(wait: false);
                    }
                }
                finally
                {
                    // Restore the node's IP address.

                    node.PrivateAddress = nodePrivateAddress;
                }
            }
        }