/// <summary>
/// Creates a Hyper-V virtual machine for a hive node.
/// </summary>
/// <param name="node">The target node.</param>
private void ProvisionVM(SshProxy <NodeDefinition> node)
{
// $todo(jeff.lill):
//
// This code currently assumes that the VM will use DHCP to obtain
// its initial network configuration so the code can SSH into the
// node to configure a static IP.
//
// It appears that it is possible to inject an IP address, but
// I wasn't able to get this to work (perhaps Windows Server is
// required. Here's a link discussing this:
//
// http://www.itprotoday.com/virtualization/modify-ip-configuration-vm-hyper-v-host
//
// An alternative technique might be to update [/etc/network/interfaces]
// remotely via PowerShell as described here:
//
// https://www.altaro.com/hyper-v/transfer-files-linux-hyper-v-guest/
using (var hyperv = new HyperVClient())
{
var vmName = GetVmName(node.Metadata);
// Extract the template file contents to the virtual machine's
// virtual hard drive file.
var drivePath = Path.Combine(vmDriveFolder, $"{vmName}-[0].vhdx");
using (var zip = new ZipFile(driveTemplatePath))
{
if (zip.Count != 1)
{
throw new ArgumentException($"[{driveTemplatePath}] ZIP archive includes more than one file.");
}
ZipEntry entry = null;
foreach (ZipEntry item in zip)
{
entry = item;
break;
}
if (!entry.IsFile)
{
throw new ArgumentException($"[{driveTemplatePath}] ZIP archive includes entry [{entry.Name}] that is not a file.");
}
if (!entry.Name.EndsWith(".vhdx", StringComparison.InvariantCultureIgnoreCase))
{
throw new ArgumentException($"[{driveTemplatePath}] ZIP archive includes a file that's not named like [*.vhdx].");
}
node.Status = $"create disk";
// $hack(jeff.lill): Update console at 2 sec intervals to avoid annoying flicker
var updateInterval = TimeSpan.FromSeconds(2);
var stopwatch = new Stopwatch();
stopwatch.Start();
using (var input = zip.GetInputStream(entry))
{
using (var output = new FileStream(drivePath, FileMode.Create, FileAccess.ReadWrite))
{
var buffer = new byte[64 * 1024];
int cb;
while (true)
{
cb = input.Read(buffer, 0, buffer.Length);
if (cb == 0)
{
break;
}
output.Write(buffer, 0, cb);
var percentComplete = (int)(((double)output.Length / (double)entry.Size) * 100.0);
if (stopwatch.Elapsed >= updateInterval || percentComplete >= 100.0)
{
node.Status = $"[{percentComplete}%] create disk";
stopwatch.Restart();
}
}
}
}
}
// Stop and delete the virtual machine if one exists.
if (hyperv.VMExists(vmName))
{
hyperv.StopVM(vmName);
hyperv.RemoveVM(vmName);
}
// We need to create a raw drive if the node hosts a Ceph OSD.
var extraDrives = new List <VirtualDrive>();
if (node.Metadata.Labels.CephOSD)
{
extraDrives.Add(
new VirtualDrive()
{
IsDynamic = true,
Size = node.Metadata.GetCephOSDDriveSize(hive.Definition),
Path = Path.Combine(vmDriveFolder, $"{vmName}-[1].vhdx")
});
}
// Create the virtual machine if it doesn't already exist.
var processors = node.Metadata.GetVmProcessors(hive.Definition);
var memoryBytes = node.Metadata.GetVmMemory(hive.Definition);
var minMemoryBytes = node.Metadata.GetVmMinimumMemory(hive.Definition);
var diskBytes = node.Metadata.GetVmDisk(hive.Definition);
node.Status = $"create virtual machine";
hyperv.AddVM(
vmName,
processorCount: processors,
diskSize: diskBytes.ToString(),
memorySize: memoryBytes.ToString(),
minimumMemorySize: minMemoryBytes.ToString(),
drivePath: drivePath,
switchName: switchName,
extraDrives: extraDrives);
node.Status = $"start virtual machine";
hyperv.StartVM(vmName);
// Retrieve the virtual machine's network adapters (there should only be one)
// to obtain the IP address we'll use to SSH into the machine and configure
// it's static IP.
node.Status = $"fetch ip address";
var adapters = hyperv.ListVMNetworkAdapters(vmName, waitForAddresses: true);
var adapter = adapters.FirstOrDefault();
var address = adapter.Addresses.First();
var subnet = NetworkCidr.Parse(hive.Definition.Network.PremiseSubnet);
var gateway = hive.Definition.Network.Gateway;
var broadcast = hive.Definition.Network.Broadcast;
if (adapter == null)
{
throw new HyperVException($"Virtual machine [{vmName}] has no network adapters.");
}
// We're going to temporarily set the node to the current VM address
// so we can connect via SSH.
var savedNodeAddress = node.PrivateAddress;
try
{
node.PrivateAddress = address;
using (var nodeProxy = hive.GetNode(node.Name))
{
node.Status = $"connecting";
nodeProxy.WaitForBoot();
// We need to ensure that the host folders exist.
nodeProxy.CreateHiveHostFolders();
// Replace the [/etc/network/interfaces] file to configure the static
// IP and then reboot to reinitialize networking subsystem.
var primaryInterface = node.GetNetworkInterface(address);
node.Status = $"set static ip [{savedNodeAddress}]";
var interfacesText =
$@"# This file describes the network interfaces available on your system
# and how to activate them. For more information, see interfaces(5).
source /etc/network/interfaces.d/*
# The loopback network interface
auto lo
iface lo inet loopback
# The primary network interface
auto {primaryInterface}
iface {primaryInterface} inet static
address {savedNodeAddress}
netmask {subnet.Mask}
gateway {gateway}
broadcast {broadcast}
";
nodeProxy.UploadText("/etc/network/interfaces", interfacesText);
// Temporarily configure the public Google DNS servers as
// the name servers so DNS will work after we reboot with
// the static IP. Note that hive setup will eventually
// configure the name servers specified in the hive
// definition.
// $todo(jeff.lill):
//
// Is there a good reason why we're not just configuring the
// DNS servers from the hive definition here???
//
// Using the Google DNS seems like it could break some hive
// network configurations (e.g. for hives that don't have
// access to the public Internet). Totally private hives
// aren't really a supported scenario right now though because
// we assume we can use [apt-get]... to pull down packages.
var resolvBaseText =
$@"nameserver 8.8.8.8
nameserver 8.8.4.4
";
nodeProxy.UploadText("/etc/resolvconf/resolv.conf.d/base", resolvBaseText);
// Extend the primary partition and file system to fill
// the virtual the drive.
node.Status = $"resize primary partition";
// $hack(jeff.lill):
//
// I've seen a transient error here but can't reproduce it. I'm going
// to assume for now that the file system might not be quite ready for
// this operation directly after the VM has been rebooted, so we're going
// to delay for a few seconds before performing the operations.
Thread.Sleep(TimeSpan.FromSeconds(5));
nodeProxy.SudoCommand("growpart /dev/sda 1");
nodeProxy.SudoCommand("resize2fs /dev/sda1");
// Reboot to pick up the changes.
node.Status = $"rebooting";
nodeProxy.Reboot(wait: false);
}
}
finally
{
// Restore the node's IP address.
node.PrivateAddress = savedNodeAddress;
}
}
}
/// <summary>
/// Creates a Hyper-V virtual machine for a cluster node.
/// </summary>
/// <param name="node">The target node.</param>
private void ProvisionVM(SshProxy <NodeDefinition> node)
{
// $todo(jeff.lill):
//
// This code currently assumes that the VM will use DHCP to obtain
// its initial network configuration so the code can SSH into the
// node to configure a static IP.
//
// It appears that it is possible to inject an IP address, but
// I wasn't able to get this to work (perhaps Windows Server is
// required). Here's a link discussing this:
//
// http://www.itprotoday.com/virtualization/modify-ip-configuration-vm-hyper-v-host
//
// An alternative technique might be to update [/etc/network/interfaces]
// remotely via PowerShell as described here:
//
// https://www.altaro.com/hyper-v/transfer-files-linux-hyper-v-guest/
using (var hyperv = new HyperVClient())
{
var vmName = GetVmName(node.Metadata);
// Copy the VHDX template file to the virtual machine's
// virtual hard drive file.
var driveTemplateInfoPath = driveTemplatePath + ".info";
var driveTemplateInfo = NeonHelper.JsonDeserialize <DriveTemplateInfo>(File.ReadAllText(driveTemplateInfoPath));
var drivePath = Path.Combine(vmDriveFolder, $"{vmName}.vhdx");
node.Status = $"create: disk";
// $hack(jeff.lill): Update console at 2 sec intervals to mitigate annoying flicker
var updateInterval = TimeSpan.FromSeconds(2);
var stopwatch = new Stopwatch();
stopwatch.Start();
using (var input = new FileStream(driveTemplatePath, FileMode.Open, FileAccess.Read))
{
if (driveTemplateInfo.Compressed)
{
using (var output = new FileStream(drivePath, FileMode.Create, FileAccess.ReadWrite))
{
using (var decompressor = new GZipStream(input, CompressionMode.Decompress))
{
var buffer = new byte[64 * 1024];
long cbRead = 0;
int cb;
while (true)
{
cb = decompressor.Read(buffer, 0, buffer.Length);
if (cb == 0)
{
break;
}
output.Write(buffer, 0, cb);
cbRead += cb;
var percentComplete = (int)(((double)output.Length / (double)cbRead) * 100.0);
if (stopwatch.Elapsed >= updateInterval || percentComplete >= 100.0)
{
node.Status = $"create: disk [{percentComplete}%]";
stopwatch.Restart();
}
}
}
}
}
else
{
using (var output = new FileStream(drivePath, FileMode.Create, FileAccess.ReadWrite))
{
var buffer = new byte[64 * 1024];
int cb;
while (true)
{
cb = input.Read(buffer, 0, buffer.Length);
if (cb == 0)
{
break;
}
output.Write(buffer, 0, cb);
var percentComplete = (int)(((double)output.Length / (double)input.Length) * 100.0);
if (stopwatch.Elapsed >= updateInterval || percentComplete >= 100.0)
{
node.Status = $"create: disk [{percentComplete}%]";
stopwatch.Restart();
}
}
}
}
}
// Stop and delete the virtual machine if one exists.
if (hyperv.VMExists(vmName))
{
hyperv.StopVM(vmName);
hyperv.RemoveVM(vmName);
}
// Create the virtual machine if it doesn't already exist.
// We need to create a raw drive if the node hosts a Ceph OSD.
var extraDrives = new List <VirtualDrive>();
if (node.Metadata.Labels.CephOSD)
{
extraDrives.Add(
new VirtualDrive()
{
IsDynamic = true,
Size = node.Metadata.GetCephOSDDriveSize(cluster.Definition),
Path = Path.Combine(vmDriveFolder, $"{vmName}-[1].vhdx")
});
}
var processors = node.Metadata.GetVmProcessors(cluster.Definition);
var memoryBytes = node.Metadata.GetVmMemory(cluster.Definition);
var diskBytes = node.Metadata.GetVmDisk(cluster.Definition);
node.Status = $"create: virtual machine";
hyperv.AddVM(
vmName,
processorCount: processors,
diskSize: diskBytes.ToString(),
memorySize: memoryBytes.ToString(),
drivePath: drivePath,
switchName: switchName,
extraDrives: extraDrives);
node.Status = $"start: virtual machine";
hyperv.StartVM(vmName);
// Retrieve the virtual machine's network adapters (there should only be one)
// to obtain the IP address we'll use to SSH into the machine and configure
// it's static IP.
node.Status = $"discover: ip address";
var adapters = hyperv.ListVMNetworkAdapters(vmName, waitForAddresses: true);
var adapter = adapters.FirstOrDefault();
var address = adapter.Addresses.First();
if (adapter == null)
{
throw new HyperVException($"Virtual machine [{vmName}] has no network adapters.");
}
// We're going to temporarily set the node to the current VM address
// so we can connect via SSH.
var nodePrivateAddress = node.PrivateAddress;
try
{
node.PrivateAddress = address;
using (var nodeProxy = cluster.GetNode(node.Name))
{
node.Status = $"connecting...";
nodeProxy.WaitForBoot();
// We need to ensure that the host folders exist.
nodeProxy.CreateHostFolders();
// Configure the node's network stack to the static IP address
// and upstream nameservers.
node.Status = $"config: network [IP={node.PrivateAddress}]";
var primaryInterface = node.GetNetworkInterface(address);
node.ConfigureNetwork(
networkInterface: primaryInterface,
address: nodePrivateAddress,
gateway: IPAddress.Parse(cluster.Definition.Network.Gateway),
subnet: NetworkCidr.Parse(cluster.Definition.Network.PremiseSubnet),
nameservers: cluster.Definition.Network.Nameservers.Select(ns => IPAddress.Parse(ns)));
// Extend the primary partition and file system to fill
// the virtual the drive.
node.Status = $"resize: primary drive";
// $hack(jeff.lill):
//
// I've seen a transient error here but can't reproduce it. I'm going
// to assume for now that the file system might not be quite ready for
// this operation directly after the VM has been rebooted, so we're going
// to delay for a few seconds before performing the operations.
Thread.Sleep(TimeSpan.FromSeconds(5));
nodeProxy.SudoCommand("growpart /dev/sda 2");
nodeProxy.SudoCommand("resize2fs /dev/sda2");
// Reboot to pick up the changes.
node.Status = $"restarting...";
nodeProxy.Reboot(wait: false);
}
}
finally
{
// Restore the node's IP address.
node.PrivateAddress = nodePrivateAddress;
}
}
}