/// <inheritdoc/>
public override void Run(HiveProxy hive)
{
Covenant.Requires <ArgumentNullException>(hive != null);
var node = hive.GetNode(nodeName);
node.SudoCommand("docker-volume-rm", volumeName);
}
/// <summary>
/// Signals the Docker orchestrator to drain all service tasks from a node.
/// </summary>
/// <param name="nodeName">Identifies the target node.</param>
/// <exception cref="KeyNotFoundException">Thrown if the named node does not exist.</exception>
/// <exception cref="InvalidOperationException">Thrown if the node is not part of the swarm.</exception>
public void DrainNode(string nodeName)
{
Covenant.Requires <ArgumentNullException>(!string.IsNullOrEmpty(nodeName));
var node = hive.GetNode(nodeName);
if (!node.Metadata.InSwarm)
{
throw new InvalidOperationException($"Node [{nodeName}] is not part of the swarm.");
}
// I've see transient errors, so we'll retry a few times.
var manager = hive.GetReachableManager();
var retry = new LinearRetryPolicy(typeof(Exception), maxAttempts: 5, retryInterval: TimeSpan.FromSeconds(5));
retry.InvokeAsync(
async() =>
{
var response = manager.SudoCommand($"docker node update --availability drain {nodeName}");
if (response.ExitCode != 0)
{
throw new Exception(response.ErrorSummary);
}
await Task.CompletedTask;
}).Wait();
// $todo(jeff.lill):
//
// Ideally, we'd wait for all of the service tasks to stop but it
// appears that there's no easy way to check for this other than
// listing all of the hive services and then doing a
//
// docker service ps SERVICE]
//
// for each until none report running on this node.
//
// A hacky alternative would be to list local containers and try
// to determine which ones look liks service tasks by examining
// the container name.
Thread.Sleep(TimeSpan.FromSeconds(30));
}
/// <inheritdoc/>
public override void Run(HiveProxy hive)
{
Covenant.Requires <ArgumentNullException>(hive != null);
var node = hive.GetNode(nodeName);
if (operationName != null)
{
node.InvokeIdempotentAction(operationName, () => action(node));
}
else
{
action(node);
}
}
/// <inheritdoc/>
public override void Run(HiveProxy hive)
{
Covenant.Requires <ArgumentNullException>(hive != null);
var node = hive.GetNode(nodeName);
var status = this.ToString();
node.UploadText(path, text, tabStop, outputEncoding);
if (!string.IsNullOrEmpty(permissions))
{
node.SudoCommand("chmod", permissions, path);
}
StatusPause();
node.Status = string.Empty;
}
/// <inheritdoc/>
public override (string Address, int Port) GetSshEndpoint(string nodeName)
{
return(Address : hive.GetNode(nodeName).PrivateAddress.ToString(), Port : NetworkPorts.SSH);
}
/// <summary>
/// Provision the virtual machines on the XenServer.
/// </summary>
/// <param name="xenSshProxy">The XenServer SSH proxy.</param>
private void ProvisionVirtualMachines(SshProxy <XenClient> xenSshProxy)
{
var xenHost = xenSshProxy.Metadata;
foreach (var node in GetHostedNodes(xenHost))
{
var vmName = GetVmName(node);
var processors = node.Metadata.GetVmProcessors(hive.Definition);
var memoryBytes = node.Metadata.GetVmMemory(hive.Definition);
var diskBytes = node.Metadata.GetVmDisk(hive.Definition);
xenSshProxy.Status = FormatVmStatus(vmName, "create virtual machine");
// We need to create a raw drive if the node hosts a Ceph OSD.
var extraDrives = new List <XenVirtualDrive>();
if (node.Metadata.Labels.CephOSD)
{
extraDrives.Add(
new XenVirtualDrive()
{
Size = node.Metadata.GetCephOSDDriveSize(hive.Definition)
});
}
var vm = xenHost.Machine.Create(vmName, hive.Definition.Hosting.XenServer.TemplateName,
processors: processors,
memoryBytes: memoryBytes,
diskBytes: diskBytes,
snapshot: hive.Definition.Hosting.XenServer.Snapshot,
extraDrives: extraDrives,
primaryStorageRepository: hive.Definition.Hosting.XenServer.StorageRepository,
extraStorageRespository: hive.Definition.Hosting.XenServer.OsdStorageRepository);
xenSshProxy.Status = FormatVmStatus(vmName, "start virtual machine");
xenHost.Machine.Start(vm);
// We need to wait for the virtual machine to start and obtain
// and IP address via DHCP.
var address = string.Empty;
xenSshProxy.Status = FormatVmStatus(vmName, "fetch ip address");
try
{
NeonHelper.WaitFor(
() =>
{
while (true)
{
vm = xenHost.Machine.Find(vmName);
if (!string.IsNullOrEmpty(vm.Address))
{
address = vm.Address;
return(true);
}
Thread.Sleep(1000);
}
},
TimeSpan.FromSeconds(120));
}
catch (TimeoutException)
{
xenSshProxy.Fault("Timeout waiting for virtual machine to start and set an IP address.");
}
// SSH into the VM using the DHCP address, configure the static IP
// address and extend the primary partition and file system to fill
// the drive and then reboot.
var subnet = NetworkCidr.Parse(hive.Definition.Network.PremiseSubnet);
var gateway = hive.Definition.Network.Gateway;
var broadcast = hive.Definition.Network.Broadcast;
// We're going to temporarily set the node to the current VM address
// so we can connect via SSH.
var savedNodeAddress = node.PrivateAddress;
try
{
node.PrivateAddress = IPAddress.Parse(address);
using (var nodeProxy = hive.GetNode(node.Name))
{
xenSshProxy.Status = FormatVmStatus(vmName, "connect");
nodeProxy.WaitForBoot();
// Replace the [/etc/network/interfaces] file to configure the static
// IP and then reboot to reinitialize networking subsystem.
var primaryInterface = node.GetNetworkInterface(node.PrivateAddress);
xenSshProxy.Status = FormatVmStatus(vmName, $"set static ip [{node.PrivateAddress}]");
var interfacesText =
$@"# This file describes the network interfaces available on your system
# and how to activate them. For more information, see interfaces(5).
source /etc/network/interfaces.d/*
# The loopback network interface
auto lo
iface lo inet loopback
# The primary network interface
auto {primaryInterface}
iface {primaryInterface} inet static
address {savedNodeAddress}
netmask {subnet.Mask}
gateway {gateway}
broadcast {broadcast}
";
nodeProxy.UploadText("/etc/network/interfaces", interfacesText);
// Temporarily configure the public Google DNS servers as
// the name servers so DNS will work after we reboot with
// the static IP. Note that hive setup will eventually
// configure the name servers specified in the hive
// definition.
// $todo(jeff.lill):
//
// Is there a good reason why we're not just configuring the
// DNS servers from the hive definition here???
//
// Using the Google DNS seems like it could break some hive
// network configurations (e.g. for hives that don't have
// access to the public Internet). Totally private hives
// aren't really a supported scenario right now though because
// we assume we can use [apt-get]... to pull down packages.
var resolvBaseText =
$@"nameserver 8.8.8.8
nameserver 8.8.4.4
";
nodeProxy.UploadText("/etc/resolvconf/resolv.conf.d/base", resolvBaseText);
// Extend the primary partition and file system to fill
// the virtual the drive.
xenSshProxy.Status = FormatVmStatus(vmName, $"resize primary partition");
// $hack(jeff.lill):
//
// I've seen a transient error here but can't reproduce it. I'm going
// to assume for now that the file system might not be quite ready for
// this operation directly after the VM has been rebooted, so we're going
// to delay for a few seconds before performing the operations.
Thread.Sleep(TimeSpan.FromSeconds(5));
nodeProxy.SudoCommand("growpart /dev/xvda 1");
nodeProxy.SudoCommand("resize2fs /dev/xvda1");
// Reboot to pick up the changes.
xenSshProxy.Status = FormatVmStatus(vmName, "reboot");
nodeProxy.Reboot(wait: false);
}
}
finally
{
// Restore the node's IP address.
node.PrivateAddress = savedNodeAddress;
}
}
}
