/// <summary>
/// Updates the hive services and containers.
/// </summary>
/// <param name="force"><c>true</c> to disable the update prompt.</param>
/// <param name="maxParallel">Maximum number of parallel operations.</param>
/// <param name="imageTag">Optionally overrides the default image tag.</param>
private void UpdateServices(bool force, int maxParallel, string imageTag = null)
{
EnsureRootPivileges();
if (!force && !Program.PromptYesNo($"*** Are you sure you want to UPDATE HIVE services on [{hive.Name}]?"))
{
Program.Exit(0);
}
var controller = new SetupController <NodeDefinition>("hive update images", hive.Nodes)
{
MaxParallel = maxParallel,
ShowStatus = true
};
controller.SetDefaultRunOptions(RunOptions.FaultOnError);
HiveUpdateManager.AddHiveUpdateSteps(hive, controller, out var restartRequired, servicesOnly: true, serviceUpdateParallism: Program.MaxParallel, imageTag: imageTag);
if (controller.StepCount == 0)
{
Console.WriteLine("The hive is already up-to-date.");
Program.Exit(0);
}
if (!controller.Run())
{
Console.Error.WriteLine("*** ERROR: One or more UPDATE steps failed.");
Program.Exit(1);
}
Console.WriteLine();
Console.WriteLine("*** Hive services and containers were updated successfully.");
}
/// <summary>
/// Updates the hive hive configuration, services and containers.
/// </summary>
/// <param name="force"><c>true</c> to disable the update prompt.</param>
/// <param name="maxParallel">Maximum number of parallel operations.</param>
/// <param name="imageTag">Optionally overrides the default image tag.</param>
private void UpdateHive(bool force, int maxParallel, string imageTag = null)
{
EnsureRootPivileges();
if (!force && !Program.PromptYesNo($"*** Are you sure you want to UPDATE HIVE components and services on [{hive.Name}]?"))
{
Program.Exit(0);
}
var controller = new SetupController <NodeDefinition>("hive update", hive.Nodes)
{
ShowStatus = !Program.Quiet
};
controller.MaxParallel = maxParallel;
controller.SetDefaultRunOptions(RunOptions.FaultOnError);
var hiveUpdateCount = HiveUpdateManager.AddHiveUpdateSteps(hive, controller, out var restartRequired, serviceUpdateParallism: Program.MaxParallel, imageTag: imageTag);
if (controller.StepCount == 0)
{
Console.WriteLine("The hive is already up-to-date.");
Program.Exit(0);
}
if (!controller.Run())
{
Console.Error.WriteLine("*** ERROR: One or more UPDATE steps failed.");
Program.Exit(1);
}
Console.WriteLine();
Console.WriteLine("*** Hive components, services, and containers were updated successfully.");
if (hive.Globals.TryGetBool(HiveGlobals.UserDisableAutoUnseal, out var disableAutoUnseal) || !disableAutoUnseal)
{
Console.WriteLine();
Console.WriteLine("*** WARNING: The hive Vault is probably sealed now because auto-unseal is disabled.");
Console.WriteLine();
Console.WriteLine("Use these command to check Vault status and manually unseal if necessary:");
Console.WriteLine();
Console.WriteLine(" neon vault -- status");
Console.WriteLine(" Neon vault -- unseal");
Console.WriteLine();
}
}
/// <summary>
/// Checks the hive for pending updates.
/// </summary>
/// <param name="maxParallel">Maximum number of parallel operations.</param>
private void CheckHive(int maxParallel)
{
EnsureRootPivileges();
// Use a temporary controller to determine how many hive
// updates are pending.
var controller = new SetupController <NodeDefinition>("hive update check", hive.Nodes)
{
MaxParallel = maxParallel,
ShowStatus = !Program.Quiet
};
controller.SetDefaultRunOptions(RunOptions.FaultOnError);
var hiveUpdateCount = HiveUpdateManager.AddHiveUpdateSteps(hive, controller, out var restartRequired, serviceUpdateParallism: Program.MaxParallel);
// Create another controller to actually scan the hive nodes to
// count the pending Linux updates as well as the system containers
// and services that need to be updated.
// $todo(jeff.lill):
//
// We need to query a new image lookup service to get the images
// compatible with the hive and then determine whether any of
// these need updating on any node. Right now, we're just checking
// the Linux package updates.
//
// We should do something similar for the host services like:
// consul, docker, powerdns, and vault.
controller = new SetupController <NodeDefinition>("hive update check", hive.Nodes)
{
MaxParallel = maxParallel,
ShowStatus = !Program.Quiet
};
controller.SetDefaultRunOptions(RunOptions.FaultOnError);
var syncLock = new object();
var maxUpdates = 0;
var maxSecurityUpdates = 0;
var componentInfo = hive.Headend.GetComponentInfo(hive.Globals.Version, ThisAssembly.Git.Branch);
var dockerVersions = new Dictionary <SemanticVersion, int>(); // Counts the numbers versions installed
var consulVersions = new Dictionary <SemanticVersion, int>(); // on hive nodes.
var vaultVersions = new Dictionary <SemanticVersion, int>();
controller.AddStep("scan hive",
(node, stepDelay) =>
{
Thread.Sleep(stepDelay);
//---------------------------------------------------------
// Look for Linux package updates.
node.Status = "run: safe-apt-get update";
node.SudoCommand("safe-apt-get update");
node.Status = "run: apt-check";
var response = node.SudoCommand("/usr/lib/update-notifier/apt-check");
// This command returns the total number of updates and
// the security updates like: TOTAL;SECURITY.
var fields = response.ErrorText.Trim().Split(';');
if (fields.Length < 2 || !int.TryParse(fields[0], out var updates) || !int.TryParse(fields[1], out var securityUpdates))
{
node.Fault($"Unexpected update response: {response.OutputText}");
return;
}
lock (syncLock)
{
maxUpdates = Math.Max(maxUpdates, updates);
maxSecurityUpdates = Math.Max(maxSecurityUpdates, securityUpdates);
}
//---------------------------------------------------------
// Determine the versions of Docker, Consul, and Vault installed
// on this node and tally the versions for the hive. Note that
// it's possible for multiple versions of a compontent to be
// installed on different nodes if a previous update did not
// run until completion.
node.Status = "docker version";
var dockerVersion = node.GetDockerVersion(faultIfNotInstalled: true);
node.Status = "consul version";
var consulVersion = node.GetConsulVersion(faultIfNotInstalled: true);
node.Status = "vault version";
var vaultVersion = node.GetVaultVersion(faultIfNotInstalled: true);
if (!node.IsFaulted)
{
lock (syncLock)
{
int count;
if (!dockerVersions.TryGetValue(dockerVersion, out count))
{
count = 0;
}
dockerVersions[dockerVersion] = count + 1;
if (!consulVersions.TryGetValue(consulVersion, out count))
{
count = 0;
}
consulVersions[consulVersion] = count + 1;
if (!vaultVersions.TryGetValue(vaultVersion, out count))
{
count = 0;
}
vaultVersions[vaultVersion] = count + 1;
}
}
});
if (!controller.Run())
{
Console.Error.WriteLine("*** ERROR: One or more CHECK steps failed.");
Program.Exit(1);
}
// Output the results.
var title = $"[{hive.Name}] hive";
Console.WriteLine();
Console.WriteLine(title);
Console.WriteLine(new string('-', title.Length));
var restartStatus = restartRequired ? " *** hive restart required ***" : string.Empty;
var hiveStatus = (hiveUpdateCount == 0 && maxUpdates == 0) ? "CURRENT" : hiveUpdateCount.ToString() + restartStatus;
var linuxPackageStatus = (maxUpdates == 0) ? "CURRENT" : maxUpdates.ToString();
var linuxSecurityStatus = (maxSecurityUpdates == 0) ? "CURRENT" : maxSecurityUpdates.ToString();
Console.WriteLine($"neonHIVE updates: {hiveStatus}");
Console.WriteLine($"Linux package updates: {linuxPackageStatus}");
Console.WriteLine($"Linux security updates: {linuxSecurityStatus}");
//-------------------------------------------------------------
// Docker status
string dockerVersionInfo;
if (dockerVersions.Count == 0)
{
dockerVersionInfo = "*** ERROR: Docker is not installed.";
}
else if (dockerVersions.Count == 1)
{
dockerVersionInfo = (string)dockerVersions.Keys.First();
}
else
{
var sb = new StringBuilder();
foreach (var version in dockerVersions.Keys.OrderBy(v => v))
{
sb.AppendWithSeparator((string)version, ", ");
}
dockerVersionInfo = sb.ToString();
}
var dockerStatus = "CURRENT";
if (dockerVersions.Count == 0)
{
dockerStatus = "ERROR: cannot detect version";
}
else if (dockerVersions.Count > 1)
{
dockerStatus = "WARNING: multiple versions installed";
}
else if (dockerVersions.Keys.Min(v => v) < (SemanticVersion)componentInfo.Docker)
{
dockerStatus = "UPDATE AVAILABLE";
}
var dockerTitle = $"Docker Engine: {dockerStatus}";
Console.WriteLine();
Console.WriteLine();
Console.WriteLine(dockerTitle);
Console.WriteLine(new string('-', dockerTitle.Length));
Console.WriteLine($"Current: {dockerVersionInfo}");
Console.WriteLine($"Latest: {componentInfo.Docker}");
//-------------------------------------------------------------
// Consul status
string consulVersionInfo;
if (consulVersions.Count == 0)
{
consulVersionInfo = "*** ERROR: Consul is not installed.";
}
else if (consulVersions.Count == 1)
{
consulVersionInfo = (string)consulVersions.Keys.First();
}
else
{
var sb = new StringBuilder();
foreach (var version in consulVersions.Keys.OrderBy(v => v))
{
sb.AppendWithSeparator((string)version, ", ");
}
consulVersionInfo = sb.ToString();
}
var consulStatus = "CURRENT";
if (consulVersions.Count == 0)
{
consulStatus = "ERROR: cannot detect version";
}
else if (consulVersions.Count > 1)
{
consulStatus = "WARNING: multiple versions installed";
}
else if (consulVersions.Keys.Min(v => v) < (SemanticVersion)componentInfo.Consul)
{
consulStatus = "UPDATE AVAILABLE";
}
var consulTitle = $"HashiCorp Consul: {consulStatus}";
Console.WriteLine();
Console.WriteLine();
Console.WriteLine(consulTitle);
Console.WriteLine(new string('-', consulTitle.Length));
Console.WriteLine($"Current: {consulVersionInfo}");
Console.WriteLine($"Latest: {componentInfo.Consul}");
//-------------------------------------------------------------
// Vault status
string vaultVersionInfo;
if (consulVersions.Count == 0)
{
vaultVersionInfo = "*** ERROR: Vault is not installed.";
}
else if (consulVersions.Count == 1)
{
vaultVersionInfo = (string)vaultVersions.Keys.First();
}
else
{
var sb = new StringBuilder();
foreach (var version in vaultVersions.Keys.OrderBy(v => v))
{
sb.AppendWithSeparator((string)version, ", ");
}
vaultVersionInfo = sb.ToString();
}
var vaultStatus = "CURRENT";
if (vaultVersions.Count == 0)
{
vaultStatus = "ERROR: cannot detect version";
}
else if (vaultVersions.Count > 1)
{
vaultStatus = "WARNING: multiple versions installed";
}
else if (vaultVersions.Keys.Min(v => v) < (SemanticVersion)componentInfo.Vault)
{
vaultStatus = "UPDATE AVAILABLE";
}
var vaultTitle = $"HashiCorp Vault: {vaultStatus}";
Console.WriteLine();
Console.WriteLine();
Console.WriteLine(vaultTitle);
Console.WriteLine(new string('-', vaultTitle.Length));
Console.WriteLine($"Current: {vaultVersionInfo}");
Console.WriteLine($"Latest: {componentInfo.Vault}");
}