/// <inheritdoc/>
public void Run(ModuleContext context)
{
var hive = HiveHelper.Hive;
var consul = HiveHelper.Consul;
if (!context.ValidateArguments(context.Arguments, validModuleArgs))
{
context.Failed = true;
return;
}
// Obtain common arguments.
context.WriteLine(AnsibleVerbosity.Trace, $"Parsing [state]");
if (!context.Arguments.TryGetValue <string>("state", out var state))
{
state = "get";
}
state = state.ToLowerInvariant();
context.WriteLine(AnsibleVerbosity.Trace, $"Parsing [name]");
if (!context.Arguments.TryGetValue <string>("name", out var name))
{
throw new ArgumentException($"[name] module argument is required.");
}
context.WriteLine(AnsibleVerbosity.Trace, $"Parsing [value]");
if (!context.Arguments.TryGetValue <string>("value", out var value) && state == "set")
{
throw new ArgumentException($"[value] module argument is required when [state={state}].");
}
var validate = context.ParseBool("validate");
validate = validate ?? true;
if (context.HasErrors)
{
return;
}
// We have the required arguments, so perform the operation.
switch (state)
{
case "get":
if (hive.Globals.TryGetString(name, out var output))
{
context.WriteLine(AnsibleVerbosity.Important, output);
}
else
{
context.WriteErrorLine($"Hive global [{name}] does not exist.");
}
break;
case "set":
if (validate.Value)
{
hive.Globals.SetUser(name, value);
}
else
{
hive.Globals.Set(name, value);
}
break;
default:
throw new ArgumentException($"[state={state}] is not one of the valid choices: [get] or [set].");
}
}
/// <inheritdoc/>
public void Run(ModuleContext context)
{
var hive = HiveHelper.Hive;
if (!context.ValidateArguments(context.Arguments, validModuleArgs))
{
context.Failed = true;
return;
}
// Obtain common arguments.
context.WriteLine(AnsibleVerbosity.Trace, $"Parsing [registry]");
if (!context.Arguments.TryGetValue <string>("registry", out var registry))
{
throw new ArgumentException($"[registry] module argument is required.");
}
context.WriteLine(AnsibleVerbosity.Trace, $"Parsing [state]");
if (!context.Arguments.TryGetValue <string>("state", out var state))
{
state = "present";
}
state = state.ToLowerInvariant();
if (context.HasErrors)
{
return;
}
context.WriteLine(AnsibleVerbosity.Trace, $"Reading existing credentials for [{registry}].");
var existingCredentials = hive.Registry.GetCredentials(registry);
if (existingCredentials != null)
{
context.WriteLine(AnsibleVerbosity.Info, $"Credentials for [{registry}] exist.");
}
else
{
context.WriteLine(AnsibleVerbosity.Info, $"Credentials for [{registry}] do not exist.");
}
var sbErrorNodes = new StringBuilder();
switch (state)
{
case "absent":
if (context.CheckMode)
{
if (existingCredentials != null)
{
context.WriteLine(AnsibleVerbosity.Important, $"Credentials for [{registry}] will be deleted when CHECK-MODE is disabled.");
}
return;
}
// Log the hive out of the registry.
if (existingCredentials != null)
{
context.Changed = true;
}
context.WriteLine(AnsibleVerbosity.Trace, $"Logging the hive out of the [{registry}] registry.");
hive.Registry.Logout(registry);
context.WriteLine(AnsibleVerbosity.Trace, $"All hive nodes are logged out.");
break;
case "present":
if (context.CheckMode)
{
if (existingCredentials == null)
{
context.WriteLine(AnsibleVerbosity.Important, $"Credentials for [{registry}] will be added when CHECK-MODE is disabled.");
}
return;
}
// Parse the [username] and [password] credentials.
context.WriteLine(AnsibleVerbosity.Trace, $"Parsing [username]");
if (!context.Arguments.TryGetValue <string>("username", out var username))
{
throw new ArgumentException($"[username] module argument is required.");
}
context.WriteLine(AnsibleVerbosity.Trace, $"Parsing [password]");
if (!context.Arguments.TryGetValue <string>("password", out var password))
{
throw new ArgumentException($"[password] module argument is required.");
}
context.WriteLine(AnsibleVerbosity.Trace, $"Logging the hive into the [{registry}] registry.");
hive.Registry.Login(registry, username, password);
// Log all of the nodes in with the new registry credentials.
//
// Note that we won't do this if the registry cache is enabled and we're
// updating credentials for the Docker public registry because for this
// configuration, only the registry cache needs the upstream credentials.
// The nodes don't authenticate against the local registry cache.
if (!hive.Definition.Docker.RegistryCache || !HiveHelper.IsDockerPublicRegistry(registry))
{
context.WriteLine(AnsibleVerbosity.Trace, $"Logging the hive into the [{registry}] registry.");
hive.Registry.Login(registry, username, password);
}
else
{
// Restart the hive registry cache containers with the new credentials.
context.WriteLine(AnsibleVerbosity.Trace, $"Restarting the hive registry caches.");
if (!hive.Registry.RestartCache(registry, username, password))
{
context.WriteErrorLine("Unable to restart one or more of the hive registry caches.");
return;
}
context.WriteLine(AnsibleVerbosity.Trace, $"Hive registry caches restarted.");
}
context.Changed = existingCredentials == null;
break;
default:
throw new ArgumentException($"[state={state}] is not one of the valid choices: [present] or [absent].");
}
}
/// <inheritdoc/>
public void Run(ModuleContext context)
{
var hive = HiveHelper.Hive;
var consul = HiveHelper.Consul;
if (!context.ValidateArguments(context.Arguments, validModuleArgs))
{
context.Failed = true;
return;
}
// Obtain common arguments.
context.WriteLine(AnsibleVerbosity.Trace, $"Parsing [hostname]");
if (!context.Arguments.TryGetValue <string>("hostname", out var hostname))
{
throw new ArgumentException($"[hostname] module argument is required.");
}
if (!HiveDefinition.DnsHostRegex.IsMatch(hostname))
{
throw new ArgumentException($"[hostname={hostname}] is not a valid DNS hostname.");
}
context.WriteLine(AnsibleVerbosity.Trace, $"Parsing [system]");
var system = context.ParseBool("system") ?? false;
context.WriteLine(AnsibleVerbosity.Trace, $"Parsing [state]");
if (!context.Arguments.TryGetValue <string>("state", out var state))
{
state = "present";
}
state = state.ToLowerInvariant();
context.WriteLine(AnsibleVerbosity.Trace, $"Parsing [endpoints]");
if (!context.Arguments.TryGetValue <JToken>("endpoints", out var endpointsToken) && state == "present")
{
throw new ArgumentException($"[endpoints] module argument is required when [state={state}].");
}
context.WriteLine(AnsibleVerbosity.Trace, $"Parsing [wait]");
var wait = context.ParseBool("wait");
wait = wait ?? false;
if (context.HasErrors)
{
return;
}
// We have the required arguments, so perform the operation.
switch (state)
{
case "absent":
context.WriteLine(AnsibleVerbosity.Trace, $"Check if DNS entry [{hostname}] exists.");
if (hive.Dns.Get(hostname) != null)
{
context.WriteLine(AnsibleVerbosity.Trace, $"DNS entry [{hostname}] does exist.");
context.WriteLine(AnsibleVerbosity.Info, $"Deleting DNS entry [{hostname}].");
if (context.CheckMode)
{
context.WriteLine(AnsibleVerbosity.Info, $"DNS entry [{hostname}] will be deleted when CHECK-MODE is disabled.");
}
else
{
hive.Dns.Remove(hostname, waitUntilPropagated: wait.Value);
context.WriteLine(AnsibleVerbosity.Trace, $"DNS entry [{hostname}] deleted.");
}
context.Changed = !context.CheckMode;
}
else
{
context.WriteLine(AnsibleVerbosity.Trace, $"DNS entry [{hostname}] does not exist.");
}
break;
case "present":
var endpointsArray = endpointsToken as JArray;
if (endpointsArray == null)
{
throw new ArgumentException($"[endpoints] module argument must be an array.");
}
var endpoints = new List <DnsEndpoint>();
foreach (var item in endpointsArray)
{
var endpointJObject = item as JObject;
if (endpointJObject == null)
{
context.WriteErrorLine("One or more [endpoints] are invalid.");
context.Failed = true;
return;
}
if (!context.ValidateArguments(endpointJObject, endpointArgs, "endpoints"))
{
context.Failed = true;
return;
}
endpoints.Add(item.ToObject <DnsEndpoint>());
}
context.WriteLine(AnsibleVerbosity.Trace, $"[{endpoints.Count}] endpoints parsed.");
// Construct the new entry.
var newEntry = new DnsEntry()
{
Hostname = hostname,
IsSystem = system,
Endpoints = endpoints
};
// Validate the new DNS entry.
context.WriteLine(AnsibleVerbosity.Trace, "Validating DNS entry.");
var errors = newEntry.Validate(hive.Definition, hive.Definition.GetHostGroups(excludeAllGroup: true));
if (errors.Count > 0)
{
context.WriteLine(AnsibleVerbosity.Trace, $"[{errors.Count}] DNS entry validation errors.");
foreach (var error in errors)
{
context.WriteLine(AnsibleVerbosity.Important, error);
context.WriteErrorLine(error);
}
context.Failed = true;
return;
}
context.WriteLine(AnsibleVerbosity.Trace, "DNS entry is valid.");
// Try reading an existing entry with this name and then determine
// whether the two versions of the entry are actually different.
context.WriteLine(AnsibleVerbosity.Trace, $"Look up existing DNS entry for [{hostname}].");
var existingEntry = hive.Dns.Get(hostname);
var changed = false;
if (existingEntry != null)
{
context.WriteLine(AnsibleVerbosity.Trace, $"DNS entry exists: checking for differences.");
changed = !NeonHelper.JsonEquals(newEntry, existingEntry);
if (changed)
{
context.WriteLine(AnsibleVerbosity.Trace, $"DNS entries are different.");
}
else
{
context.WriteLine(AnsibleVerbosity.Info, $"DNS entries are the same. No need to update.");
}
}
else
{
changed = true;
context.WriteLine(AnsibleVerbosity.Trace, $"DNS entry for [hostname={hostname}] does not exist.");
}
if (changed)
{
if (context.CheckMode)
{
context.WriteLine(AnsibleVerbosity.Info, $"DNS entry [{hostname}] will be updated when CHECK-MODE is disabled.");
}
else
{
context.WriteLine(AnsibleVerbosity.Trace, $"Updating DNS entry.");
hive.Dns.Set(newEntry, waitUntilPropagated: wait.Value);
context.WriteLine(AnsibleVerbosity.Info, $"DNS entry updated.");
}
context.Changed = !context.CheckMode;
}
break;
default:
throw new ArgumentException($"[state={state}] is not one of the valid choices: [present] or [absent].");
}
}
/// <inheritdoc/>
public void Run(ModuleContext context)
{
var hive = HiveHelper.Hive;
if (!context.ValidateArguments(context.Arguments, validModuleArgs))
{
context.Failed = true;
return;
}
// Obtain common arguments.
if (!context.Arguments.TryGetValue <string>("name", out var name))
{
throw new ArgumentException($"[name] module argument is required.");
}
if (!HiveDefinition.IsValidName(name))
{
throw new ArgumentException($"[name={name}] is not a valid certificate name.");
}
if (!context.Arguments.TryGetValue <string>("state", out var state))
{
state = "present";
}
state = state.ToLowerInvariant();
if (!context.Arguments.TryGetValue <bool>("force", out var force))
{
force = false;
}
if (context.HasErrors)
{
return;
}
// We have the required arguments, so perform the operation.
if (!context.Login.HasVaultRootCredentials)
{
throw new ArgumentException("Access Denied: Root Vault credentials are required.");
}
switch (state)
{
case "absent":
context.WriteLine(AnsibleVerbosity.Trace, $"Vault: checking for [{name}] certificate");
if (hive.Certificate.Get(name) != null)
{
context.WriteLine(AnsibleVerbosity.Trace, $"Vault: [{name}] certificate exists");
if (context.CheckMode)
{
context.WriteLine(AnsibleVerbosity.Info, $"Certificate [{name}] will be removed when CHECK-MODE is disabled.");
}
else
{
context.WriteLine(AnsibleVerbosity.Trace, $"Removing [{name}] certyificate.");
hive.Certificate.Remove(name);
context.WriteLine(AnsibleVerbosity.Info, $"[{name}] certificate removed");
}
context.Changed = !context.CheckMode;
}
else
{
context.WriteLine(AnsibleVerbosity.Info, $"[{name}] certificate does not exist");
}
break;
case "present":
if (!context.Arguments.TryGetValue <string>("value", out var value))
{
throw new ArgumentException($"[value] module argument is required.");
}
var certificate = TlsCertificate.Parse(value); // This validates the certificate/private key
context.WriteLine(AnsibleVerbosity.Trace, $"Reading [{name}] certificate");
var existingCert = hive.Certificate.Get(name);
var changed = false;
if (existingCert == null)
{
context.WriteLine(AnsibleVerbosity.Info, $"[{name}] certificate does not exist");
context.Changed = !context.CheckMode;
changed = true;
}
else if (!NeonHelper.JsonEquals(existingCert, certificate) || force)
{
context.WriteLine(AnsibleVerbosity.Info, $"[{name}] certificate does exists but is different");
context.Changed = !context.CheckMode;
changed = true;
}
else
{
context.WriteLine(AnsibleVerbosity.Info, $"[{name}] certificate is unchanged");
}
if (changed)
{
if (context.CheckMode)
{
context.WriteLine(AnsibleVerbosity.Info, $"Certificate [{name}] will be updated when CHECK-MODE is disabled.");
}
else
{
context.WriteLine(AnsibleVerbosity.Trace, $"Saving [{name}] certificate");
hive.Certificate.Set(name, certificate);
context.WriteLine(AnsibleVerbosity.Info, $"[{name}] certificate saved");
}
}
break;
default:
throw new ArgumentException($"[state={state}] is not one of the valid choices: [present] or [absent].");
}
}
/// <inheritdoc/>
public void Run(ModuleContext context)
{
var hive = HiveHelper.Hive;
var nodeGroups = hive.Definition.GetHostGroups(excludeAllGroup: true);
//-----------------------------------------------------------------
// Parse the module arguments.
if (!context.ValidateArguments(context.Arguments, validModuleArgs))
{
context.Failed = true;
return;
}
var couchbaseArgs = CouchbaseArgs.Parse(context);
if (couchbaseArgs == null)
{
return;
}
var query = context.ParseString("query", q => !string.IsNullOrWhiteSpace(q));
if (context.HasErrors)
{
return;
}
var format = context.ParseEnum <CouchbaseFileFormat>("format");
if (!format.HasValue)
{
format = CouchbaseFileFormat.JsonLines;
}
var limit = context.ParseLong("limit", v => v >= 0);
if (!limit.HasValue || limit.Value == 0)
{
limit = long.MaxValue;
}
var output = context.ParseString("output");
//-----------------------------------------------------------------
// Execute the query.
using (var bucket = couchbaseArgs.Settings.OpenBucket(couchbaseArgs.Credentials))
{
try
{
var results = bucket.QuerySafeAsync <JObject>(query).Result;
var count = Math.Min(results.Count, limit.Value);
using (var writer = new CouchbaseQueryResultWriter(context, format.Value, output))
{
for (int i = 0; i < count; i++)
{
var document = results[i];
var isLast = i == count - 1;
writer.WriteDocument(document, isLast);
}
}
}
catch (AggregateException e)
{
var queryException = e.Find <CouchbaseQueryResponseException>();
if (queryException == null)
{
throw;
}
foreach (var error in queryException.Errors)
{
context.WriteErrorLine($"Couchbase [{error.Code}]: {error.Message}");
}
}
}
}
/// <inheritdoc/>
public void Run(ModuleContext context)
{
var hive = HiveHelper.Hive;
string hostname;
if (!context.ValidateArguments(context.Arguments, validModuleArgs))
{
context.Failed = true;
return;
}
// Obtain common arguments.
context.WriteLine(AnsibleVerbosity.Trace, $"Parsing [state]");
if (!context.Arguments.TryGetValue <string>("state", out var state))
{
state = "present";
}
state = state.ToLowerInvariant();
if (context.HasErrors)
{
return;
}
var manager = hive.GetReachableManager();
var sbErrorNodes = new StringBuilder();
// Determine whether the registry service is already deployed and
// also retrieve the registry credentials from Vault if present.
// Note that the current registry hostname will be persisted to
// Consul at [neon/service/neon-registry/hostname] when a registry
// is deployed.
context.WriteLine(AnsibleVerbosity.Trace, $"Inspecting the [neon-registry] service.");
var currentService = hive.Docker.InspectService("neon-registry");
context.WriteLine(AnsibleVerbosity.Trace, $"Getting current registry hostname from Consul.");
var currentHostname = hive.Registry.GetLocalHostname();
var currentSecret = hive.Registry.GetLocalSecret();
var currentImage = currentService?.Spec.TaskTemplate.ContainerSpec.ImageWithoutSHA;
var currentCredentials = // Set blank properties for the change detection below.
new RegistryCredentials()
{
Registry = string.Empty,
Username = string.Empty,
Password = string.Empty
};
if (!string.IsNullOrEmpty(currentHostname))
{
context.WriteLine(AnsibleVerbosity.Trace, $"Reading existing registry credentials for [{currentHostname}].");
currentCredentials = hive.Registry.GetCredentials(currentHostname);
if (currentCredentials != null)
{
context.WriteLine(AnsibleVerbosity.Info, $"Registry credentials for [{currentHostname}] exist.");
}
else
{
context.WriteLine(AnsibleVerbosity.Info, $"Registry credentials for [{currentHostname}] do not exist.");
}
}
// Obtain the current registry TLS certificate (if any).
var currentCertificate = hive.Certificate.Get("neon-registry");
// Perform the operation.
switch (state)
{
case "absent":
context.WriteLine(AnsibleVerbosity.Trace, $"Parsing [hostname]");
if (!context.Arguments.TryGetValue <string>("hostname", out hostname))
{
throw new ArgumentException($"[hostname] module argument is required.");
}
if (currentService == null)
{
context.WriteLine(AnsibleVerbosity.Important, "[neon-registry] is not currently deployed.");
}
if (context.CheckMode)
{
context.WriteLine(AnsibleVerbosity.Important, $"Local registry will be removed when CHECK-MODE is disabled.");
return;
}
if (currentService == null)
{
return; // Nothing to do
}
context.Changed = true;
// Logout of the registry.
if (currentCredentials != null)
{
context.WriteLine(AnsibleVerbosity.Trace, $"Logging the hive out of the [{currentHostname}] registry.");
hive.Registry.Logout(currentHostname);
}
// Delete the [neon-registry] service and volume. Note that
// the volume should exist on all of the manager nodes.
context.WriteLine(AnsibleVerbosity.Trace, $"Removing the [neon-registry] service.");
manager.DockerCommand(RunOptions.None, "docker", "service", "rm", "neon-registry");
context.WriteLine(AnsibleVerbosity.Trace, $"Removing the [neon-registry] volumes.");
var volumeRemoveActions = new List <Action>();
var volumeRetryPolicy = new LinearRetryPolicy(typeof(TransientException), maxAttempts: 10, retryInterval: TimeSpan.FromSeconds(2));
foreach (var node in hive.Managers)
{
volumeRemoveActions.Add(
() =>
{
// $hack(jeff.lill):
//
// Docker service removal appears to be synchronous but the removal of the
// actual service task containers is not. We're going to detect this and
// throw a [TransientException] and then retry.
using (var clonedNode = node.Clone())
{
lock (context)
{
context.WriteLine(AnsibleVerbosity.Trace, $"Removing [neon-registry] volume on [{clonedNode.Name}].");
}
volumeRetryPolicy.InvokeAsync(
async() =>
{
var response = clonedNode.DockerCommand(RunOptions.None, "docker", "volume", "rm", "neon-registry");
if (response.ExitCode != 0)
{
var message = $"Error removing [neon-registry] volume from [{clonedNode.Name}: {response.ErrorText}";
lock (syncLock)
{
context.WriteLine(AnsibleVerbosity.Info, message);
}
if (response.AllText.Contains("volume is in use"))
{
throw new TransientException(message);
}
}
else
{
lock (context)
{
context.WriteLine(AnsibleVerbosity.Trace, $"Removed [neon-registry] volume on [{clonedNode.Name}].");
}
}
await Task.Delay(0);
}).Wait();
}
});
}
NeonHelper.WaitForParallel(volumeRemoveActions);
// Remove the traffic manager rule and certificate.
context.WriteLine(AnsibleVerbosity.Trace, $"Removing the [neon-registry] traffic manager rule.");
hive.PublicTraffic.RemoveRule("neon-registry");
context.WriteLine(AnsibleVerbosity.Trace, $"Removing the [neon-registry] traffic manager certificate.");
hive.Certificate.Remove("neon-registry");
// Remove any related Consul state.
context.WriteLine(AnsibleVerbosity.Trace, $"Removing the [neon-registry] Consul [hostname] and [secret].");
hive.Registry.SetLocalHostname(null);
hive.Registry.SetLocalSecret(null);
// Logout the hive from the registry.
context.WriteLine(AnsibleVerbosity.Trace, $"Logging the hive out of the [{currentHostname}] registry.");
hive.Registry.Logout(currentHostname);
// Remove the hive DNS host entry.
context.WriteLine(AnsibleVerbosity.Trace, $"Removing the [{currentHostname}] registry DNS hosts entry.");
hive.Dns.Remove(hostname);
break;
case "present":
if (!hive.Definition.HiveFS.Enabled)
{
context.WriteErrorLine("The local registry service requires hive CephFS.");
return;
}
// Parse the [hostname], [certificate], [username] and [password] arguments.
context.WriteLine(AnsibleVerbosity.Trace, $"Parsing [hostname]");
if (!context.Arguments.TryGetValue <string>("hostname", out hostname))
{
throw new ArgumentException($"[hostname] module argument is required.");
}
context.WriteLine(AnsibleVerbosity.Trace, $"Parsing [certificate]");
if (!context.Arguments.TryGetValue <string>("certificate", out var certificatePem))
{
throw new ArgumentException($"[certificate] module argument is required.");
}
if (!TlsCertificate.TryParse(certificatePem, out var certificate))
{
throw new ArgumentException($"[certificate] is not a valid certificate.");
}
context.WriteLine(AnsibleVerbosity.Trace, $"Parsing [username]");
if (!context.Arguments.TryGetValue <string>("username", out var username))
{
throw new ArgumentException($"[username] module argument is required.");
}
context.WriteLine(AnsibleVerbosity.Trace, $"Parsing [password]");
if (!context.Arguments.TryGetValue <string>("password", out var password))
{
throw new ArgumentException($"[password] module argument is required.");
}
context.WriteLine(AnsibleVerbosity.Trace, $"Parsing [secret]");
if (!context.Arguments.TryGetValue <string>("secret", out var secret) || string.IsNullOrEmpty(secret))
{
throw new ArgumentException($"[secret] module argument is required.");
}
context.WriteLine(AnsibleVerbosity.Trace, $"Parsing [image]");
if (!context.Arguments.TryGetValue <string>("image", out var image))
{
image = HiveConst.NeonProdRegistry + "/neon-registry:latest";
}
// Detect service changes.
var hostnameChanged = hostname != currentCredentials?.Registry;
var usernameChanged = username != currentCredentials?.Username;
var passwordChanged = password != currentCredentials?.Password;
var secretChanged = secret != currentSecret;
var imageChanged = image != currentImage;
var certificateChanged = certificate?.CombinedPemNormalized != currentCertificate?.CombinedPemNormalized;
var updateRequired = hostnameChanged ||
usernameChanged ||
passwordChanged ||
secretChanged ||
imageChanged ||
certificateChanged;
if (hostnameChanged)
{
context.WriteLine(AnsibleVerbosity.Info, $"[hostname] changed from [{currentCredentials?.Registry}] --> [{hostname}]");
}
if (usernameChanged)
{
context.WriteLine(AnsibleVerbosity.Info, $"[username] changed from [{currentCredentials?.Username}] --> [{username}]");
}
if (usernameChanged)
{
context.WriteLine(AnsibleVerbosity.Info, $"[password] changed from [{currentCredentials?.Password}] --> [**REDACTED**]");
}
if (secretChanged)
{
context.WriteLine(AnsibleVerbosity.Info, $"[secret] changed from [{currentSecret}] --> [**REDACTED**]");
}
if (imageChanged)
{
context.WriteLine(AnsibleVerbosity.Info, $"[image] changed from [{currentImage}] --> [{image}]");
}
if (certificateChanged)
{
var currentCertRedacted = currentCertificate != null ? "**REDACTED**" : "**NONE**";
context.WriteLine(AnsibleVerbosity.Info, $"[certificate] changed from [{currentCertRedacted}] --> [**REDACTED**]");
}
// Handle CHECK-MODE.
if (context.CheckMode)
{
if (currentService == null)
{
context.WriteLine(AnsibleVerbosity.Important, $"Local registry will be deployed when CHECK-MODE is disabled.");
return;
}
if (updateRequired)
{
context.WriteLine(AnsibleVerbosity.Important, $"One or more of the arguments have changed so the registry will be updated when CHECK-MODE is disabled.");
return;
}
return;
}
// Create the hive DNS host entry we'll use to redirect traffic targeting the registry
// hostname to the hive managers. We need to do this because registry IP addresses
// are typically public, typically targeting the external firewall or load balancer
// interface.
//
// The problem is that hive nodes will generally be unable to connect to the
// local managers through the firewall/load balancer because most network routers
// block network traffic that originates from inside the hive, then leaves
// to hit the external router interface with the expectation of being routed
// back inside. I believe this is an anti-spoofing security measure.
var dnsRedirect = GetRegistryDnsEntry(hostname);
// Perform the operation.
if (currentService == null)
{
context.WriteLine(AnsibleVerbosity.Important, $"[neon-registry] service needs to be created.");
context.Changed = true;
// The registry service isn't running, so we'll do a full deployment.
context.WriteLine(AnsibleVerbosity.Trace, $"Setting certificate.");
hive.Certificate.Set("neon-registry", certificate);
context.WriteLine(AnsibleVerbosity.Trace, $"Updating Consul settings.");
hive.Registry.SetLocalHostname(hostname);
hive.Registry.SetLocalSecret(secret);
context.WriteLine(AnsibleVerbosity.Trace, $"Adding hive DNS host entry for [{hostname}].");
hive.Dns.Set(dnsRedirect, waitUntilPropagated: true);
context.WriteLine(AnsibleVerbosity.Trace, $"Writing traffic manager rule.");
hive.PublicTraffic.SetRule(GetRegistryTrafficManagerRule(hostname));
context.WriteLine(AnsibleVerbosity.Trace, $"Creating the [neon-registry] service.");
var createResponse = manager.DockerCommand(RunOptions.None,
"docker service create",
"--name", "neon-registry",
"--mode", "global",
"--constraint", "node.role==manager",
"--env", $"USERNAME={username}",
"--env", $"PASSWORD={password}",
"--env", $"SECRET={secret}",
"--env", $"LOG_LEVEL=info",
"--env", $"READ_ONLY=false",
"--mount", "type=volume,src=neon-registry,volume-driver=neon,dst=/var/lib/neon-registry",
"--network", "neon-public",
"--restart-delay", "10s",
image);
if (createResponse.ExitCode != 0)
{
context.WriteErrorLine($"[neon-registry] service create failed: {createResponse.ErrorText}");
return;
}
context.WriteLine(AnsibleVerbosity.Trace, $"Service created.");
context.WriteLine(AnsibleVerbosity.Trace, $"Wait for [neon-registry] service to stabilize (30s).");
Thread.Sleep(TimeSpan.FromSeconds(30));
context.WriteLine(AnsibleVerbosity.Trace, $"Logging the hive into the [{hostname}] registry.");
hive.Registry.Login(hostname, username, password);
}
else if (updateRequired)
{
context.WriteLine(AnsibleVerbosity.Important, $"[neon-registry] service update is required.");
context.Changed = true;
// Update the service and related settings as required.
if (certificateChanged)
{
context.WriteLine(AnsibleVerbosity.Trace, $"Updating certificate.");
hive.Certificate.Set("neon-registry", certificate);
}
if (hostnameChanged)
{
context.WriteLine(AnsibleVerbosity.Trace, $"Updating traffic manager rule.");
hive.PublicTraffic.SetRule(GetRegistryTrafficManagerRule(hostname));
context.WriteLine(AnsibleVerbosity.Trace, $"Updating hive DNS host entry for [{hostname}] (60 seconds).");
hive.Dns.Set(dnsRedirect, waitUntilPropagated: true);
context.WriteLine(AnsibleVerbosity.Trace, $"Updating local hive hostname [{hostname}].");
hive.Registry.SetLocalHostname(hostname);
if (!string.IsNullOrEmpty(currentHostname))
{
context.WriteLine(AnsibleVerbosity.Trace, $"Logging the hive out of the [{currentHostname}] registry.");
hive.Registry.Logout(currentHostname);
}
}
if (secretChanged)
{
context.WriteLine(AnsibleVerbosity.Trace, $"Updating local hive secret.");
hive.Registry.SetLocalSecret(secret);
}
context.WriteLine(AnsibleVerbosity.Trace, $"Updating service.");
var updateResponse = manager.DockerCommand(RunOptions.None,
"docker service update",
"--env-add", $"USERNAME={username}",
"--env-add", $"PASSWORD={password}",
"--env-add", $"SECRET={secret}",
"--env-add", $"LOG_LEVEL=info",
"--env-add", $"READ_ONLY=false",
"--image", image,
"neon-registry");
if (updateResponse.ExitCode != 0)
{
context.WriteErrorLine($"[neon-registry] service update failed: {updateResponse.ErrorText}");
return;
}
context.WriteLine(AnsibleVerbosity.Trace, $"Service updated.");
context.WriteLine(AnsibleVerbosity.Trace, $"Logging the hive into the [{hostname}] registry.");
hive.Registry.Login(hostname, username, password);
}
else
{
context.WriteLine(AnsibleVerbosity.Important, $"[neon-registry] service update is not required but we're logging all nodes into [{hostname}] to ensure hive consistency.");
hive.Registry.Login(hostname, username, password);
context.Changed = false;
}
break;
case "prune":
if (currentService == null)
{
context.WriteLine(AnsibleVerbosity.Important, "Registry service is not running.");
return;
}
if (context.CheckMode)
{
context.WriteLine(AnsibleVerbosity.Important, "Registry will be pruned when CHECK-MODE is disabled.");
return;
}
context.Changed = true; // Always set this to TRUE for prune.
// We're going to upload a script to one of the managers that handles
// putting the [neon-registry] service into READ-ONLY mode, running
// the garbage collection container and then restoring [neon-registry]
// to READ/WRITE mode.
//
// The nice thing about this is that the operation will continue to
// completion on the manager node even if we lose the SSH connection.
var updateScript =
$@"#!/bin/bash
# Update [neon-registry] to READ-ONLY mode:
docker service update --env-rm READ_ONLY --env-add READ_ONLY=true neon-registry
# Prune the registry:
docker run \
--name neon-registry-prune \
--restart-condition=none \
--mount type=volume,src=neon-registry,volume-driver=neon,dst=/var/lib/neon-registry \
{HiveConst.NeonProdRegistry}/neon-registry garbage-collect
# Restore [neon-registry] to READ/WRITE mode:
docker service update --env-rm READ_ONLY --env-add READ_ONLY=false neon-registry
";
var bundle = new CommandBundle("./collect.sh");
bundle.AddFile("collect.sh", updateScript, isExecutable: true);
context.WriteLine(AnsibleVerbosity.Info, "Registry prune started.");
var pruneResponse = manager.SudoCommand(bundle, RunOptions.None);
if (pruneResponse.ExitCode != 0)
{
context.WriteErrorLine($"The prune operation failed. The registry may be running in READ-ONLY mode: {pruneResponse.ErrorText}");
return;
}
context.WriteLine(AnsibleVerbosity.Info, "Registry prune completed.");
break;
default:
throw new ArgumentException($"[state={state}] is not one of the valid choices: [present], [absent], or [prune].");
}
}
/// <inheritdoc/>
public void Run(ModuleContext context)
{
var hive = HiveHelper.Hive;
if (!context.ValidateArguments(context.Arguments, validModuleArgs))
{
context.Failed = true;
return;
}
// Obtain common arguments.
if (!context.Arguments.TryGetValue <string>("name", out var name))
{
throw new ArgumentException($"[name] module argument is required.");
}
if (!context.Arguments.TryGetValue <string>("state", out var state))
{
state = "deploy";
}
state = state.ToLowerInvariant();
var manager = hive.GetReachableManager();
var response = (CommandResponse)null;
switch (state)
{
case "deploy":
if (!context.Arguments.TryGetValue("stack", out var stackObject))
{
throw new ArgumentException($"[stack] module argument is required when [state=deploy].");
}
var stackJson = NeonHelper.JsonSerialize(stackObject);
var stackYaml = NeonHelper.JsonToYaml(stackJson);
var bundle = new CommandBundle($"docker stack deploy --compose-file ./compose.yaml {name}");
bundle.AddFile("compose.yaml", stackYaml);
response = manager.SudoCommand(bundle, RunOptions.None);
if (response.ExitCode != 0)
{
context.WriteErrorLine(response.ErrorText);
}
context.Changed = true;
break;
case "remove":
response = manager.SudoCommand("docker stack rm", RunOptions.None, name);
if (response.ExitCode != 0)
{
context.WriteErrorLine(response.ErrorText);
}
context.Changed = true;
break;
default:
throw new ArgumentException($"[state={state}] is not one of the valid choices: [present], [absent], or [rollback].");
}
}
/// <inheritdoc/>
public void Run(ModuleContext context)
{
var hive = HiveHelper.Hive;
if (!context.ValidateArguments(context.Arguments, validModuleArgs))
{
context.Failed = true;
return;
}
// Obtain common arguments.
if (!context.Arguments.TryGetValue <string>("state", out var state))
{
state = "command";
}
state = state.ToLowerInvariant();
var node = (SshProxy <NodeDefinition>)null;
if (context.Arguments.TryGetValue <string>("node", out var nodeName) && !string.IsNullOrEmpty(nodeName))
{
try
{
node = hive.GetNode(nodeName);
}
catch (KeyNotFoundException)
{
context.WriteErrorLine($"*** ERROR: Node [{nodeName}] does not exist.");
return;
}
if (!node.Metadata.Labels.HiveMQ && !node.Metadata.Labels.HiveMQManager)
{
context.WriteErrorLine($"*** ERROR: Node [{nodeName}] does not host HiveMQ.");
return;
}
}
else
{
// Find a reachable hive node hosting a RabbitMQ management node.
node = hive.GetReachableNode(n => n.Metadata.Labels.HiveMQManager, ReachableHostMode.ReturnNull);
if (node == null)
{
context.WriteErrorLine($"*** ERROR: None of the hive nodes hosting HiveMQ appear to be online.");
return;
}
}
var manager = hive.GetReachableManager();
var response = (CommandResponse)null;
switch (state)
{
case "command":
if (!context.Arguments.TryGetValue("command", out var command))
{
throw new ArgumentException($"[command] module argument is required when [state=command].");
}
var commandArray = command as JArray;
if (commandArray == null)
{
throw new ArgumentException("[command] module argument must be an array specifying the command and arguments.");
}
if (commandArray.Count == 0)
{
throw new ArgumentException("[command] module argument must be specify either the [rabbitmqctl] or [rabbitmqadmin] command.");
}
var commandAndArgs = new List <string>();
foreach (var item in commandArray)
{
commandAndArgs.Add(item.ToString());
}
var rabbitCommand = commandAndArgs.First();
switch (rabbitCommand)
{
case "rabbitmqctl":
case "rabbitmqadmin":
break;
default:
throw new ArgumentException($"[{commandAndArgs.First()}] is not a valid command. Only [rabbitmqctl] or [rabbitmqadmin] are supported.");
}
response = node.SudoCommand($"docker exec neon-hivemq {rabbitCommand}", RunOptions.None, commandAndArgs.Skip(1));
if (response.ExitCode != 0)
{
context.WriteErrorLine(response.AllText);
}
else
{
using (var reader = new StringReader(response.OutputText))
{
foreach (var line in reader.Lines())
{
context.WriteLine(AnsibleVerbosity.Important, line);
}
}
}
context.Changed = true;
break;
default:
throw new ArgumentException($"[state={state}] is not one of the valid choices: [present], [absent], or [rollback].");
}
}
/// <inheritdoc/>
public void Run(ModuleContext context)
{
var hive = HiveHelper.Hive;
var nodeGroups = hive.Definition.GetHostGroups(excludeAllGroup: true);
//-----------------------------------------------------------------
// Parse the module arguments.
if (!context.ValidateArguments(context.Arguments, validModuleArgs))
{
context.Failed = true;
return;
}
var couchbaseArgs = CouchbaseArgs.Parse(context);
if (couchbaseArgs == null)
{
return;
}
var format = context.ParseEnum <CouchbaseFileFormat>("format");
if (!format.HasValue)
{
format = default(CouchbaseFileFormat);
}
var source = context.ParseString("source");
if (string.IsNullOrEmpty(source))
{
context.WriteErrorLine("[source] module parameter is required.");
return;
}
if (!File.Exists(source))
{
context.WriteErrorLine($"File [{source}] does not exist.");
return;
}
var keyPattern = context.ParseString("key");
var firstKey = context.ParseLong("first_key") ?? 1;
if (context.HasErrors)
{
return;
}
//-----------------------------------------------------------------
// Import the data.
using (var bucket = couchbaseArgs.Settings.OpenBucket(couchbaseArgs.Credentials))
{
var importer = new CouchbaseImporter(message => context.WriteErrorLine(message), bucket, keyPattern, firstKey, context.CheckMode);
switch (format.Value)
{
case CouchbaseFileFormat.JsonArray:
// $todo(jeff.lill):
//
// Would be nice not to read this whole thing in memory and then
// effectibely duplicating it in memory again when parsing.
var jToken = JToken.Parse(File.ReadAllText(source));
if (jToken.Type != JTokenType.Array)
{
context.WriteErrorLine($"[{source}] is not a JSON array of documents.");
return;
}
var jArray = (JArray)jToken;
foreach (var item in jArray)
{
if (item.Type != JTokenType.Object)
{
context.WriteErrorLine($"[{source}] includes one or more non-document objects in the array.");
return;
}
importer.WriteDocument((JObject)item);
}
break;
case CouchbaseFileFormat.JsonLines:
using (var reader = new StreamReader(source, Encoding.UTF8))
{
foreach (var line in reader.Lines())
{
if (line.Trim() == string.Empty)
{
continue; // Ignore blank lines
}
var item = JToken.Parse(line);
if (item.Type != JTokenType.Object)
{
context.WriteErrorLine($"[{source}] includes one or more lines with non-document objects.");
return;
}
importer.WriteDocument((JObject)item);
}
}
break;
default:
throw new NotImplementedException($"Format [{format}] is not implemented.");
}
context.Changed = importer.DocumentCount > 0;
if (context.CheckMode)
{
context.WriteLine(AnsibleVerbosity.Info, $"[{importer.DocumentCount}] documents will be added when CHECK-MODE is disabled.");
}
else
{
context.WriteLine(AnsibleVerbosity.Info, $"[{importer.DocumentCount}] documents were imported.");
}
}
}
/// <inheritdoc/>
public void Run(ModuleContext context)
{
var hive = HiveHelper.Hive;
var nodeGroups = hive.Definition.GetHostGroups(excludeAllGroup: true);
//-----------------------------------------------------------------
// Parse the module arguments.
if (!context.ValidateArguments(context.Arguments, validModuleArgs))
{
context.Failed = true;
return;
}
var couchbaseArgs = CouchbaseArgs.Parse(context);
if (couchbaseArgs == null)
{
return;
}
if (!context.Arguments.TryGetValue <string>("state", out var state))
{
state = "present";
}
state = state.ToLowerInvariant();
if (!context.Arguments.TryGetValue <bool>("force", out var force))
{
force = false;
}
var buildNow = context.ParseBool("build_all");
if (!buildNow.HasValue)
{
buildNow = false;
}
var primary = context.ParseBool("primary") ?? false;
string name;
if (primary)
{
name = "#primary";
}
else
{
name = context.ParseString("name", v => new Regex(@"[a-z][a-z0-0#_]*", RegexOptions.IgnoreCase).IsMatch(v));
}
if (string.IsNullOrEmpty(name) && state != "build")
{
context.WriteErrorLine("[name] argument is required.");
}
var type = (context.ParseEnum <IndexType>("using") ?? default(IndexType)).ToString().ToUpperInvariant();
var namespaceId = context.ParseString("namespace");
if (string.IsNullOrEmpty(namespaceId))
{
namespaceId = "default";
}
var keys = context.ParseStringArray("keys");
var where = context.ParseString("where");
var nodes = context.ParseStringArray("nodes");
var defer = context.ParseBool("build_defer") ?? false;
var wait = context.ParseBool("build_wait") ?? true;
var replicas = context.ParseInt("replicas");
if (state == "present")
{
if (primary)
{
if (keys.Count > 0)
{
context.WriteErrorLine("PRIMARY indexes do not allow any [keys].");
return;
}
if (!string.IsNullOrEmpty(where))
{
context.WriteErrorLine("PRIMARY indexes do not support the [where] clause.");
return;
}
}
else
{
if (keys.Count == 0)
{
context.WriteErrorLine("Non-PRIMARY indexes must specify at least one [key].");
return;
}
}
if (type == "GSI" && replicas.HasValue && nodes.Count > 0)
{
context.WriteErrorLine("Only one of [nodes] or [replicas] may be specified for GSI indexes.");
return;
}
}
string keyspace;
if (!string.IsNullOrEmpty(namespaceId) && namespaceId != "default")
{
keyspace = $"{namespaceId}:{couchbaseArgs.Settings.Bucket}";
}
else
{
keyspace = couchbaseArgs.Settings.Bucket;
}
if (context.HasErrors)
{
return;
}
//-----------------------------------------------------------------
// Perform the operation.
Task.Run(
async() =>
{
using (var bucket = couchbaseArgs.Settings.OpenBucket(couchbaseArgs.Credentials))
{
var indexId = $"{bucket.Name}.{name}";
// Fetch the index if it already exists.
var existingIndex = await bucket.GetIndexAsync(name);
if (existingIndex == null)
{
context.WriteLine(AnsibleVerbosity.Trace, $"Index [{name}] does not exist.");
}
else
{
context.WriteLine(AnsibleVerbosity.Trace, $"Index [{name}] exists.");
}
var existingIsPrimary = existingIndex != null && existingIndex.IsPrimary;
switch (state.ToLowerInvariant())
{
case "present":
// Generate the index creation query.
var sbCreateIndexCommand = new StringBuilder();
if (primary)
{
sbCreateIndexCommand.Append($"create primary index {CbHelper.LiteralName(name)} on {CbHelper.LiteralName(bucket.Name)}");
}
else
{
var sbKeys = new StringBuilder();
foreach (var key in keys)
{
sbKeys.AppendWithSeparator(key, ", ");
}
sbCreateIndexCommand.Append($"create index {CbHelper.LiteralName(name)} on {CbHelper.LiteralName(bucket.Name)} ( {sbKeys} )");
}
// Append the WHERE clause for non-PRIMARY indexes.
if (!primary && !string.IsNullOrEmpty(where))
{
// Ensure that the WHERE clause is surrounded by "( ... )".
if (!where.StartsWith("(") && !where.EndsWith(")"))
{
where = $"({where})";
}
// Now strip the parens off the where clause to be added
// to the query.
var queryWhere = where.Substring(1, where.Length - 2);
// Append the clause.
sbCreateIndexCommand.AppendWithSeparator($"where {queryWhere}");
}
// Append the USING clause.
sbCreateIndexCommand.AppendWithSeparator($"using {type}");
// Append the WITH clause for GSI indexes.
if (type == "GSI")
{
var sbWithSettings = new StringBuilder();
if (defer)
{
sbWithSettings.AppendWithSeparator("\"defer_build\":true", ", ");
}
context.WriteLine(AnsibleVerbosity.Trace, "Query for the hive nodes.");
var clusterNodes = await bucket.QuerySafeAsync <dynamic>("select nodes.name from system:nodes");
context.WriteLine(AnsibleVerbosity.Trace, $"Hive has [{clusterNodes.Count}] nodes.");
if ((!replicas.HasValue || replicas.Value == 0) && nodes.Count == 0)
{
// We're going to default to hosting GSI indexes explicitly
// on all nodes unless directed otherwise. We'll need query
// the database for the current nodes.
foreach (JObject node in clusterNodes)
{
nodes.Add((string)node.GetValue("name"));
}
}
else if (replicas.HasValue && replicas.Value > 0)
{
if (clusterNodes.Count <= replicas.Value)
{
context.WriteErrorLine($"[replicas={replicas.Value}] cannot equal or exceed the number of Couchbase nodes. [replicas={clusterNodes.Count - 1}] is the maximum allowed value for this hive.");
return;
}
}
if (nodes.Count > 0)
{
sbWithSettings.AppendWithSeparator("\"nodes\": [", ", ");
var first = true;
foreach (var server in nodes)
{
if (first)
{
first = false;
}
else
{
sbCreateIndexCommand.Append(",");
}
sbWithSettings.Append(CbHelper.Literal(server));
}
sbWithSettings.Append("]");
}
if (replicas.HasValue && type == "GSI")
{
sbWithSettings.AppendWithSeparator($"\"num_replica\":{CbHelper.Literal(replicas.Value)}", ", ");
}
if (sbWithSettings.Length > 0)
{
sbCreateIndexCommand.AppendWithSeparator($"with {{ {sbWithSettings} }}");
}
}
// Add or update the index.
if (existingIndex != null)
{
context.WriteLine(AnsibleVerbosity.Info, $"Index [{indexId}] already exists.");
// An index with this name already exists, so we'll compare its
// properties with the module parameters to determine whether we
// need to remove and recreate it.
var changed = false;
if (force)
{
changed = true;
context.WriteLine(AnsibleVerbosity.Info, "Rebuilding index because [force=yes].");
}
// Compare the old/new index types.
var orgType = existingIndex.Type.ToUpperInvariant();
if (!string.Equals(orgType, type, StringComparison.InvariantCultureIgnoreCase))
{
changed = true;
context.WriteLine(AnsibleVerbosity.Info, $"Rebuilding index because using changed from [{orgType}] to [{type}].");
}
// Compare the old/new index keys.
var keysChanged = false;
if (!primary)
{
var orgKeys = existingIndex.Keys;
if (orgKeys.Length != keys.Count)
{
keysChanged = true;
}
else
{
// This assumes that the order of the indexed keys doesn't
// matter.
var keysSet = new Dictionary <string, bool>(StringComparer.InvariantCultureIgnoreCase);
for (int i = 0; i < orgKeys.Length; i++)
{
keysSet[(string)orgKeys[i]] = false;
}
for (int i = 0; i < orgKeys.Length; i++)
{
keysSet[CbHelper.LiteralName(keys[i])] = true;
}
keysChanged = keysSet.Values.Count(k => !k) > 0;
}
if (keysChanged)
{
changed = true;
var sbOrgKeys = new StringBuilder();
var sbNewKeys = new StringBuilder();
foreach (string key in orgKeys)
{
sbOrgKeys.AppendWithSeparator(key, ", ");
}
foreach (string key in keys)
{
sbNewKeys.AppendWithSeparator(key, ", ");
}
context.WriteLine(AnsibleVerbosity.Info, $"Rebuilding index because keys changed from [{sbOrgKeys}] to [{sbNewKeys}].");
}
}
// Compare the filter condition.
var orgWhere = existingIndex.Where;
if (orgWhere != where)
{
changed = true;
context.WriteLine(AnsibleVerbosity.Info, $"Rebuilding index because where clause changed from [{orgWhere ?? string.Empty}] to [{where ?? string.Empty}].");
}
// We need to remove and recreate the index if it differs
// from what was requested.
if (changed)
{
if (context.CheckMode)
{
context.WriteLine(AnsibleVerbosity.Important, $"Index [{indexId}] will be rebuilt when CHECK-MODE is disabled.");
}
else
{
context.Changed = !context.CheckMode;
context.WriteLine(AnsibleVerbosity.Trace, $"Removing existing index [{indexId}].");
string dropCommand;
if (existingIsPrimary)
{
dropCommand = $"drop primary index on {CbHelper.LiteralName(bucket.Name)} using {orgType.ToUpperInvariant()}";
}
else
{
dropCommand = $"drop index {CbHelper.LiteralName(bucket.Name)}.{CbHelper.LiteralName(name)} using {orgType.ToUpperInvariant()}";
}
context.WriteLine(AnsibleVerbosity.Trace, $"DROP COMMAND: {dropCommand}");
await bucket.QuerySafeAsync <dynamic>(dropCommand);
context.WriteLine(AnsibleVerbosity.Trace, $"Dropped index [{indexId}].");
context.WriteLine(AnsibleVerbosity.Trace, $"CREATE COMMAND: {sbCreateIndexCommand}");
await bucket.QuerySafeAsync <dynamic>(sbCreateIndexCommand.ToString());
context.WriteLine(AnsibleVerbosity.Info, $"Created index [{indexId}].");
if (!defer && wait)
{
// Wait for the index to come online.
context.WriteLine(AnsibleVerbosity.Info, $"Waiting for index [{name}] to be built.");
await bucket.WaitForIndexAsync(name, "online");
context.WriteLine(AnsibleVerbosity.Info, $"Completed building index [{name}].");
}
}
}
else
{
context.WriteLine(AnsibleVerbosity.Trace, $"No changes detected for index [{indexId}].");
}
}
else
{
if (context.CheckMode)
{
context.WriteLine(AnsibleVerbosity.Important, $"Index [{indexId}] will be created when CHECK-MODE is disabled.");
context.WriteLine(AnsibleVerbosity.Trace, $"{sbCreateIndexCommand}");
}
else
{
context.Changed = true;
context.WriteLine(AnsibleVerbosity.Trace, $"Creating index.");
context.WriteLine(AnsibleVerbosity.Trace, $"CREATE COMMAND: {sbCreateIndexCommand}");
await bucket.QuerySafeAsync <dynamic>(sbCreateIndexCommand.ToString());
context.WriteLine(AnsibleVerbosity.Info, $"Created index [{indexId}].");
if (!defer && wait)
{
// Wait for the index to come online.
context.WriteLine(AnsibleVerbosity.Info, $"Waiting for index [{name}] to be built.");
await bucket.WaitForIndexAsync(name, "online");
context.WriteLine(AnsibleVerbosity.Info, $"Completed building index [{name}].");
}
}
}
break;
case "absent":
if (existingIndex != null)
{
if (context.CheckMode)
{
context.WriteLine(AnsibleVerbosity.Important, $"Index [{indexId}] will be dropped when CHECK-MODE is disabled.");
}
else
{
context.Changed = true;
context.WriteLine(AnsibleVerbosity.Info, $"Dropping index [{indexId}].");
string orgType = existingIndex.Type;
string dropCommand;
if (existingIsPrimary)
{
dropCommand = $"drop primary index on {CbHelper.LiteralName(bucket.Name)} using {orgType.ToUpperInvariant()}";
}
else
{
dropCommand = $"drop index {CbHelper.LiteralName(bucket.Name)}.{CbHelper.LiteralName(name)} using {orgType.ToUpperInvariant()}";
}
context.WriteLine(AnsibleVerbosity.Trace, $"COMMAND: {dropCommand}");
await bucket.QuerySafeAsync <dynamic>(dropCommand);
context.WriteLine(AnsibleVerbosity.Trace, $"Index [{indexId}] was dropped.");
}
}
else
{
context.WriteLine(AnsibleVerbosity.Important, $"Index [{indexId}] does not exist so there's no need to drop it.");
}
break;
case "build":
// List the names of the deferred GSI indexes.
var deferredIndexes = ((await bucket.ListIndexesAsync()).Where(index => index.State == "deferred" && index.Type == "gsi")).ToList();
context.WriteLine(AnsibleVerbosity.Info, $"[{deferredIndexes.Count}] deferred GSI indexes exist.");
if (deferredIndexes.Count == 0)
{
context.WriteLine(AnsibleVerbosity.Important, $"All GSI indexes have already been built.");
context.Changed = false;
return;
}
// Build the indexes (unless we're in CHECK-MODE).
var sbIndexList = new StringBuilder();
foreach (var deferredIndex in deferredIndexes)
{
sbIndexList.AppendWithSeparator($"{CbHelper.LiteralName(deferredIndex.Name)}", ", ");
}
if (context.CheckMode)
{
context.WriteLine(AnsibleVerbosity.Important, $"These GSI indexes will be built when CHECK-MODE is disabled: {sbIndexList}.");
context.Changed = false;
return;
}
var buildCommand = $"BUILD INDEX ON {CbHelper.LiteralName(bucket.Name)} ({sbIndexList})";
context.WriteLine(AnsibleVerbosity.Trace, $"BUILD COMMAND: {buildCommand}");
context.WriteLine(AnsibleVerbosity.Info, $"Building indexes: {sbIndexList}");
await bucket.QuerySafeAsync <dynamic>(buildCommand);
context.WriteLine(AnsibleVerbosity.Info, $"Build command submitted.");
context.Changed = true;
// The Couchbase BUILD INDEX command doesn't wait for the index
// building to complete so, we'll just spin until all of the
// indexes we're building are online.
if (wait)
{
context.WriteLine(AnsibleVerbosity.Info, $"Waiting for the indexes to be built.");
foreach (var deferredIndex in deferredIndexes)
{
await bucket.WaitForIndexAsync(deferredIndex.Name, "online");
}
context.WriteLine(AnsibleVerbosity.Info, $"Completed building [{deferredIndexes.Count}] indexes.");
}
break;
default:
throw new ArgumentException($"[state={state}] is not one of the valid choices: [present], [absent], or [build].");
}
}
}).Wait();
}
/// <inheritdoc/>
public void Run(ModuleContext context)
{
var hive = HiveHelper.Hive;
if (!context.ValidateArguments(context.Arguments, validModuleArgs))
{
context.Failed = true;
return;
}
// Obtain common arguments.
context.WriteLine(AnsibleVerbosity.Trace, $"Parsing [name]");
if (!context.Arguments.TryGetValue <string>("name", out var configName))
{
throw new ArgumentException($"[name] module argument is required.");
}
context.WriteLine(AnsibleVerbosity.Trace, $"Parsing [state]");
if (!context.Arguments.TryGetValue <string>("state", out var state))
{
state = "present";
}
state = state.ToLowerInvariant();
context.WriteLine(AnsibleVerbosity.Trace, $"Parsing [text]");
context.Arguments.TryGetValue <string>("text", out var configText);
context.Arguments.TryGetValue <string>("bytes", out var configBytes);
if (context.HasErrors)
{
return;
}
// We have the required arguments, so perform the operation.
context.WriteLine(AnsibleVerbosity.Trace, $"Inspecting [{configName}] config.");
var manager = hive.GetReachableManager();
var exists = hive.Docker.Config.Exists(configName);
var bytes = (byte[])null;
if (exists)
{
context.WriteLine(AnsibleVerbosity.Trace, $"{configName}] config exists.");
}
else
{
context.WriteLine(AnsibleVerbosity.Trace, $"[{configName}] config does not exist.");
}
switch (state)
{
case "absent":
if (exists)
{
context.Changed = !context.CheckMode;
if (context.CheckMode)
{
context.WriteLine(AnsibleVerbosity.Info, $"Config [{configName}] will be removed when CHECK-MODE is disabled.");
}
else
{
context.Changed = true;
context.WriteLine(AnsibleVerbosity.Trace, $"Removing config [{configName}].");
hive.Docker.Config.Remove(configName);
}
}
else
{
context.WriteLine(AnsibleVerbosity.Info, $"Config [{configName}] does not exist.");
}
break;
case "present":
if (configText == null && configBytes == null)
{
context.WriteErrorLine("One of the [text] or [bytes] module parameters is required.");
return;
}
else if (configText != null && configBytes != null)
{
context.WriteErrorLine("Only one of [text] or [bytes] can be specified.");
return;
}
if (configBytes != null)
{
try
{
bytes = Convert.FromBase64String(configBytes);
}
catch
{
context.WriteErrorLine("[bytes] is not a valid base-64 encoded value.");
return;
}
}
if (exists)
{
context.WriteLine(AnsibleVerbosity.Info, $"Config [{configName}] already exists.");
}
else
{
if (context.CheckMode)
{
context.WriteLine(AnsibleVerbosity.Info, $"Config [{configName}] will be created when CHECK-MODE is disabled.");
}
else
{
context.Changed = true;
context.WriteLine(AnsibleVerbosity.Trace, $"Creating config [{configName}].");
if (bytes != null)
{
hive.Docker.Config.Set(configName, bytes);
}
else
{
hive.Docker.Config.Set(configName, configText);
}
}
}
break;
default:
throw new ArgumentException($"[state={state}] is not one of the valid choices: [present] or [absent].");
}
}
/// <inheritdoc/>
public void Run(ModuleContext context)
{
TrafficManager trafficManager = null;
bool isPublic = false;
string name = null;
string ruleName = null;
bool deferUpdate = false;
if (!context.ValidateArguments(context.Arguments, validModuleArgs))
{
context.Failed = true;
return;
}
// Obtain common arguments.
context.WriteLine(AnsibleVerbosity.Trace, $"Parsing [state]");
if (!context.Arguments.TryGetValue <string>("state", out var state))
{
state = "present";
}
state = state.ToLowerInvariant();
if (context.HasErrors)
{
return;
}
context.WriteLine(AnsibleVerbosity.Trace, $"Parsing [name]");
if (!context.Arguments.TryGetValue <string>("name", out name))
{
throw new ArgumentException($"[name] module argument is required.");
}
switch (name)
{
case "private":
trafficManager = HiveHelper.Hive.PrivateTraffic;
isPublic = false;
break;
case "public":
trafficManager = HiveHelper.Hive.PublicTraffic;
isPublic = true;
break;
default:
throw new ArgumentException($"[name={name}] is not a one of the valid traffic manager names: [private] or [public].");
}
if (state == "present" || state == "absent")
{
context.WriteLine(AnsibleVerbosity.Trace, $"Parsing [rule_name]");
if (!context.Arguments.TryGetValue <string>("rule_name", out ruleName))
{
throw new ArgumentException($"[rule_name] module argument is required.");
}
if (!HiveDefinition.IsValidName(ruleName))
{
throw new ArgumentException($"[rule_name={ruleName}] is not a valid traffic manager rule name.");
}
context.WriteLine(AnsibleVerbosity.Trace, $"Parsing [defer_update]");
if (!context.Arguments.TryGetValue <bool>("defer_update", out deferUpdate))
{
deferUpdate = false;
}
}
// We have the required arguments, so perform the operation.
switch (state)
{
case "absent":
context.WriteLine(AnsibleVerbosity.Trace, $"Check if rule [{ruleName}] exists.");
if (trafficManager.GetRule(ruleName) != null)
{
context.WriteLine(AnsibleVerbosity.Trace, $"Rule [{ruleName}] does exist.");
context.WriteLine(AnsibleVerbosity.Info, $"Deleting rule [{ruleName}].");
if (context.CheckMode)
{
context.WriteLine(AnsibleVerbosity.Info, $"Rule [{ruleName}] will be deleted when CHECK-MODE is disabled.");
}
else
{
trafficManager.RemoveRule(ruleName, deferUpdate: deferUpdate);
context.WriteLine(AnsibleVerbosity.Trace, $"Rule [{ruleName}] deleted.");
context.Changed = true;
}
}
else
{
context.WriteLine(AnsibleVerbosity.Trace, $"Rule [{ruleName}] does not exist.");
}
break;
case "present":
context.WriteLine(AnsibleVerbosity.Trace, $"Parsing [rule]");
if (!context.Arguments.TryGetValue <JObject>("rule", out var routeObject))
{
throw new ArgumentException($"[rule] module argument is required when [state={state}].");
}
var ruleText = routeObject.ToString();
context.WriteLine(AnsibleVerbosity.Trace, "Parsing rule");
var newRule = TrafficRule.Parse(ruleText, strict: true);
context.WriteLine(AnsibleVerbosity.Trace, "Rule parsed successfully");
// Use the name argument if the deserialized rule doesn't
// have a name. This will make it easier on operators because
// they won't need to specify the name twice.
if (string.IsNullOrWhiteSpace(newRule.Name))
{
newRule.Name = ruleName;
}
// Ensure that the name passed as an argument and the
// name within the rule definition match.
if (!string.Equals(ruleName, newRule.Name, StringComparison.InvariantCultureIgnoreCase))
{
throw new ArgumentException($"The [rule_name={ruleName}] argument and the rule's [{nameof(TrafficRule.Name)}={newRule.Name}] property are not the same.");
}
context.WriteLine(AnsibleVerbosity.Trace, "Rule name matched.");
// Validate the rule.
context.WriteLine(AnsibleVerbosity.Trace, "Validating rule.");
var proxySettings = trafficManager.GetSettings();
var validationContext = new TrafficValidationContext(name, proxySettings);
// $hack(jeff.lill):
//
// This ensures that [proxySettings.Resolvers] is initialized with
// the built-in Docker DNS resolver.
proxySettings.Validate(validationContext);
// Load the TLS certificates into the validation context so we'll
// be able to verify that any referenced certificates mactually exist.
// $todo(jeff.lill):
//
// This code assumes that the operator is currently logged in with
// root Vault privileges. We'll have to do something else for
// non-root logins.
//
// One idea might be to save two versions of the certificates.
// The primary certificate with private key in Vault and then
// just the public certificate in Consul and then load just
// the public ones here.
//
// A good time to make this change might be when we convert to
// use the .NET X.509 certificate implementation.
if (!context.Login.HasVaultRootCredentials)
{
throw new ArgumentException("Access Denied: Root Vault credentials are required.");
}
context.WriteLine(AnsibleVerbosity.Trace, "Reading hive certificates.");
using (var vault = HiveHelper.OpenVault(Program.HiveLogin.VaultCredentials.RootToken))
{
// List the certificate key/names and then fetch each one
// to capture details like the expiration date and covered
// hostnames.
foreach (var certName in vault.ListAsync("neon-secret/cert").Result)
{
context.WriteLine(AnsibleVerbosity.Trace, $"Reading: {certName}");
var certificate = vault.ReadJsonAsync <TlsCertificate>(HiveHelper.GetVaultCertificateKey(certName)).Result;
validationContext.Certificates.Add(certName, certificate);
}
}
context.WriteLine(AnsibleVerbosity.Trace, $"[{validationContext.Certificates.Count}] hive certificates downloaded.");
// Actually perform the rule validation.
newRule.Validate(validationContext);
if (validationContext.HasErrors)
{
context.WriteLine(AnsibleVerbosity.Trace, $"[{validationContext.Errors.Count}] Route validation errors.");
foreach (var error in validationContext.Errors)
{
context.WriteLine(AnsibleVerbosity.Important, error);
context.WriteErrorLine(error);
}
context.Failed = true;
return;
}
context.WriteLine(AnsibleVerbosity.Trace, "Rule is valid.");
// Try reading any existing rule with this name and then determine
// whether the two versions of the rule are actually different.
context.WriteLine(AnsibleVerbosity.Trace, $"Looking for existing rule [{ruleName}]");
var existingRule = trafficManager.GetRule(ruleName);
var changed = false;
if (existingRule != null)
{
context.WriteLine(AnsibleVerbosity.Trace, $"Rule exists: checking for differences.");
// Normalize the new and existing rules so the JSON text comparision
// will work properly.
newRule.Normalize(isPublic);
existingRule.Normalize(isPublic);
changed = !NeonHelper.JsonEquals(newRule, existingRule);
if (changed)
{
context.WriteLine(AnsibleVerbosity.Trace, $"Rules are different.");
}
else
{
context.WriteLine(AnsibleVerbosity.Info, $"Rules are the same. No need to update.");
}
}
else
{
changed = true;
context.WriteLine(AnsibleVerbosity.Trace, $"Rule [name={ruleName}] does not exist.");
}
if (changed)
{
if (context.CheckMode)
{
context.WriteLine(AnsibleVerbosity.Info, $"Rule [{ruleName}] will be updated when CHECK-MODE is disabled.");
}
else
{
context.WriteLine(AnsibleVerbosity.Trace, $"Writing rule [{ruleName}].");
trafficManager.SetRule(newRule);
context.WriteLine(AnsibleVerbosity.Info, $"Rule updated.");
context.Changed = !context.CheckMode;
}
}
break;
case "update":
trafficManager.Update();
context.Changed = true;
context.WriteLine(AnsibleVerbosity.Info, $"Update signalled.");
break;
case "purge":
var purgeItems = context.ParseStringArray("purge_list");
var purgeCaseSensitive = context.ParseBool("purge_case_sensitive");
if (!purgeCaseSensitive.HasValue)
{
purgeCaseSensitive = false;
}
if (purgeItems.Count == 0)
{
context.WriteLine(AnsibleVerbosity.Important, $"[purge_list] is missing or empty.");
break;
}
trafficManager.Purge(purgeItems.ToArray(), caseSensitive: purgeCaseSensitive.Value);
context.Changed = true;
context.WriteLine(AnsibleVerbosity.Info, $"Purge request submitted.");
break;
default:
throw new ArgumentException($"[state={state}] is not one of the valid choices: [present], [absent], or [update].");
}
}
/// <summary>
/// Implements the built-in <b>neon_dashboard</b> module.
/// </summary>
/// <param name="context">The module context.</param>
public void Run(ModuleContext context)
{
var hive = HiveHelper.Hive;
var consul = HiveHelper.Consul;
if (!context.ValidateArguments(context.Arguments, validModuleArgs))
{
context.Failed = true;
return;
}
// Obtain common arguments.
context.WriteLine(AnsibleVerbosity.Trace, $"Parsing [name]");
if (!context.Arguments.TryGetValue <string>("name", out var name))
{
throw new ArgumentException($"[name] module argument is required.");
}
if (!HiveDefinition.IsValidName(name))
{
throw new ArgumentException($"[{name}] is not a valid dashboard name.");
}
context.WriteLine(AnsibleVerbosity.Trace, $"Parsing [state]");
if (!context.Arguments.TryGetValue <string>("state", out var state))
{
state = "present";
}
state = state.ToLowerInvariant();
if (context.HasErrors)
{
return;
}
// We have the required arguments, so perform the operation.
switch (state)
{
case "absent":
context.WriteLine(AnsibleVerbosity.Trace, $"Check if dashboard [{name}] exists.");
if (hive.Dashboard.Get(name) != null)
{
context.WriteLine(AnsibleVerbosity.Trace, $"Dashboard [{name}] already exists.");
if (context.CheckMode)
{
context.WriteLine(AnsibleVerbosity.Info, $"Dashboard [{name}] will be deleted when CHECK-MODE is disabled.");
}
else
{
context.WriteLine(AnsibleVerbosity.Info, $"Deleting dashboard [{name}].");
hive.Dashboard.Remove(name);
context.WriteLine(AnsibleVerbosity.Trace, $"Dashboard [{name}] deleted.");
context.Changed = true;
}
}
else
{
context.WriteLine(AnsibleVerbosity.Trace, $"Dashboard [{name}] does not exist.");
}
break;
case "present":
// Parse the PRESENT arguments.
context.WriteLine(AnsibleVerbosity.Trace, $"Parsing [url]");
if (!context.Arguments.TryGetValue <string>("url", out var url) && state == "present")
{
throw new ArgumentException($"[url] module argument is required when [state={state}].");
}
if (!Uri.TryCreate(url, UriKind.Absolute, out var urlParsed))
{
throw new ArgumentException($"[url={url}] is not valid.");
}
url = urlParsed.ToString();
context.Arguments.TryGetValue <string>("title", out var title);
context.Arguments.TryGetValue <string>("folder", out var folder);
context.Arguments.TryGetValue <string>("description", out var description);
if (context.HasErrors)
{
return;
}
// Build the dashboard definition from the arguments.
var newDashboard = new HiveDashboard()
{
Name = name,
Title = title,
Folder = folder,
Url = url,
Description = description
};
// Validate the dashboard.
context.WriteLine(AnsibleVerbosity.Trace, "Validating dashboard.");
var errors = newDashboard.Validate(hive.Definition);
if (errors.Count > 0)
{
context.WriteLine(AnsibleVerbosity.Trace, $"[{errors.Count}] dashboard validation errors.");
foreach (var error in errors)
{
context.WriteLine(AnsibleVerbosity.Important, error);
context.WriteErrorLine(error);
}
context.Failed = true;
return;
}
context.WriteLine(AnsibleVerbosity.Trace, "Dashboard is valid.");
// Try reading any existing dashboard with this name and then determine
// whether the two versions are actually different.
context.WriteLine(AnsibleVerbosity.Trace, $"Looking for existing dashboard [{name}]");
var existingDashboard = hive.Dashboard.Get(name);
var changed = false;
if (existingDashboard != null)
{
context.WriteLine(AnsibleVerbosity.Trace, $"Dashboard exists: checking for differences.");
changed = !NeonHelper.JsonEquals(newDashboard, existingDashboard);
if (changed)
{
context.WriteLine(AnsibleVerbosity.Trace, $"Dashboards are different.");
}
else
{
context.WriteLine(AnsibleVerbosity.Info, $"Dashboards are the same. No need to update.");
}
}
else
{
changed = true;
context.WriteLine(AnsibleVerbosity.Trace, $"Dashboard for [{name}] does not exist.");
}
if (changed)
{
if (context.CheckMode)
{
context.WriteLine(AnsibleVerbosity.Info, $"Dashboard [{name}] will be updated when CHECK-MODE is disabled.");
}
else
{
context.WriteLine(AnsibleVerbosity.Trace, $"Updating dashboard.");
hive.Dashboard.Set(newDashboard);
context.WriteLine(AnsibleVerbosity.Info, $"Dashboard updated.");
context.Changed = true;
}
context.CheckMode = !context.CheckMode;
}
break;
default:
throw new ArgumentException($"[state={state}] is not one of the valid choices: [present] or [absent].");
}
}