/// <inheritdoc/>
public void Run(ModuleContext context)
{
var hive = HiveHelper.Hive;
var nodeGroups = hive.Definition.GetHostGroups(excludeAllGroup: true);
//-----------------------------------------------------------------
// Parse the module arguments.
if (!context.ValidateArguments(context.Arguments, validModuleArgs))
{
context.Failed = true;
return;
}
var couchbaseArgs = CouchbaseArgs.Parse(context);
if (couchbaseArgs == null)
{
return;
}
if (!context.Arguments.TryGetValue <string>("state", out var state))
{
state = "present";
}
state = state.ToLowerInvariant();
if (!context.Arguments.TryGetValue <bool>("force", out var force))
{
force = false;
}
var buildNow = context.ParseBool("build_all");
if (!buildNow.HasValue)
{
buildNow = false;
}
var primary = context.ParseBool("primary") ?? false;
string name;
if (primary)
{
name = "#primary";
}
else
{
name = context.ParseString("name", v => new Regex(@"[a-z][a-z0-0#_]*", RegexOptions.IgnoreCase).IsMatch(v));
}
if (string.IsNullOrEmpty(name) && state != "build")
{
context.WriteErrorLine("[name] argument is required.");
}
var type = (context.ParseEnum <IndexType>("using") ?? default(IndexType)).ToString().ToUpperInvariant();
var namespaceId = context.ParseString("namespace");
if (string.IsNullOrEmpty(namespaceId))
{
namespaceId = "default";
}
var keys = context.ParseStringArray("keys");
var where = context.ParseString("where");
var nodes = context.ParseStringArray("nodes");
var defer = context.ParseBool("build_defer") ?? false;
var wait = context.ParseBool("build_wait") ?? true;
var replicas = context.ParseInt("replicas");
if (state == "present")
{
if (primary)
{
if (keys.Count > 0)
{
context.WriteErrorLine("PRIMARY indexes do not allow any [keys].");
return;
}
if (!string.IsNullOrEmpty(where))
{
context.WriteErrorLine("PRIMARY indexes do not support the [where] clause.");
return;
}
}
else
{
if (keys.Count == 0)
{
context.WriteErrorLine("Non-PRIMARY indexes must specify at least one [key].");
return;
}
}
if (type == "GSI" && replicas.HasValue && nodes.Count > 0)
{
context.WriteErrorLine("Only one of [nodes] or [replicas] may be specified for GSI indexes.");
return;
}
}
string keyspace;
if (!string.IsNullOrEmpty(namespaceId) && namespaceId != "default")
{
keyspace = $"{namespaceId}:{couchbaseArgs.Settings.Bucket}";
}
else
{
keyspace = couchbaseArgs.Settings.Bucket;
}
if (context.HasErrors)
{
return;
}
//-----------------------------------------------------------------
// Perform the operation.
Task.Run(
async() =>
{
using (var bucket = couchbaseArgs.Settings.OpenBucket(couchbaseArgs.Credentials))
{
var indexId = $"{bucket.Name}.{name}";
// Fetch the index if it already exists.
var existingIndex = await bucket.GetIndexAsync(name);
if (existingIndex == null)
{
context.WriteLine(AnsibleVerbosity.Trace, $"Index [{name}] does not exist.");
}
else
{
context.WriteLine(AnsibleVerbosity.Trace, $"Index [{name}] exists.");
}
var existingIsPrimary = existingIndex != null && existingIndex.IsPrimary;
switch (state.ToLowerInvariant())
{
case "present":
// Generate the index creation query.
var sbCreateIndexCommand = new StringBuilder();
if (primary)
{
sbCreateIndexCommand.Append($"create primary index {CbHelper.LiteralName(name)} on {CbHelper.LiteralName(bucket.Name)}");
}
else
{
var sbKeys = new StringBuilder();
foreach (var key in keys)
{
sbKeys.AppendWithSeparator(key, ", ");
}
sbCreateIndexCommand.Append($"create index {CbHelper.LiteralName(name)} on {CbHelper.LiteralName(bucket.Name)} ( {sbKeys} )");
}
// Append the WHERE clause for non-PRIMARY indexes.
if (!primary && !string.IsNullOrEmpty(where))
{
// Ensure that the WHERE clause is surrounded by "( ... )".
if (!where.StartsWith("(") && !where.EndsWith(")"))
{
where = $"({where})";
}
// Now strip the parens off the where clause to be added
// to the query.
var queryWhere = where.Substring(1, where.Length - 2);
// Append the clause.
sbCreateIndexCommand.AppendWithSeparator($"where {queryWhere}");
}
// Append the USING clause.
sbCreateIndexCommand.AppendWithSeparator($"using {type}");
// Append the WITH clause for GSI indexes.
if (type == "GSI")
{
var sbWithSettings = new StringBuilder();
if (defer)
{
sbWithSettings.AppendWithSeparator("\"defer_build\":true", ", ");
}
context.WriteLine(AnsibleVerbosity.Trace, "Query for the hive nodes.");
var clusterNodes = await bucket.QuerySafeAsync <dynamic>("select nodes.name from system:nodes");
context.WriteLine(AnsibleVerbosity.Trace, $"Hive has [{clusterNodes.Count}] nodes.");
if ((!replicas.HasValue || replicas.Value == 0) && nodes.Count == 0)
{
// We're going to default to hosting GSI indexes explicitly
// on all nodes unless directed otherwise. We'll need query
// the database for the current nodes.
foreach (JObject node in clusterNodes)
{
nodes.Add((string)node.GetValue("name"));
}
}
else if (replicas.HasValue && replicas.Value > 0)
{
if (clusterNodes.Count <= replicas.Value)
{
context.WriteErrorLine($"[replicas={replicas.Value}] cannot equal or exceed the number of Couchbase nodes. [replicas={clusterNodes.Count - 1}] is the maximum allowed value for this hive.");
return;
}
}
if (nodes.Count > 0)
{
sbWithSettings.AppendWithSeparator("\"nodes\": [", ", ");
var first = true;
foreach (var server in nodes)
{
if (first)
{
first = false;
}
else
{
sbCreateIndexCommand.Append(",");
}
sbWithSettings.Append(CbHelper.Literal(server));
}
sbWithSettings.Append("]");
}
if (replicas.HasValue && type == "GSI")
{
sbWithSettings.AppendWithSeparator($"\"num_replica\":{CbHelper.Literal(replicas.Value)}", ", ");
}
if (sbWithSettings.Length > 0)
{
sbCreateIndexCommand.AppendWithSeparator($"with {{ {sbWithSettings} }}");
}
}
// Add or update the index.
if (existingIndex != null)
{
context.WriteLine(AnsibleVerbosity.Info, $"Index [{indexId}] already exists.");
// An index with this name already exists, so we'll compare its
// properties with the module parameters to determine whether we
// need to remove and recreate it.
var changed = false;
if (force)
{
changed = true;
context.WriteLine(AnsibleVerbosity.Info, "Rebuilding index because [force=yes].");
}
// Compare the old/new index types.
var orgType = existingIndex.Type.ToUpperInvariant();
if (!string.Equals(orgType, type, StringComparison.InvariantCultureIgnoreCase))
{
changed = true;
context.WriteLine(AnsibleVerbosity.Info, $"Rebuilding index because using changed from [{orgType}] to [{type}].");
}
// Compare the old/new index keys.
var keysChanged = false;
if (!primary)
{
var orgKeys = existingIndex.Keys;
if (orgKeys.Length != keys.Count)
{
keysChanged = true;
}
else
{
// This assumes that the order of the indexed keys doesn't
// matter.
var keysSet = new Dictionary <string, bool>(StringComparer.InvariantCultureIgnoreCase);
for (int i = 0; i < orgKeys.Length; i++)
{
keysSet[(string)orgKeys[i]] = false;
}
for (int i = 0; i < orgKeys.Length; i++)
{
keysSet[CbHelper.LiteralName(keys[i])] = true;
}
keysChanged = keysSet.Values.Count(k => !k) > 0;
}
if (keysChanged)
{
changed = true;
var sbOrgKeys = new StringBuilder();
var sbNewKeys = new StringBuilder();
foreach (string key in orgKeys)
{
sbOrgKeys.AppendWithSeparator(key, ", ");
}
foreach (string key in keys)
{
sbNewKeys.AppendWithSeparator(key, ", ");
}
context.WriteLine(AnsibleVerbosity.Info, $"Rebuilding index because keys changed from [{sbOrgKeys}] to [{sbNewKeys}].");
}
}
// Compare the filter condition.
var orgWhere = existingIndex.Where;
if (orgWhere != where)
{
changed = true;
context.WriteLine(AnsibleVerbosity.Info, $"Rebuilding index because where clause changed from [{orgWhere ?? string.Empty}] to [{where ?? string.Empty}].");
}
// We need to remove and recreate the index if it differs
// from what was requested.
if (changed)
{
if (context.CheckMode)
{
context.WriteLine(AnsibleVerbosity.Important, $"Index [{indexId}] will be rebuilt when CHECK-MODE is disabled.");
}
else
{
context.Changed = !context.CheckMode;
context.WriteLine(AnsibleVerbosity.Trace, $"Removing existing index [{indexId}].");
string dropCommand;
if (existingIsPrimary)
{
dropCommand = $"drop primary index on {CbHelper.LiteralName(bucket.Name)} using {orgType.ToUpperInvariant()}";
}
else
{
dropCommand = $"drop index {CbHelper.LiteralName(bucket.Name)}.{CbHelper.LiteralName(name)} using {orgType.ToUpperInvariant()}";
}
context.WriteLine(AnsibleVerbosity.Trace, $"DROP COMMAND: {dropCommand}");
await bucket.QuerySafeAsync <dynamic>(dropCommand);
context.WriteLine(AnsibleVerbosity.Trace, $"Dropped index [{indexId}].");
context.WriteLine(AnsibleVerbosity.Trace, $"CREATE COMMAND: {sbCreateIndexCommand}");
await bucket.QuerySafeAsync <dynamic>(sbCreateIndexCommand.ToString());
context.WriteLine(AnsibleVerbosity.Info, $"Created index [{indexId}].");
if (!defer && wait)
{
// Wait for the index to come online.
context.WriteLine(AnsibleVerbosity.Info, $"Waiting for index [{name}] to be built.");
await bucket.WaitForIndexAsync(name, "online");
context.WriteLine(AnsibleVerbosity.Info, $"Completed building index [{name}].");
}
}
}
else
{
context.WriteLine(AnsibleVerbosity.Trace, $"No changes detected for index [{indexId}].");
}
}
else
{
if (context.CheckMode)
{
context.WriteLine(AnsibleVerbosity.Important, $"Index [{indexId}] will be created when CHECK-MODE is disabled.");
context.WriteLine(AnsibleVerbosity.Trace, $"{sbCreateIndexCommand}");
}
else
{
context.Changed = true;
context.WriteLine(AnsibleVerbosity.Trace, $"Creating index.");
context.WriteLine(AnsibleVerbosity.Trace, $"CREATE COMMAND: {sbCreateIndexCommand}");
await bucket.QuerySafeAsync <dynamic>(sbCreateIndexCommand.ToString());
context.WriteLine(AnsibleVerbosity.Info, $"Created index [{indexId}].");
if (!defer && wait)
{
// Wait for the index to come online.
context.WriteLine(AnsibleVerbosity.Info, $"Waiting for index [{name}] to be built.");
await bucket.WaitForIndexAsync(name, "online");
context.WriteLine(AnsibleVerbosity.Info, $"Completed building index [{name}].");
}
}
}
break;
case "absent":
if (existingIndex != null)
{
if (context.CheckMode)
{
context.WriteLine(AnsibleVerbosity.Important, $"Index [{indexId}] will be dropped when CHECK-MODE is disabled.");
}
else
{
context.Changed = true;
context.WriteLine(AnsibleVerbosity.Info, $"Dropping index [{indexId}].");
string orgType = existingIndex.Type;
string dropCommand;
if (existingIsPrimary)
{
dropCommand = $"drop primary index on {CbHelper.LiteralName(bucket.Name)} using {orgType.ToUpperInvariant()}";
}
else
{
dropCommand = $"drop index {CbHelper.LiteralName(bucket.Name)}.{CbHelper.LiteralName(name)} using {orgType.ToUpperInvariant()}";
}
context.WriteLine(AnsibleVerbosity.Trace, $"COMMAND: {dropCommand}");
await bucket.QuerySafeAsync <dynamic>(dropCommand);
context.WriteLine(AnsibleVerbosity.Trace, $"Index [{indexId}] was dropped.");
}
}
else
{
context.WriteLine(AnsibleVerbosity.Important, $"Index [{indexId}] does not exist so there's no need to drop it.");
}
break;
case "build":
// List the names of the deferred GSI indexes.
var deferredIndexes = ((await bucket.ListIndexesAsync()).Where(index => index.State == "deferred" && index.Type == "gsi")).ToList();
context.WriteLine(AnsibleVerbosity.Info, $"[{deferredIndexes.Count}] deferred GSI indexes exist.");
if (deferredIndexes.Count == 0)
{
context.WriteLine(AnsibleVerbosity.Important, $"All GSI indexes have already been built.");
context.Changed = false;
return;
}
// Build the indexes (unless we're in CHECK-MODE).
var sbIndexList = new StringBuilder();
foreach (var deferredIndex in deferredIndexes)
{
sbIndexList.AppendWithSeparator($"{CbHelper.LiteralName(deferredIndex.Name)}", ", ");
}
if (context.CheckMode)
{
context.WriteLine(AnsibleVerbosity.Important, $"These GSI indexes will be built when CHECK-MODE is disabled: {sbIndexList}.");
context.Changed = false;
return;
}
var buildCommand = $"BUILD INDEX ON {CbHelper.LiteralName(bucket.Name)} ({sbIndexList})";
context.WriteLine(AnsibleVerbosity.Trace, $"BUILD COMMAND: {buildCommand}");
context.WriteLine(AnsibleVerbosity.Info, $"Building indexes: {sbIndexList}");
await bucket.QuerySafeAsync <dynamic>(buildCommand);
context.WriteLine(AnsibleVerbosity.Info, $"Build command submitted.");
context.Changed = true;
// The Couchbase BUILD INDEX command doesn't wait for the index
// building to complete so, we'll just spin until all of the
// indexes we're building are online.
if (wait)
{
context.WriteLine(AnsibleVerbosity.Info, $"Waiting for the indexes to be built.");
foreach (var deferredIndex in deferredIndexes)
{
await bucket.WaitForIndexAsync(deferredIndex.Name, "online");
}
context.WriteLine(AnsibleVerbosity.Info, $"Completed building [{deferredIndexes.Count}] indexes.");
}
break;
default:
throw new ArgumentException($"[state={state}] is not one of the valid choices: [present], [absent], or [build].");
}
}
}).Wait();
}
/// <inheritdoc/>
public void Run(ModuleContext context)
{
TrafficManager trafficManager = null;
bool isPublic = false;
string name = null;
string ruleName = null;
bool deferUpdate = false;
if (!context.ValidateArguments(context.Arguments, validModuleArgs))
{
context.Failed = true;
return;
}
// Obtain common arguments.
context.WriteLine(AnsibleVerbosity.Trace, $"Parsing [state]");
if (!context.Arguments.TryGetValue <string>("state", out var state))
{
state = "present";
}
state = state.ToLowerInvariant();
if (context.HasErrors)
{
return;
}
context.WriteLine(AnsibleVerbosity.Trace, $"Parsing [name]");
if (!context.Arguments.TryGetValue <string>("name", out name))
{
throw new ArgumentException($"[name] module argument is required.");
}
switch (name)
{
case "private":
trafficManager = HiveHelper.Hive.PrivateTraffic;
isPublic = false;
break;
case "public":
trafficManager = HiveHelper.Hive.PublicTraffic;
isPublic = true;
break;
default:
throw new ArgumentException($"[name={name}] is not a one of the valid traffic manager names: [private] or [public].");
}
if (state == "present" || state == "absent")
{
context.WriteLine(AnsibleVerbosity.Trace, $"Parsing [rule_name]");
if (!context.Arguments.TryGetValue <string>("rule_name", out ruleName))
{
throw new ArgumentException($"[rule_name] module argument is required.");
}
if (!HiveDefinition.IsValidName(ruleName))
{
throw new ArgumentException($"[rule_name={ruleName}] is not a valid traffic manager rule name.");
}
context.WriteLine(AnsibleVerbosity.Trace, $"Parsing [defer_update]");
if (!context.Arguments.TryGetValue <bool>("defer_update", out deferUpdate))
{
deferUpdate = false;
}
}
// We have the required arguments, so perform the operation.
switch (state)
{
case "absent":
context.WriteLine(AnsibleVerbosity.Trace, $"Check if rule [{ruleName}] exists.");
if (trafficManager.GetRule(ruleName) != null)
{
context.WriteLine(AnsibleVerbosity.Trace, $"Rule [{ruleName}] does exist.");
context.WriteLine(AnsibleVerbosity.Info, $"Deleting rule [{ruleName}].");
if (context.CheckMode)
{
context.WriteLine(AnsibleVerbosity.Info, $"Rule [{ruleName}] will be deleted when CHECK-MODE is disabled.");
}
else
{
trafficManager.RemoveRule(ruleName, deferUpdate: deferUpdate);
context.WriteLine(AnsibleVerbosity.Trace, $"Rule [{ruleName}] deleted.");
context.Changed = true;
}
}
else
{
context.WriteLine(AnsibleVerbosity.Trace, $"Rule [{ruleName}] does not exist.");
}
break;
case "present":
context.WriteLine(AnsibleVerbosity.Trace, $"Parsing [rule]");
if (!context.Arguments.TryGetValue <JObject>("rule", out var routeObject))
{
throw new ArgumentException($"[rule] module argument is required when [state={state}].");
}
var ruleText = routeObject.ToString();
context.WriteLine(AnsibleVerbosity.Trace, "Parsing rule");
var newRule = TrafficRule.Parse(ruleText, strict: true);
context.WriteLine(AnsibleVerbosity.Trace, "Rule parsed successfully");
// Use the name argument if the deserialized rule doesn't
// have a name. This will make it easier on operators because
// they won't need to specify the name twice.
if (string.IsNullOrWhiteSpace(newRule.Name))
{
newRule.Name = ruleName;
}
// Ensure that the name passed as an argument and the
// name within the rule definition match.
if (!string.Equals(ruleName, newRule.Name, StringComparison.InvariantCultureIgnoreCase))
{
throw new ArgumentException($"The [rule_name={ruleName}] argument and the rule's [{nameof(TrafficRule.Name)}={newRule.Name}] property are not the same.");
}
context.WriteLine(AnsibleVerbosity.Trace, "Rule name matched.");
// Validate the rule.
context.WriteLine(AnsibleVerbosity.Trace, "Validating rule.");
var proxySettings = trafficManager.GetSettings();
var validationContext = new TrafficValidationContext(name, proxySettings);
// $hack(jeff.lill):
//
// This ensures that [proxySettings.Resolvers] is initialized with
// the built-in Docker DNS resolver.
proxySettings.Validate(validationContext);
// Load the TLS certificates into the validation context so we'll
// be able to verify that any referenced certificates mactually exist.
// $todo(jeff.lill):
//
// This code assumes that the operator is currently logged in with
// root Vault privileges. We'll have to do something else for
// non-root logins.
//
// One idea might be to save two versions of the certificates.
// The primary certificate with private key in Vault and then
// just the public certificate in Consul and then load just
// the public ones here.
//
// A good time to make this change might be when we convert to
// use the .NET X.509 certificate implementation.
if (!context.Login.HasVaultRootCredentials)
{
throw new ArgumentException("Access Denied: Root Vault credentials are required.");
}
context.WriteLine(AnsibleVerbosity.Trace, "Reading hive certificates.");
using (var vault = HiveHelper.OpenVault(Program.HiveLogin.VaultCredentials.RootToken))
{
// List the certificate key/names and then fetch each one
// to capture details like the expiration date and covered
// hostnames.
foreach (var certName in vault.ListAsync("neon-secret/cert").Result)
{
context.WriteLine(AnsibleVerbosity.Trace, $"Reading: {certName}");
var certificate = vault.ReadJsonAsync <TlsCertificate>(HiveHelper.GetVaultCertificateKey(certName)).Result;
validationContext.Certificates.Add(certName, certificate);
}
}
context.WriteLine(AnsibleVerbosity.Trace, $"[{validationContext.Certificates.Count}] hive certificates downloaded.");
// Actually perform the rule validation.
newRule.Validate(validationContext);
if (validationContext.HasErrors)
{
context.WriteLine(AnsibleVerbosity.Trace, $"[{validationContext.Errors.Count}] Route validation errors.");
foreach (var error in validationContext.Errors)
{
context.WriteLine(AnsibleVerbosity.Important, error);
context.WriteErrorLine(error);
}
context.Failed = true;
return;
}
context.WriteLine(AnsibleVerbosity.Trace, "Rule is valid.");
// Try reading any existing rule with this name and then determine
// whether the two versions of the rule are actually different.
context.WriteLine(AnsibleVerbosity.Trace, $"Looking for existing rule [{ruleName}]");
var existingRule = trafficManager.GetRule(ruleName);
var changed = false;
if (existingRule != null)
{
context.WriteLine(AnsibleVerbosity.Trace, $"Rule exists: checking for differences.");
// Normalize the new and existing rules so the JSON text comparision
// will work properly.
newRule.Normalize(isPublic);
existingRule.Normalize(isPublic);
changed = !NeonHelper.JsonEquals(newRule, existingRule);
if (changed)
{
context.WriteLine(AnsibleVerbosity.Trace, $"Rules are different.");
}
else
{
context.WriteLine(AnsibleVerbosity.Info, $"Rules are the same. No need to update.");
}
}
else
{
changed = true;
context.WriteLine(AnsibleVerbosity.Trace, $"Rule [name={ruleName}] does not exist.");
}
if (changed)
{
if (context.CheckMode)
{
context.WriteLine(AnsibleVerbosity.Info, $"Rule [{ruleName}] will be updated when CHECK-MODE is disabled.");
}
else
{
context.WriteLine(AnsibleVerbosity.Trace, $"Writing rule [{ruleName}].");
trafficManager.SetRule(newRule);
context.WriteLine(AnsibleVerbosity.Info, $"Rule updated.");
context.Changed = !context.CheckMode;
}
}
break;
case "update":
trafficManager.Update();
context.Changed = true;
context.WriteLine(AnsibleVerbosity.Info, $"Update signalled.");
break;
case "purge":
var purgeItems = context.ParseStringArray("purge_list");
var purgeCaseSensitive = context.ParseBool("purge_case_sensitive");
if (!purgeCaseSensitive.HasValue)
{
purgeCaseSensitive = false;
}
if (purgeItems.Count == 0)
{
context.WriteLine(AnsibleVerbosity.Important, $"[purge_list] is missing or empty.");
break;
}
trafficManager.Purge(purgeItems.ToArray(), caseSensitive: purgeCaseSensitive.Value);
context.Changed = true;
context.WriteLine(AnsibleVerbosity.Info, $"Purge request submitted.");
break;
default:
throw new ArgumentException($"[state={state}] is not one of the valid choices: [present], [absent], or [update].");
}
}