/// <inheritdoc/> public void Run(ModuleContext context) { var hive = HiveHelper.Hive; var nodeGroups = hive.Definition.GetHostGroups(excludeAllGroup: true); //----------------------------------------------------------------- // Parse the module arguments. if (!context.ValidateArguments(context.Arguments, validModuleArgs)) { context.Failed = true; return; } var couchbaseArgs = CouchbaseArgs.Parse(context); if (couchbaseArgs == null) { return; } if (!context.Arguments.TryGetValue <string>("state", out var state)) { state = "present"; } state = state.ToLowerInvariant(); if (!context.Arguments.TryGetValue <bool>("force", out var force)) { force = false; } var buildNow = context.ParseBool("build_all"); if (!buildNow.HasValue) { buildNow = false; } var primary = context.ParseBool("primary") ?? false; string name; if (primary) { name = "#primary"; } else { name = context.ParseString("name", v => new Regex(@"[a-z][a-z0-0#_]*", RegexOptions.IgnoreCase).IsMatch(v)); } if (string.IsNullOrEmpty(name) && state != "build") { context.WriteErrorLine("[name] argument is required."); } var type = (context.ParseEnum <IndexType>("using") ?? default(IndexType)).ToString().ToUpperInvariant(); var namespaceId = context.ParseString("namespace"); if (string.IsNullOrEmpty(namespaceId)) { namespaceId = "default"; } var keys = context.ParseStringArray("keys"); var where = context.ParseString("where"); var nodes = context.ParseStringArray("nodes"); var defer = context.ParseBool("build_defer") ?? false; var wait = context.ParseBool("build_wait") ?? true; var replicas = context.ParseInt("replicas"); if (state == "present") { if (primary) { if (keys.Count > 0) { context.WriteErrorLine("PRIMARY indexes do not allow any [keys]."); return; } if (!string.IsNullOrEmpty(where)) { context.WriteErrorLine("PRIMARY indexes do not support the [where] clause."); return; } } else { if (keys.Count == 0) { context.WriteErrorLine("Non-PRIMARY indexes must specify at least one [key]."); return; } } if (type == "GSI" && replicas.HasValue && nodes.Count > 0) { context.WriteErrorLine("Only one of [nodes] or [replicas] may be specified for GSI indexes."); return; } } string keyspace; if (!string.IsNullOrEmpty(namespaceId) && namespaceId != "default") { keyspace = $"{namespaceId}:{couchbaseArgs.Settings.Bucket}"; } else { keyspace = couchbaseArgs.Settings.Bucket; } if (context.HasErrors) { return; } //----------------------------------------------------------------- // Perform the operation. Task.Run( async() => { using (var bucket = couchbaseArgs.Settings.OpenBucket(couchbaseArgs.Credentials)) { var indexId = $"{bucket.Name}.{name}"; // Fetch the index if it already exists. var existingIndex = await bucket.GetIndexAsync(name); if (existingIndex == null) { context.WriteLine(AnsibleVerbosity.Trace, $"Index [{name}] does not exist."); } else { context.WriteLine(AnsibleVerbosity.Trace, $"Index [{name}] exists."); } var existingIsPrimary = existingIndex != null && existingIndex.IsPrimary; switch (state.ToLowerInvariant()) { case "present": // Generate the index creation query. var sbCreateIndexCommand = new StringBuilder(); if (primary) { sbCreateIndexCommand.Append($"create primary index {CbHelper.LiteralName(name)} on {CbHelper.LiteralName(bucket.Name)}"); } else { var sbKeys = new StringBuilder(); foreach (var key in keys) { sbKeys.AppendWithSeparator(key, ", "); } sbCreateIndexCommand.Append($"create index {CbHelper.LiteralName(name)} on {CbHelper.LiteralName(bucket.Name)} ( {sbKeys} )"); } // Append the WHERE clause for non-PRIMARY indexes. if (!primary && !string.IsNullOrEmpty(where)) { // Ensure that the WHERE clause is surrounded by "( ... )". if (!where.StartsWith("(") && !where.EndsWith(")")) { where = $"({where})"; } // Now strip the parens off the where clause to be added // to the query. var queryWhere = where.Substring(1, where.Length - 2); // Append the clause. sbCreateIndexCommand.AppendWithSeparator($"where {queryWhere}"); } // Append the USING clause. sbCreateIndexCommand.AppendWithSeparator($"using {type}"); // Append the WITH clause for GSI indexes. if (type == "GSI") { var sbWithSettings = new StringBuilder(); if (defer) { sbWithSettings.AppendWithSeparator("\"defer_build\":true", ", "); } context.WriteLine(AnsibleVerbosity.Trace, "Query for the hive nodes."); var clusterNodes = await bucket.QuerySafeAsync <dynamic>("select nodes.name from system:nodes"); context.WriteLine(AnsibleVerbosity.Trace, $"Hive has [{clusterNodes.Count}] nodes."); if ((!replicas.HasValue || replicas.Value == 0) && nodes.Count == 0) { // We're going to default to hosting GSI indexes explicitly // on all nodes unless directed otherwise. We'll need query // the database for the current nodes. foreach (JObject node in clusterNodes) { nodes.Add((string)node.GetValue("name")); } } else if (replicas.HasValue && replicas.Value > 0) { if (clusterNodes.Count <= replicas.Value) { context.WriteErrorLine($"[replicas={replicas.Value}] cannot equal or exceed the number of Couchbase nodes. [replicas={clusterNodes.Count - 1}] is the maximum allowed value for this hive."); return; } } if (nodes.Count > 0) { sbWithSettings.AppendWithSeparator("\"nodes\": [", ", "); var first = true; foreach (var server in nodes) { if (first) { first = false; } else { sbCreateIndexCommand.Append(","); } sbWithSettings.Append(CbHelper.Literal(server)); } sbWithSettings.Append("]"); } if (replicas.HasValue && type == "GSI") { sbWithSettings.AppendWithSeparator($"\"num_replica\":{CbHelper.Literal(replicas.Value)}", ", "); } if (sbWithSettings.Length > 0) { sbCreateIndexCommand.AppendWithSeparator($"with {{ {sbWithSettings} }}"); } } // Add or update the index. if (existingIndex != null) { context.WriteLine(AnsibleVerbosity.Info, $"Index [{indexId}] already exists."); // An index with this name already exists, so we'll compare its // properties with the module parameters to determine whether we // need to remove and recreate it. var changed = false; if (force) { changed = true; context.WriteLine(AnsibleVerbosity.Info, "Rebuilding index because [force=yes]."); } // Compare the old/new index types. var orgType = existingIndex.Type.ToUpperInvariant(); if (!string.Equals(orgType, type, StringComparison.InvariantCultureIgnoreCase)) { changed = true; context.WriteLine(AnsibleVerbosity.Info, $"Rebuilding index because using changed from [{orgType}] to [{type}]."); } // Compare the old/new index keys. var keysChanged = false; if (!primary) { var orgKeys = existingIndex.Keys; if (orgKeys.Length != keys.Count) { keysChanged = true; } else { // This assumes that the order of the indexed keys doesn't // matter. var keysSet = new Dictionary <string, bool>(StringComparer.InvariantCultureIgnoreCase); for (int i = 0; i < orgKeys.Length; i++) { keysSet[(string)orgKeys[i]] = false; } for (int i = 0; i < orgKeys.Length; i++) { keysSet[CbHelper.LiteralName(keys[i])] = true; } keysChanged = keysSet.Values.Count(k => !k) > 0; } if (keysChanged) { changed = true; var sbOrgKeys = new StringBuilder(); var sbNewKeys = new StringBuilder(); foreach (string key in orgKeys) { sbOrgKeys.AppendWithSeparator(key, ", "); } foreach (string key in keys) { sbNewKeys.AppendWithSeparator(key, ", "); } context.WriteLine(AnsibleVerbosity.Info, $"Rebuilding index because keys changed from [{sbOrgKeys}] to [{sbNewKeys}]."); } } // Compare the filter condition. var orgWhere = existingIndex.Where; if (orgWhere != where) { changed = true; context.WriteLine(AnsibleVerbosity.Info, $"Rebuilding index because where clause changed from [{orgWhere ?? string.Empty}] to [{where ?? string.Empty}]."); } // We need to remove and recreate the index if it differs // from what was requested. if (changed) { if (context.CheckMode) { context.WriteLine(AnsibleVerbosity.Important, $"Index [{indexId}] will be rebuilt when CHECK-MODE is disabled."); } else { context.Changed = !context.CheckMode; context.WriteLine(AnsibleVerbosity.Trace, $"Removing existing index [{indexId}]."); string dropCommand; if (existingIsPrimary) { dropCommand = $"drop primary index on {CbHelper.LiteralName(bucket.Name)} using {orgType.ToUpperInvariant()}"; } else { dropCommand = $"drop index {CbHelper.LiteralName(bucket.Name)}.{CbHelper.LiteralName(name)} using {orgType.ToUpperInvariant()}"; } context.WriteLine(AnsibleVerbosity.Trace, $"DROP COMMAND: {dropCommand}"); await bucket.QuerySafeAsync <dynamic>(dropCommand); context.WriteLine(AnsibleVerbosity.Trace, $"Dropped index [{indexId}]."); context.WriteLine(AnsibleVerbosity.Trace, $"CREATE COMMAND: {sbCreateIndexCommand}"); await bucket.QuerySafeAsync <dynamic>(sbCreateIndexCommand.ToString()); context.WriteLine(AnsibleVerbosity.Info, $"Created index [{indexId}]."); if (!defer && wait) { // Wait for the index to come online. context.WriteLine(AnsibleVerbosity.Info, $"Waiting for index [{name}] to be built."); await bucket.WaitForIndexAsync(name, "online"); context.WriteLine(AnsibleVerbosity.Info, $"Completed building index [{name}]."); } } } else { context.WriteLine(AnsibleVerbosity.Trace, $"No changes detected for index [{indexId}]."); } } else { if (context.CheckMode) { context.WriteLine(AnsibleVerbosity.Important, $"Index [{indexId}] will be created when CHECK-MODE is disabled."); context.WriteLine(AnsibleVerbosity.Trace, $"{sbCreateIndexCommand}"); } else { context.Changed = true; context.WriteLine(AnsibleVerbosity.Trace, $"Creating index."); context.WriteLine(AnsibleVerbosity.Trace, $"CREATE COMMAND: {sbCreateIndexCommand}"); await bucket.QuerySafeAsync <dynamic>(sbCreateIndexCommand.ToString()); context.WriteLine(AnsibleVerbosity.Info, $"Created index [{indexId}]."); if (!defer && wait) { // Wait for the index to come online. context.WriteLine(AnsibleVerbosity.Info, $"Waiting for index [{name}] to be built."); await bucket.WaitForIndexAsync(name, "online"); context.WriteLine(AnsibleVerbosity.Info, $"Completed building index [{name}]."); } } } break; case "absent": if (existingIndex != null) { if (context.CheckMode) { context.WriteLine(AnsibleVerbosity.Important, $"Index [{indexId}] will be dropped when CHECK-MODE is disabled."); } else { context.Changed = true; context.WriteLine(AnsibleVerbosity.Info, $"Dropping index [{indexId}]."); string orgType = existingIndex.Type; string dropCommand; if (existingIsPrimary) { dropCommand = $"drop primary index on {CbHelper.LiteralName(bucket.Name)} using {orgType.ToUpperInvariant()}"; } else { dropCommand = $"drop index {CbHelper.LiteralName(bucket.Name)}.{CbHelper.LiteralName(name)} using {orgType.ToUpperInvariant()}"; } context.WriteLine(AnsibleVerbosity.Trace, $"COMMAND: {dropCommand}"); await bucket.QuerySafeAsync <dynamic>(dropCommand); context.WriteLine(AnsibleVerbosity.Trace, $"Index [{indexId}] was dropped."); } } else { context.WriteLine(AnsibleVerbosity.Important, $"Index [{indexId}] does not exist so there's no need to drop it."); } break; case "build": // List the names of the deferred GSI indexes. var deferredIndexes = ((await bucket.ListIndexesAsync()).Where(index => index.State == "deferred" && index.Type == "gsi")).ToList(); context.WriteLine(AnsibleVerbosity.Info, $"[{deferredIndexes.Count}] deferred GSI indexes exist."); if (deferredIndexes.Count == 0) { context.WriteLine(AnsibleVerbosity.Important, $"All GSI indexes have already been built."); context.Changed = false; return; } // Build the indexes (unless we're in CHECK-MODE). var sbIndexList = new StringBuilder(); foreach (var deferredIndex in deferredIndexes) { sbIndexList.AppendWithSeparator($"{CbHelper.LiteralName(deferredIndex.Name)}", ", "); } if (context.CheckMode) { context.WriteLine(AnsibleVerbosity.Important, $"These GSI indexes will be built when CHECK-MODE is disabled: {sbIndexList}."); context.Changed = false; return; } var buildCommand = $"BUILD INDEX ON {CbHelper.LiteralName(bucket.Name)} ({sbIndexList})"; context.WriteLine(AnsibleVerbosity.Trace, $"BUILD COMMAND: {buildCommand}"); context.WriteLine(AnsibleVerbosity.Info, $"Building indexes: {sbIndexList}"); await bucket.QuerySafeAsync <dynamic>(buildCommand); context.WriteLine(AnsibleVerbosity.Info, $"Build command submitted."); context.Changed = true; // The Couchbase BUILD INDEX command doesn't wait for the index // building to complete so, we'll just spin until all of the // indexes we're building are online. if (wait) { context.WriteLine(AnsibleVerbosity.Info, $"Waiting for the indexes to be built."); foreach (var deferredIndex in deferredIndexes) { await bucket.WaitForIndexAsync(deferredIndex.Name, "online"); } context.WriteLine(AnsibleVerbosity.Info, $"Completed building [{deferredIndexes.Count}] indexes."); } break; default: throw new ArgumentException($"[state={state}] is not one of the valid choices: [present], [absent], or [build]."); } } }).Wait(); }
/// <inheritdoc/> public void Run(ModuleContext context) { TrafficManager trafficManager = null; bool isPublic = false; string name = null; string ruleName = null; bool deferUpdate = false; if (!context.ValidateArguments(context.Arguments, validModuleArgs)) { context.Failed = true; return; } // Obtain common arguments. context.WriteLine(AnsibleVerbosity.Trace, $"Parsing [state]"); if (!context.Arguments.TryGetValue <string>("state", out var state)) { state = "present"; } state = state.ToLowerInvariant(); if (context.HasErrors) { return; } context.WriteLine(AnsibleVerbosity.Trace, $"Parsing [name]"); if (!context.Arguments.TryGetValue <string>("name", out name)) { throw new ArgumentException($"[name] module argument is required."); } switch (name) { case "private": trafficManager = HiveHelper.Hive.PrivateTraffic; isPublic = false; break; case "public": trafficManager = HiveHelper.Hive.PublicTraffic; isPublic = true; break; default: throw new ArgumentException($"[name={name}] is not a one of the valid traffic manager names: [private] or [public]."); } if (state == "present" || state == "absent") { context.WriteLine(AnsibleVerbosity.Trace, $"Parsing [rule_name]"); if (!context.Arguments.TryGetValue <string>("rule_name", out ruleName)) { throw new ArgumentException($"[rule_name] module argument is required."); } if (!HiveDefinition.IsValidName(ruleName)) { throw new ArgumentException($"[rule_name={ruleName}] is not a valid traffic manager rule name."); } context.WriteLine(AnsibleVerbosity.Trace, $"Parsing [defer_update]"); if (!context.Arguments.TryGetValue <bool>("defer_update", out deferUpdate)) { deferUpdate = false; } } // We have the required arguments, so perform the operation. switch (state) { case "absent": context.WriteLine(AnsibleVerbosity.Trace, $"Check if rule [{ruleName}] exists."); if (trafficManager.GetRule(ruleName) != null) { context.WriteLine(AnsibleVerbosity.Trace, $"Rule [{ruleName}] does exist."); context.WriteLine(AnsibleVerbosity.Info, $"Deleting rule [{ruleName}]."); if (context.CheckMode) { context.WriteLine(AnsibleVerbosity.Info, $"Rule [{ruleName}] will be deleted when CHECK-MODE is disabled."); } else { trafficManager.RemoveRule(ruleName, deferUpdate: deferUpdate); context.WriteLine(AnsibleVerbosity.Trace, $"Rule [{ruleName}] deleted."); context.Changed = true; } } else { context.WriteLine(AnsibleVerbosity.Trace, $"Rule [{ruleName}] does not exist."); } break; case "present": context.WriteLine(AnsibleVerbosity.Trace, $"Parsing [rule]"); if (!context.Arguments.TryGetValue <JObject>("rule", out var routeObject)) { throw new ArgumentException($"[rule] module argument is required when [state={state}]."); } var ruleText = routeObject.ToString(); context.WriteLine(AnsibleVerbosity.Trace, "Parsing rule"); var newRule = TrafficRule.Parse(ruleText, strict: true); context.WriteLine(AnsibleVerbosity.Trace, "Rule parsed successfully"); // Use the name argument if the deserialized rule doesn't // have a name. This will make it easier on operators because // they won't need to specify the name twice. if (string.IsNullOrWhiteSpace(newRule.Name)) { newRule.Name = ruleName; } // Ensure that the name passed as an argument and the // name within the rule definition match. if (!string.Equals(ruleName, newRule.Name, StringComparison.InvariantCultureIgnoreCase)) { throw new ArgumentException($"The [rule_name={ruleName}] argument and the rule's [{nameof(TrafficRule.Name)}={newRule.Name}] property are not the same."); } context.WriteLine(AnsibleVerbosity.Trace, "Rule name matched."); // Validate the rule. context.WriteLine(AnsibleVerbosity.Trace, "Validating rule."); var proxySettings = trafficManager.GetSettings(); var validationContext = new TrafficValidationContext(name, proxySettings); // $hack(jeff.lill): // // This ensures that [proxySettings.Resolvers] is initialized with // the built-in Docker DNS resolver. proxySettings.Validate(validationContext); // Load the TLS certificates into the validation context so we'll // be able to verify that any referenced certificates mactually exist. // $todo(jeff.lill): // // This code assumes that the operator is currently logged in with // root Vault privileges. We'll have to do something else for // non-root logins. // // One idea might be to save two versions of the certificates. // The primary certificate with private key in Vault and then // just the public certificate in Consul and then load just // the public ones here. // // A good time to make this change might be when we convert to // use the .NET X.509 certificate implementation. if (!context.Login.HasVaultRootCredentials) { throw new ArgumentException("Access Denied: Root Vault credentials are required."); } context.WriteLine(AnsibleVerbosity.Trace, "Reading hive certificates."); using (var vault = HiveHelper.OpenVault(Program.HiveLogin.VaultCredentials.RootToken)) { // List the certificate key/names and then fetch each one // to capture details like the expiration date and covered // hostnames. foreach (var certName in vault.ListAsync("neon-secret/cert").Result) { context.WriteLine(AnsibleVerbosity.Trace, $"Reading: {certName}"); var certificate = vault.ReadJsonAsync <TlsCertificate>(HiveHelper.GetVaultCertificateKey(certName)).Result; validationContext.Certificates.Add(certName, certificate); } } context.WriteLine(AnsibleVerbosity.Trace, $"[{validationContext.Certificates.Count}] hive certificates downloaded."); // Actually perform the rule validation. newRule.Validate(validationContext); if (validationContext.HasErrors) { context.WriteLine(AnsibleVerbosity.Trace, $"[{validationContext.Errors.Count}] Route validation errors."); foreach (var error in validationContext.Errors) { context.WriteLine(AnsibleVerbosity.Important, error); context.WriteErrorLine(error); } context.Failed = true; return; } context.WriteLine(AnsibleVerbosity.Trace, "Rule is valid."); // Try reading any existing rule with this name and then determine // whether the two versions of the rule are actually different. context.WriteLine(AnsibleVerbosity.Trace, $"Looking for existing rule [{ruleName}]"); var existingRule = trafficManager.GetRule(ruleName); var changed = false; if (existingRule != null) { context.WriteLine(AnsibleVerbosity.Trace, $"Rule exists: checking for differences."); // Normalize the new and existing rules so the JSON text comparision // will work properly. newRule.Normalize(isPublic); existingRule.Normalize(isPublic); changed = !NeonHelper.JsonEquals(newRule, existingRule); if (changed) { context.WriteLine(AnsibleVerbosity.Trace, $"Rules are different."); } else { context.WriteLine(AnsibleVerbosity.Info, $"Rules are the same. No need to update."); } } else { changed = true; context.WriteLine(AnsibleVerbosity.Trace, $"Rule [name={ruleName}] does not exist."); } if (changed) { if (context.CheckMode) { context.WriteLine(AnsibleVerbosity.Info, $"Rule [{ruleName}] will be updated when CHECK-MODE is disabled."); } else { context.WriteLine(AnsibleVerbosity.Trace, $"Writing rule [{ruleName}]."); trafficManager.SetRule(newRule); context.WriteLine(AnsibleVerbosity.Info, $"Rule updated."); context.Changed = !context.CheckMode; } } break; case "update": trafficManager.Update(); context.Changed = true; context.WriteLine(AnsibleVerbosity.Info, $"Update signalled."); break; case "purge": var purgeItems = context.ParseStringArray("purge_list"); var purgeCaseSensitive = context.ParseBool("purge_case_sensitive"); if (!purgeCaseSensitive.HasValue) { purgeCaseSensitive = false; } if (purgeItems.Count == 0) { context.WriteLine(AnsibleVerbosity.Important, $"[purge_list] is missing or empty."); break; } trafficManager.Purge(purgeItems.ToArray(), caseSensitive: purgeCaseSensitive.Value); context.Changed = true; context.WriteLine(AnsibleVerbosity.Info, $"Purge request submitted."); break; default: throw new ArgumentException($"[state={state}] is not one of the valid choices: [present], [absent], or [update]."); } }