/// <inheritdoc/>
public void Run(ModuleContext context)
{
var hive = HiveHelper.Hive;
var nodeGroups = hive.Definition.GetHostGroups(excludeAllGroup: true);
//-----------------------------------------------------------------
// Parse the module arguments.
if (!context.ValidateArguments(context.Arguments, validModuleArgs))
{
context.Failed = true;
return;
}
var couchbaseArgs = CouchbaseArgs.Parse(context);
if (couchbaseArgs == null)
{
return;
}
var format = context.ParseEnum <CouchbaseFileFormat>("format");
if (!format.HasValue)
{
format = default(CouchbaseFileFormat);
}
var source = context.ParseString("source");
if (string.IsNullOrEmpty(source))
{
context.WriteErrorLine("[source] module parameter is required.");
return;
}
if (!File.Exists(source))
{
context.WriteErrorLine($"File [{source}] does not exist.");
return;
}
var keyPattern = context.ParseString("key");
var firstKey = context.ParseLong("first_key") ?? 1;
if (context.HasErrors)
{
return;
}
//-----------------------------------------------------------------
// Import the data.
using (var bucket = couchbaseArgs.Settings.OpenBucket(couchbaseArgs.Credentials))
{
var importer = new CouchbaseImporter(message => context.WriteErrorLine(message), bucket, keyPattern, firstKey, context.CheckMode);
switch (format.Value)
{
case CouchbaseFileFormat.JsonArray:
// $todo(jeff.lill):
//
// Would be nice not to read this whole thing in memory and then
// effectibely duplicating it in memory again when parsing.
var jToken = JToken.Parse(File.ReadAllText(source));
if (jToken.Type != JTokenType.Array)
{
context.WriteErrorLine($"[{source}] is not a JSON array of documents.");
return;
}
var jArray = (JArray)jToken;
foreach (var item in jArray)
{
if (item.Type != JTokenType.Object)
{
context.WriteErrorLine($"[{source}] includes one or more non-document objects in the array.");
return;
}
importer.WriteDocument((JObject)item);
}
break;
case CouchbaseFileFormat.JsonLines:
using (var reader = new StreamReader(source, Encoding.UTF8))
{
foreach (var line in reader.Lines())
{
if (line.Trim() == string.Empty)
{
continue; // Ignore blank lines
}
var item = JToken.Parse(line);
if (item.Type != JTokenType.Object)
{
context.WriteErrorLine($"[{source}] includes one or more lines with non-document objects.");
return;
}
importer.WriteDocument((JObject)item);
}
}
break;
default:
throw new NotImplementedException($"Format [{format}] is not implemented.");
}
context.Changed = importer.DocumentCount > 0;
if (context.CheckMode)
{
context.WriteLine(AnsibleVerbosity.Info, $"[{importer.DocumentCount}] documents will be added when CHECK-MODE is disabled.");
}
else
{
context.WriteLine(AnsibleVerbosity.Info, $"[{importer.DocumentCount}] documents were imported.");
}
}
}
/// <inheritdoc/>
public void Run(ModuleContext context)
{
var hive = HiveHelper.Hive;
var nodeGroups = hive.Definition.GetHostGroups(excludeAllGroup: true);
//-----------------------------------------------------------------
// Parse the module arguments.
if (!context.ValidateArguments(context.Arguments, validModuleArgs))
{
context.Failed = true;
return;
}
var couchbaseArgs = CouchbaseArgs.Parse(context);
if (couchbaseArgs == null)
{
return;
}
var query = context.ParseString("query", q => !string.IsNullOrWhiteSpace(q));
if (context.HasErrors)
{
return;
}
var format = context.ParseEnum <CouchbaseFileFormat>("format");
if (!format.HasValue)
{
format = CouchbaseFileFormat.JsonLines;
}
var limit = context.ParseLong("limit", v => v >= 0);
if (!limit.HasValue || limit.Value == 0)
{
limit = long.MaxValue;
}
var output = context.ParseString("output");
//-----------------------------------------------------------------
// Execute the query.
using (var bucket = couchbaseArgs.Settings.OpenBucket(couchbaseArgs.Credentials))
{
try
{
var results = bucket.QuerySafeAsync <JObject>(query).Result;
var count = Math.Min(results.Count, limit.Value);
using (var writer = new CouchbaseQueryResultWriter(context, format.Value, output))
{
for (int i = 0; i < count; i++)
{
var document = results[i];
var isLast = i == count - 1;
writer.WriteDocument(document, isLast);
}
}
}
catch (AggregateException e)
{
var queryException = e.Find <CouchbaseQueryResponseException>();
if (queryException == null)
{
throw;
}
foreach (var error in queryException.Errors)
{
context.WriteErrorLine($"Couchbase [{error.Code}]: {error.Message}");
}
}
}
}
/// <inheritdoc/>
public void Run(ModuleContext context)
{
var hive = HiveHelper.Hive;
var nodeGroups = hive.Definition.GetHostGroups(excludeAllGroup: true);
//-----------------------------------------------------------------
// Parse the module arguments.
if (!context.ValidateArguments(context.Arguments, validModuleArgs))
{
context.Failed = true;
return;
}
var couchbaseArgs = CouchbaseArgs.Parse(context);
if (couchbaseArgs == null)
{
return;
}
if (!context.Arguments.TryGetValue <string>("state", out var state))
{
state = "present";
}
state = state.ToLowerInvariant();
if (!context.Arguments.TryGetValue <bool>("force", out var force))
{
force = false;
}
var buildNow = context.ParseBool("build_all");
if (!buildNow.HasValue)
{
buildNow = false;
}
var primary = context.ParseBool("primary") ?? false;
string name;
if (primary)
{
name = "#primary";
}
else
{
name = context.ParseString("name", v => new Regex(@"[a-z][a-z0-0#_]*", RegexOptions.IgnoreCase).IsMatch(v));
}
if (string.IsNullOrEmpty(name) && state != "build")
{
context.WriteErrorLine("[name] argument is required.");
}
var type = (context.ParseEnum <IndexType>("using") ?? default(IndexType)).ToString().ToUpperInvariant();
var namespaceId = context.ParseString("namespace");
if (string.IsNullOrEmpty(namespaceId))
{
namespaceId = "default";
}
var keys = context.ParseStringArray("keys");
var where = context.ParseString("where");
var nodes = context.ParseStringArray("nodes");
var defer = context.ParseBool("build_defer") ?? false;
var wait = context.ParseBool("build_wait") ?? true;
var replicas = context.ParseInt("replicas");
if (state == "present")
{
if (primary)
{
if (keys.Count > 0)
{
context.WriteErrorLine("PRIMARY indexes do not allow any [keys].");
return;
}
if (!string.IsNullOrEmpty(where))
{
context.WriteErrorLine("PRIMARY indexes do not support the [where] clause.");
return;
}
}
else
{
if (keys.Count == 0)
{
context.WriteErrorLine("Non-PRIMARY indexes must specify at least one [key].");
return;
}
}
if (type == "GSI" && replicas.HasValue && nodes.Count > 0)
{
context.WriteErrorLine("Only one of [nodes] or [replicas] may be specified for GSI indexes.");
return;
}
}
string keyspace;
if (!string.IsNullOrEmpty(namespaceId) && namespaceId != "default")
{
keyspace = $"{namespaceId}:{couchbaseArgs.Settings.Bucket}";
}
else
{
keyspace = couchbaseArgs.Settings.Bucket;
}
if (context.HasErrors)
{
return;
}
//-----------------------------------------------------------------
// Perform the operation.
Task.Run(
async() =>
{
using (var bucket = couchbaseArgs.Settings.OpenBucket(couchbaseArgs.Credentials))
{
var indexId = $"{bucket.Name}.{name}";
// Fetch the index if it already exists.
var existingIndex = await bucket.GetIndexAsync(name);
if (existingIndex == null)
{
context.WriteLine(AnsibleVerbosity.Trace, $"Index [{name}] does not exist.");
}
else
{
context.WriteLine(AnsibleVerbosity.Trace, $"Index [{name}] exists.");
}
var existingIsPrimary = existingIndex != null && existingIndex.IsPrimary;
switch (state.ToLowerInvariant())
{
case "present":
// Generate the index creation query.
var sbCreateIndexCommand = new StringBuilder();
if (primary)
{
sbCreateIndexCommand.Append($"create primary index {CbHelper.LiteralName(name)} on {CbHelper.LiteralName(bucket.Name)}");
}
else
{
var sbKeys = new StringBuilder();
foreach (var key in keys)
{
sbKeys.AppendWithSeparator(key, ", ");
}
sbCreateIndexCommand.Append($"create index {CbHelper.LiteralName(name)} on {CbHelper.LiteralName(bucket.Name)} ( {sbKeys} )");
}
// Append the WHERE clause for non-PRIMARY indexes.
if (!primary && !string.IsNullOrEmpty(where))
{
// Ensure that the WHERE clause is surrounded by "( ... )".
if (!where.StartsWith("(") && !where.EndsWith(")"))
{
where = $"({where})";
}
// Now strip the parens off the where clause to be added
// to the query.
var queryWhere = where.Substring(1, where.Length - 2);
// Append the clause.
sbCreateIndexCommand.AppendWithSeparator($"where {queryWhere}");
}
// Append the USING clause.
sbCreateIndexCommand.AppendWithSeparator($"using {type}");
// Append the WITH clause for GSI indexes.
if (type == "GSI")
{
var sbWithSettings = new StringBuilder();
if (defer)
{
sbWithSettings.AppendWithSeparator("\"defer_build\":true", ", ");
}
context.WriteLine(AnsibleVerbosity.Trace, "Query for the hive nodes.");
var clusterNodes = await bucket.QuerySafeAsync <dynamic>("select nodes.name from system:nodes");
context.WriteLine(AnsibleVerbosity.Trace, $"Hive has [{clusterNodes.Count}] nodes.");
if ((!replicas.HasValue || replicas.Value == 0) && nodes.Count == 0)
{
// We're going to default to hosting GSI indexes explicitly
// on all nodes unless directed otherwise. We'll need query
// the database for the current nodes.
foreach (JObject node in clusterNodes)
{
nodes.Add((string)node.GetValue("name"));
}
}
else if (replicas.HasValue && replicas.Value > 0)
{
if (clusterNodes.Count <= replicas.Value)
{
context.WriteErrorLine($"[replicas={replicas.Value}] cannot equal or exceed the number of Couchbase nodes. [replicas={clusterNodes.Count - 1}] is the maximum allowed value for this hive.");
return;
}
}
if (nodes.Count > 0)
{
sbWithSettings.AppendWithSeparator("\"nodes\": [", ", ");
var first = true;
foreach (var server in nodes)
{
if (first)
{
first = false;
}
else
{
sbCreateIndexCommand.Append(",");
}
sbWithSettings.Append(CbHelper.Literal(server));
}
sbWithSettings.Append("]");
}
if (replicas.HasValue && type == "GSI")
{
sbWithSettings.AppendWithSeparator($"\"num_replica\":{CbHelper.Literal(replicas.Value)}", ", ");
}
if (sbWithSettings.Length > 0)
{
sbCreateIndexCommand.AppendWithSeparator($"with {{ {sbWithSettings} }}");
}
}
// Add or update the index.
if (existingIndex != null)
{
context.WriteLine(AnsibleVerbosity.Info, $"Index [{indexId}] already exists.");
// An index with this name already exists, so we'll compare its
// properties with the module parameters to determine whether we
// need to remove and recreate it.
var changed = false;
if (force)
{
changed = true;
context.WriteLine(AnsibleVerbosity.Info, "Rebuilding index because [force=yes].");
}
// Compare the old/new index types.
var orgType = existingIndex.Type.ToUpperInvariant();
if (!string.Equals(orgType, type, StringComparison.InvariantCultureIgnoreCase))
{
changed = true;
context.WriteLine(AnsibleVerbosity.Info, $"Rebuilding index because using changed from [{orgType}] to [{type}].");
}
// Compare the old/new index keys.
var keysChanged = false;
if (!primary)
{
var orgKeys = existingIndex.Keys;
if (orgKeys.Length != keys.Count)
{
keysChanged = true;
}
else
{
// This assumes that the order of the indexed keys doesn't
// matter.
var keysSet = new Dictionary <string, bool>(StringComparer.InvariantCultureIgnoreCase);
for (int i = 0; i < orgKeys.Length; i++)
{
keysSet[(string)orgKeys[i]] = false;
}
for (int i = 0; i < orgKeys.Length; i++)
{
keysSet[CbHelper.LiteralName(keys[i])] = true;
}
keysChanged = keysSet.Values.Count(k => !k) > 0;
}
if (keysChanged)
{
changed = true;
var sbOrgKeys = new StringBuilder();
var sbNewKeys = new StringBuilder();
foreach (string key in orgKeys)
{
sbOrgKeys.AppendWithSeparator(key, ", ");
}
foreach (string key in keys)
{
sbNewKeys.AppendWithSeparator(key, ", ");
}
context.WriteLine(AnsibleVerbosity.Info, $"Rebuilding index because keys changed from [{sbOrgKeys}] to [{sbNewKeys}].");
}
}
// Compare the filter condition.
var orgWhere = existingIndex.Where;
if (orgWhere != where)
{
changed = true;
context.WriteLine(AnsibleVerbosity.Info, $"Rebuilding index because where clause changed from [{orgWhere ?? string.Empty}] to [{where ?? string.Empty}].");
}
// We need to remove and recreate the index if it differs
// from what was requested.
if (changed)
{
if (context.CheckMode)
{
context.WriteLine(AnsibleVerbosity.Important, $"Index [{indexId}] will be rebuilt when CHECK-MODE is disabled.");
}
else
{
context.Changed = !context.CheckMode;
context.WriteLine(AnsibleVerbosity.Trace, $"Removing existing index [{indexId}].");
string dropCommand;
if (existingIsPrimary)
{
dropCommand = $"drop primary index on {CbHelper.LiteralName(bucket.Name)} using {orgType.ToUpperInvariant()}";
}
else
{
dropCommand = $"drop index {CbHelper.LiteralName(bucket.Name)}.{CbHelper.LiteralName(name)} using {orgType.ToUpperInvariant()}";
}
context.WriteLine(AnsibleVerbosity.Trace, $"DROP COMMAND: {dropCommand}");
await bucket.QuerySafeAsync <dynamic>(dropCommand);
context.WriteLine(AnsibleVerbosity.Trace, $"Dropped index [{indexId}].");
context.WriteLine(AnsibleVerbosity.Trace, $"CREATE COMMAND: {sbCreateIndexCommand}");
await bucket.QuerySafeAsync <dynamic>(sbCreateIndexCommand.ToString());
context.WriteLine(AnsibleVerbosity.Info, $"Created index [{indexId}].");
if (!defer && wait)
{
// Wait for the index to come online.
context.WriteLine(AnsibleVerbosity.Info, $"Waiting for index [{name}] to be built.");
await bucket.WaitForIndexAsync(name, "online");
context.WriteLine(AnsibleVerbosity.Info, $"Completed building index [{name}].");
}
}
}
else
{
context.WriteLine(AnsibleVerbosity.Trace, $"No changes detected for index [{indexId}].");
}
}
else
{
if (context.CheckMode)
{
context.WriteLine(AnsibleVerbosity.Important, $"Index [{indexId}] will be created when CHECK-MODE is disabled.");
context.WriteLine(AnsibleVerbosity.Trace, $"{sbCreateIndexCommand}");
}
else
{
context.Changed = true;
context.WriteLine(AnsibleVerbosity.Trace, $"Creating index.");
context.WriteLine(AnsibleVerbosity.Trace, $"CREATE COMMAND: {sbCreateIndexCommand}");
await bucket.QuerySafeAsync <dynamic>(sbCreateIndexCommand.ToString());
context.WriteLine(AnsibleVerbosity.Info, $"Created index [{indexId}].");
if (!defer && wait)
{
// Wait for the index to come online.
context.WriteLine(AnsibleVerbosity.Info, $"Waiting for index [{name}] to be built.");
await bucket.WaitForIndexAsync(name, "online");
context.WriteLine(AnsibleVerbosity.Info, $"Completed building index [{name}].");
}
}
}
break;
case "absent":
if (existingIndex != null)
{
if (context.CheckMode)
{
context.WriteLine(AnsibleVerbosity.Important, $"Index [{indexId}] will be dropped when CHECK-MODE is disabled.");
}
else
{
context.Changed = true;
context.WriteLine(AnsibleVerbosity.Info, $"Dropping index [{indexId}].");
string orgType = existingIndex.Type;
string dropCommand;
if (existingIsPrimary)
{
dropCommand = $"drop primary index on {CbHelper.LiteralName(bucket.Name)} using {orgType.ToUpperInvariant()}";
}
else
{
dropCommand = $"drop index {CbHelper.LiteralName(bucket.Name)}.{CbHelper.LiteralName(name)} using {orgType.ToUpperInvariant()}";
}
context.WriteLine(AnsibleVerbosity.Trace, $"COMMAND: {dropCommand}");
await bucket.QuerySafeAsync <dynamic>(dropCommand);
context.WriteLine(AnsibleVerbosity.Trace, $"Index [{indexId}] was dropped.");
}
}
else
{
context.WriteLine(AnsibleVerbosity.Important, $"Index [{indexId}] does not exist so there's no need to drop it.");
}
break;
case "build":
// List the names of the deferred GSI indexes.
var deferredIndexes = ((await bucket.ListIndexesAsync()).Where(index => index.State == "deferred" && index.Type == "gsi")).ToList();
context.WriteLine(AnsibleVerbosity.Info, $"[{deferredIndexes.Count}] deferred GSI indexes exist.");
if (deferredIndexes.Count == 0)
{
context.WriteLine(AnsibleVerbosity.Important, $"All GSI indexes have already been built.");
context.Changed = false;
return;
}
// Build the indexes (unless we're in CHECK-MODE).
var sbIndexList = new StringBuilder();
foreach (var deferredIndex in deferredIndexes)
{
sbIndexList.AppendWithSeparator($"{CbHelper.LiteralName(deferredIndex.Name)}", ", ");
}
if (context.CheckMode)
{
context.WriteLine(AnsibleVerbosity.Important, $"These GSI indexes will be built when CHECK-MODE is disabled: {sbIndexList}.");
context.Changed = false;
return;
}
var buildCommand = $"BUILD INDEX ON {CbHelper.LiteralName(bucket.Name)} ({sbIndexList})";
context.WriteLine(AnsibleVerbosity.Trace, $"BUILD COMMAND: {buildCommand}");
context.WriteLine(AnsibleVerbosity.Info, $"Building indexes: {sbIndexList}");
await bucket.QuerySafeAsync <dynamic>(buildCommand);
context.WriteLine(AnsibleVerbosity.Info, $"Build command submitted.");
context.Changed = true;
// The Couchbase BUILD INDEX command doesn't wait for the index
// building to complete so, we'll just spin until all of the
// indexes we're building are online.
if (wait)
{
context.WriteLine(AnsibleVerbosity.Info, $"Waiting for the indexes to be built.");
foreach (var deferredIndex in deferredIndexes)
{
await bucket.WaitForIndexAsync(deferredIndex.Name, "online");
}
context.WriteLine(AnsibleVerbosity.Info, $"Completed building [{deferredIndexes.Count}] indexes.");
}
break;
default:
throw new ArgumentException($"[state={state}] is not one of the valid choices: [present], [absent], or [build].");
}
}
}).Wait();
}
