public void Literal_Name()
{
Assert.Throws <ArgumentNullException>(() => CbHelper.LiteralName(null));
Assert.Throws <ArgumentNullException>(() => CbHelper.LiteralName(string.Empty));
Assert.Equal("`test`", CbHelper.LiteralName("test"));
Assert.Equal("`test\\\\foo`", CbHelper.LiteralName("test\\foo"));
}
public void Literal_String()
{
Assert.Equal("NULL", CbHelper.Literal(null));
Assert.Equal("\"Hello World!\"", CbHelper.Literal("Hello World!"));
Assert.Equal("\"'\"", CbHelper.Literal("'"));
Assert.Equal("\"\\\"\"", CbHelper.Literal("\""));
Assert.Equal("\"\\\\\"", CbHelper.Literal("\\"));
Assert.Equal("\"\\b\"", CbHelper.Literal("\b"));
Assert.Equal("\"\\f\"", CbHelper.Literal("\f"));
Assert.Equal("\"\\n\"", CbHelper.Literal("\n"));
Assert.Equal("\"\\r\"", CbHelper.Literal("\r"));
Assert.Equal("\"\\t\"", CbHelper.Literal("\t"));
}
public async Task Flush()
{
var indexQuery = $"select * from system:indexes where keyspace_id={CbHelper.Literal(bucket.Name)}";
// Flush and verify that the primary index was created by default.
couchbase.Clear();
var indexes = await bucket.QuerySafeAsync <JObject>(indexQuery);
Assert.Single(indexes);
var index = (JObject)indexes.First().GetValue("indexes");
Assert.True((bool)index.GetValue("is_primary"));
Assert.Equal("idx_primary", (string)index.GetValue("name"));
// Write some data, verify that it was written then flush
// the bucket and verify that the data is gone.
await bucket.UpsertSafeAsync("hello", "world!");
Assert.Equal("world!", await bucket.GetSafeAsync <string>("hello"));
couchbase.Clear();
Assert.Null(await bucket.FindSafeAsync <string>("hello"));
// Create a secondary index and verify.
await bucket.QuerySafeAsync <dynamic>($"create index idx_foo on {CbHelper.LiteralName(bucket.Name)} ( {CbHelper.LiteralName("Test")} )");
indexes = await bucket.QuerySafeAsync <JObject>(indexQuery);
Assert.Equal(2, indexes.Count); // Expecting the primary and new secondary index
// Clear the database and then verify that only the
// recreated primary index exists.
couchbase.Clear();
indexes = await bucket.QuerySafeAsync <JObject>(indexQuery);
Assert.Single(indexes);
index = (JObject)indexes.First().GetValue("indexes");
Assert.True((bool)index.GetValue("is_primary"));
Assert.Equal("idx_primary", (string)index.GetValue("name"));
}
/// <inheritdoc/>
public void Run(ModuleContext context)
{
var hive = HiveHelper.Hive;
var nodeGroups = hive.Definition.GetHostGroups(excludeAllGroup: true);
//-----------------------------------------------------------------
// Parse the module arguments.
if (!context.ValidateArguments(context.Arguments, validModuleArgs))
{
context.Failed = true;
return;
}
var couchbaseArgs = CouchbaseArgs.Parse(context);
if (couchbaseArgs == null)
{
return;
}
if (!context.Arguments.TryGetValue <string>("state", out var state))
{
state = "present";
}
state = state.ToLowerInvariant();
if (!context.Arguments.TryGetValue <bool>("force", out var force))
{
force = false;
}
var buildNow = context.ParseBool("build_all");
if (!buildNow.HasValue)
{
buildNow = false;
}
var primary = context.ParseBool("primary") ?? false;
string name;
if (primary)
{
name = "#primary";
}
else
{
name = context.ParseString("name", v => new Regex(@"[a-z][a-z0-0#_]*", RegexOptions.IgnoreCase).IsMatch(v));
}
if (string.IsNullOrEmpty(name) && state != "build")
{
context.WriteErrorLine("[name] argument is required.");
}
var type = (context.ParseEnum <IndexType>("using") ?? default(IndexType)).ToString().ToUpperInvariant();
var namespaceId = context.ParseString("namespace");
if (string.IsNullOrEmpty(namespaceId))
{
namespaceId = "default";
}
var keys = context.ParseStringArray("keys");
var where = context.ParseString("where");
var nodes = context.ParseStringArray("nodes");
var defer = context.ParseBool("build_defer") ?? false;
var wait = context.ParseBool("build_wait") ?? true;
var replicas = context.ParseInt("replicas");
if (state == "present")
{
if (primary)
{
if (keys.Count > 0)
{
context.WriteErrorLine("PRIMARY indexes do not allow any [keys].");
return;
}
if (!string.IsNullOrEmpty(where))
{
context.WriteErrorLine("PRIMARY indexes do not support the [where] clause.");
return;
}
}
else
{
if (keys.Count == 0)
{
context.WriteErrorLine("Non-PRIMARY indexes must specify at least one [key].");
return;
}
}
if (type == "GSI" && replicas.HasValue && nodes.Count > 0)
{
context.WriteErrorLine("Only one of [nodes] or [replicas] may be specified for GSI indexes.");
return;
}
}
string keyspace;
if (!string.IsNullOrEmpty(namespaceId) && namespaceId != "default")
{
keyspace = $"{namespaceId}:{couchbaseArgs.Settings.Bucket}";
}
else
{
keyspace = couchbaseArgs.Settings.Bucket;
}
if (context.HasErrors)
{
return;
}
//-----------------------------------------------------------------
// Perform the operation.
Task.Run(
async() =>
{
using (var bucket = couchbaseArgs.Settings.OpenBucket(couchbaseArgs.Credentials))
{
var indexId = $"{bucket.Name}.{name}";
// Fetch the index if it already exists.
var existingIndex = await bucket.GetIndexAsync(name);
if (existingIndex == null)
{
context.WriteLine(AnsibleVerbosity.Trace, $"Index [{name}] does not exist.");
}
else
{
context.WriteLine(AnsibleVerbosity.Trace, $"Index [{name}] exists.");
}
var existingIsPrimary = existingIndex != null && existingIndex.IsPrimary;
switch (state.ToLowerInvariant())
{
case "present":
// Generate the index creation query.
var sbCreateIndexCommand = new StringBuilder();
if (primary)
{
sbCreateIndexCommand.Append($"create primary index {CbHelper.LiteralName(name)} on {CbHelper.LiteralName(bucket.Name)}");
}
else
{
var sbKeys = new StringBuilder();
foreach (var key in keys)
{
sbKeys.AppendWithSeparator(key, ", ");
}
sbCreateIndexCommand.Append($"create index {CbHelper.LiteralName(name)} on {CbHelper.LiteralName(bucket.Name)} ( {sbKeys} )");
}
// Append the WHERE clause for non-PRIMARY indexes.
if (!primary && !string.IsNullOrEmpty(where))
{
// Ensure that the WHERE clause is surrounded by "( ... )".
if (!where.StartsWith("(") && !where.EndsWith(")"))
{
where = $"({where})";
}
// Now strip the parens off the where clause to be added
// to the query.
var queryWhere = where.Substring(1, where.Length - 2);
// Append the clause.
sbCreateIndexCommand.AppendWithSeparator($"where {queryWhere}");
}
// Append the USING clause.
sbCreateIndexCommand.AppendWithSeparator($"using {type}");
// Append the WITH clause for GSI indexes.
if (type == "GSI")
{
var sbWithSettings = new StringBuilder();
if (defer)
{
sbWithSettings.AppendWithSeparator("\"defer_build\":true", ", ");
}
context.WriteLine(AnsibleVerbosity.Trace, "Query for the hive nodes.");
var clusterNodes = await bucket.QuerySafeAsync <dynamic>("select nodes.name from system:nodes");
context.WriteLine(AnsibleVerbosity.Trace, $"Hive has [{clusterNodes.Count}] nodes.");
if ((!replicas.HasValue || replicas.Value == 0) && nodes.Count == 0)
{
// We're going to default to hosting GSI indexes explicitly
// on all nodes unless directed otherwise. We'll need query
// the database for the current nodes.
foreach (JObject node in clusterNodes)
{
nodes.Add((string)node.GetValue("name"));
}
}
else if (replicas.HasValue && replicas.Value > 0)
{
if (clusterNodes.Count <= replicas.Value)
{
context.WriteErrorLine($"[replicas={replicas.Value}] cannot equal or exceed the number of Couchbase nodes. [replicas={clusterNodes.Count - 1}] is the maximum allowed value for this hive.");
return;
}
}
if (nodes.Count > 0)
{
sbWithSettings.AppendWithSeparator("\"nodes\": [", ", ");
var first = true;
foreach (var server in nodes)
{
if (first)
{
first = false;
}
else
{
sbCreateIndexCommand.Append(",");
}
sbWithSettings.Append(CbHelper.Literal(server));
}
sbWithSettings.Append("]");
}
if (replicas.HasValue && type == "GSI")
{
sbWithSettings.AppendWithSeparator($"\"num_replica\":{CbHelper.Literal(replicas.Value)}", ", ");
}
if (sbWithSettings.Length > 0)
{
sbCreateIndexCommand.AppendWithSeparator($"with {{ {sbWithSettings} }}");
}
}
// Add or update the index.
if (existingIndex != null)
{
context.WriteLine(AnsibleVerbosity.Info, $"Index [{indexId}] already exists.");
// An index with this name already exists, so we'll compare its
// properties with the module parameters to determine whether we
// need to remove and recreate it.
var changed = false;
if (force)
{
changed = true;
context.WriteLine(AnsibleVerbosity.Info, "Rebuilding index because [force=yes].");
}
// Compare the old/new index types.
var orgType = existingIndex.Type.ToUpperInvariant();
if (!string.Equals(orgType, type, StringComparison.InvariantCultureIgnoreCase))
{
changed = true;
context.WriteLine(AnsibleVerbosity.Info, $"Rebuilding index because using changed from [{orgType}] to [{type}].");
}
// Compare the old/new index keys.
var keysChanged = false;
if (!primary)
{
var orgKeys = existingIndex.Keys;
if (orgKeys.Length != keys.Count)
{
keysChanged = true;
}
else
{
// This assumes that the order of the indexed keys doesn't
// matter.
var keysSet = new Dictionary <string, bool>(StringComparer.InvariantCultureIgnoreCase);
for (int i = 0; i < orgKeys.Length; i++)
{
keysSet[(string)orgKeys[i]] = false;
}
for (int i = 0; i < orgKeys.Length; i++)
{
keysSet[CbHelper.LiteralName(keys[i])] = true;
}
keysChanged = keysSet.Values.Count(k => !k) > 0;
}
if (keysChanged)
{
changed = true;
var sbOrgKeys = new StringBuilder();
var sbNewKeys = new StringBuilder();
foreach (string key in orgKeys)
{
sbOrgKeys.AppendWithSeparator(key, ", ");
}
foreach (string key in keys)
{
sbNewKeys.AppendWithSeparator(key, ", ");
}
context.WriteLine(AnsibleVerbosity.Info, $"Rebuilding index because keys changed from [{sbOrgKeys}] to [{sbNewKeys}].");
}
}
// Compare the filter condition.
var orgWhere = existingIndex.Where;
if (orgWhere != where)
{
changed = true;
context.WriteLine(AnsibleVerbosity.Info, $"Rebuilding index because where clause changed from [{orgWhere ?? string.Empty}] to [{where ?? string.Empty}].");
}
// We need to remove and recreate the index if it differs
// from what was requested.
if (changed)
{
if (context.CheckMode)
{
context.WriteLine(AnsibleVerbosity.Important, $"Index [{indexId}] will be rebuilt when CHECK-MODE is disabled.");
}
else
{
context.Changed = !context.CheckMode;
context.WriteLine(AnsibleVerbosity.Trace, $"Removing existing index [{indexId}].");
string dropCommand;
if (existingIsPrimary)
{
dropCommand = $"drop primary index on {CbHelper.LiteralName(bucket.Name)} using {orgType.ToUpperInvariant()}";
}
else
{
dropCommand = $"drop index {CbHelper.LiteralName(bucket.Name)}.{CbHelper.LiteralName(name)} using {orgType.ToUpperInvariant()}";
}
context.WriteLine(AnsibleVerbosity.Trace, $"DROP COMMAND: {dropCommand}");
await bucket.QuerySafeAsync <dynamic>(dropCommand);
context.WriteLine(AnsibleVerbosity.Trace, $"Dropped index [{indexId}].");
context.WriteLine(AnsibleVerbosity.Trace, $"CREATE COMMAND: {sbCreateIndexCommand}");
await bucket.QuerySafeAsync <dynamic>(sbCreateIndexCommand.ToString());
context.WriteLine(AnsibleVerbosity.Info, $"Created index [{indexId}].");
if (!defer && wait)
{
// Wait for the index to come online.
context.WriteLine(AnsibleVerbosity.Info, $"Waiting for index [{name}] to be built.");
await bucket.WaitForIndexAsync(name, "online");
context.WriteLine(AnsibleVerbosity.Info, $"Completed building index [{name}].");
}
}
}
else
{
context.WriteLine(AnsibleVerbosity.Trace, $"No changes detected for index [{indexId}].");
}
}
else
{
if (context.CheckMode)
{
context.WriteLine(AnsibleVerbosity.Important, $"Index [{indexId}] will be created when CHECK-MODE is disabled.");
context.WriteLine(AnsibleVerbosity.Trace, $"{sbCreateIndexCommand}");
}
else
{
context.Changed = true;
context.WriteLine(AnsibleVerbosity.Trace, $"Creating index.");
context.WriteLine(AnsibleVerbosity.Trace, $"CREATE COMMAND: {sbCreateIndexCommand}");
await bucket.QuerySafeAsync <dynamic>(sbCreateIndexCommand.ToString());
context.WriteLine(AnsibleVerbosity.Info, $"Created index [{indexId}].");
if (!defer && wait)
{
// Wait for the index to come online.
context.WriteLine(AnsibleVerbosity.Info, $"Waiting for index [{name}] to be built.");
await bucket.WaitForIndexAsync(name, "online");
context.WriteLine(AnsibleVerbosity.Info, $"Completed building index [{name}].");
}
}
}
break;
case "absent":
if (existingIndex != null)
{
if (context.CheckMode)
{
context.WriteLine(AnsibleVerbosity.Important, $"Index [{indexId}] will be dropped when CHECK-MODE is disabled.");
}
else
{
context.Changed = true;
context.WriteLine(AnsibleVerbosity.Info, $"Dropping index [{indexId}].");
string orgType = existingIndex.Type;
string dropCommand;
if (existingIsPrimary)
{
dropCommand = $"drop primary index on {CbHelper.LiteralName(bucket.Name)} using {orgType.ToUpperInvariant()}";
}
else
{
dropCommand = $"drop index {CbHelper.LiteralName(bucket.Name)}.{CbHelper.LiteralName(name)} using {orgType.ToUpperInvariant()}";
}
context.WriteLine(AnsibleVerbosity.Trace, $"COMMAND: {dropCommand}");
await bucket.QuerySafeAsync <dynamic>(dropCommand);
context.WriteLine(AnsibleVerbosity.Trace, $"Index [{indexId}] was dropped.");
}
}
else
{
context.WriteLine(AnsibleVerbosity.Important, $"Index [{indexId}] does not exist so there's no need to drop it.");
}
break;
case "build":
// List the names of the deferred GSI indexes.
var deferredIndexes = ((await bucket.ListIndexesAsync()).Where(index => index.State == "deferred" && index.Type == "gsi")).ToList();
context.WriteLine(AnsibleVerbosity.Info, $"[{deferredIndexes.Count}] deferred GSI indexes exist.");
if (deferredIndexes.Count == 0)
{
context.WriteLine(AnsibleVerbosity.Important, $"All GSI indexes have already been built.");
context.Changed = false;
return;
}
// Build the indexes (unless we're in CHECK-MODE).
var sbIndexList = new StringBuilder();
foreach (var deferredIndex in deferredIndexes)
{
sbIndexList.AppendWithSeparator($"{CbHelper.LiteralName(deferredIndex.Name)}", ", ");
}
if (context.CheckMode)
{
context.WriteLine(AnsibleVerbosity.Important, $"These GSI indexes will be built when CHECK-MODE is disabled: {sbIndexList}.");
context.Changed = false;
return;
}
var buildCommand = $"BUILD INDEX ON {CbHelper.LiteralName(bucket.Name)} ({sbIndexList})";
context.WriteLine(AnsibleVerbosity.Trace, $"BUILD COMMAND: {buildCommand}");
context.WriteLine(AnsibleVerbosity.Info, $"Building indexes: {sbIndexList}");
await bucket.QuerySafeAsync <dynamic>(buildCommand);
context.WriteLine(AnsibleVerbosity.Info, $"Build command submitted.");
context.Changed = true;
// The Couchbase BUILD INDEX command doesn't wait for the index
// building to complete so, we'll just spin until all of the
// indexes we're building are online.
if (wait)
{
context.WriteLine(AnsibleVerbosity.Info, $"Waiting for the indexes to be built.");
foreach (var deferredIndex in deferredIndexes)
{
await bucket.WaitForIndexAsync(deferredIndex.Name, "online");
}
context.WriteLine(AnsibleVerbosity.Info, $"Completed building [{deferredIndexes.Count}] indexes.");
}
break;
default:
throw new ArgumentException($"[state={state}] is not one of the valid choices: [present], [absent], or [build].");
}
}
}).Wait();
}
/// <summary>
/// Removes all data and indexes from the database bucket and then recreates the
/// primary index if an index was specified when the fixture was started.
/// </summary>
public void Clear()
{
CheckDisposed();
// $todo(jeff.lill):
//
// The code below was originally intended to clear the Couchbase bucket
// in place and this seemed to work for several months and then it just stopped
// working in Nov 2018 after I upgraded the CouchbaseNetClient nuget package.
// The weird thing is that is still didn't work after I reverted.
//
// The problem seems to be due to a timing or race condition because if I pause
// execution after clearing, the subsequent unit test passes. Unfortunately, I
// haven't been able to figure out how to determine when everything is ready.
// I even tried executing the unit test query that fails but it succeeded here
// but still failed in the test. I can't really explain that: perhaps
// Couchbase restarted one or more services sometime after I cleared the
// bucket but after I checked for health below.
//
// It seems like the first test ran against a clean container always works
// so I'm going to revert to simply restarting the container and come back
// someday and remove this section.
#if DIDNT_WORK
// Drop all of the bucket indexes.
var existingIndexes = Bucket.ListIndexesAsync().Result;
if (existingIndexes.Count > 0)
{
foreach (var index in existingIndexes)
{
Bucket.QuerySafeAsync <dynamic>($"drop index {CbHelper.LiteralName(Bucket.Name)}.{CbHelper.LiteralName(index.Name)} using {index.Type}").Wait();
}
}
// Flush the bucket data.
using (var bucketManager = Bucket.CreateManager())
{
NeonBucket.ReadyRetry.InvokeAsync(
async() =>
{
bucketManager.Flush();
await Bucket.WaitUntilReadyAsync();
}).Wait();
}
// Wait until all of the indexes are actually deleted.
NeonHelper.WaitFor(
() =>
{
var indexes = Bucket.ListIndexesAsync().Result;
return(indexes.Count == 0);
},
timeout: NeonBucket.ReadyTimeout,
pollTime: TimeSpan.FromMilliseconds(500));
// Recreate the primary index if one was enabled when the fixture was started.
if (createPrimaryIndex)
{
Bucket.QuerySafeAsync <dynamic>($"create primary index on {CbHelper.LiteralName(Bucket.Name)} using gsi").Wait();
Bucket.WaitForIndexAsync("#primary").Wait();
}
#endif
base.Restart();
Thread.Sleep(warmupDelay);
ConnectBucket();
}
/// <summary>
/// Establishes the bucket connection and waits until the Couchbase container is ready
/// to start handling requests.
/// </summary>
private void ConnectBucket()
{
// Give the Couchbase container a chance to spin up.
Thread.Sleep(warmupDelay);
// Dispose any existing underlying cluster and bucket.
if (Bucket != null)
{
var existingBucket = Bucket.GetInternalBucket();
if (existingBucket != null)
{
existingBucket.Cluster.CloseBucket(existingBucket);
existingBucket.Cluster.Dispose();
Bucket.SetInternalBucket(null);
}
}
// It appears that it may take a bit of time for the Couchbase query
// service to start in new container we started above. We're going to
// retry creating the primary index (or a dummy index) until it works.
var bucket = (NeonBucket)null;
var indexCreated = false;
var indexReady = false;
var queryReady = false;
NeonBucket.ReadyRetry.InvokeAsync(
async() =>
{
if (bucket == null)
{
bucket = Settings.OpenBucket(Username, Password);
}
try
{
if (createPrimaryIndex)
{
// Create the primary index if requested.
if (!indexCreated)
{
await bucket.QuerySafeAsync <dynamic>($"create primary index on {CbHelper.LiteralName(bucket.Name)} using gsi");
indexCreated = true;
}
if (!indexReady)
{
await bucket.WaitForIndexAsync("#primary");
indexReady = true;
}
// Ensure that the query service is running too.
if (!queryReady)
{
var query = new QueryRequest($"select meta({bucket.Name}).id, {bucket.Name}.* from {bucket.Name}")
.ScanConsistency(ScanConsistency.RequestPlus);
await bucket.QuerySafeAsync <dynamic>(query);
queryReady = true;
}
}
else
{
// List the indexes to ensure the index service is ready when we didn't create a primary index.
if (!queryReady)
{
await bucket.ListIndexesAsync();
queryReady = true;
}
}
}
catch
{
// $hack(jeff.lill):
//
// It looks like we need to create a new bucket if the query service
// wasn't ready. I'm guessing that this is due to the Couchbase index
// service not being ready at the time the bucket was connected and
// the bucket isn't smart enough to retry looking for the index service
// afterwards. This won't be a problem for most real-world scenarios
// because for those, Couchbase will have been started long ago and
// will continue running indefinitely.
//
// We'll dispose the old bucket and set it to NULL here and then
// open a fresh bucket above when the retry policy tries again.
bucket.Dispose();
bucket = null;
throw;
}
}).Wait();
// Use the new bucket if this is the first Couchbase container created or
// else substitute the new underlying bucket into the existing bucket so
// that unit tests don't need to be aware of the change.
if (this.Bucket == null)
{
this.Bucket = bucket;
}
else
{
this.Bucket.SetInternalBucket(bucket.GetInternalBucket());
}
}