public void Test_ScanWait()
{
var doc = new Document <DocumentContent>
{
Id = "Test_ReadYourOwnWrite_WaitScan",
Content = new DocumentContent
{
Value = new Random().Next(0, 100000)
}
};
var result = _bucket.Upsert(doc);
Assert.True(result.Success);
var state = MutationState.From(result.Document);
var request = new QueryRequest("SELECT d.* FROM default as d WHERE `value` = $1 LIMIT 1")
.AddPositionalParameter(doc.Content.Value)
.ConsistentWith(state)
.ScanWait(TimeSpan.FromSeconds(10));
var queryResult = _bucket.Query <DocumentContent>(request);
Assert.True(queryResult.Success, queryResult.ToString());
Assert.IsNotEmpty(queryResult.Rows);
Assert.AreEqual(doc.Content.Value, queryResult.Rows.First().Value);
}
public void Test_ReadYourOwnWrite()
{
Assert.IsTrue(_bucket.SupportsEnhancedDurability);
var doc = new Document<DocumentContent>
{
Id = "Test_ReadYourOwnWrite",
Content = new DocumentContent()
{
Value = new Random().Next(0, 100000)
}
};
var result = _bucket.Upsert(doc);
Assert.True(result.Success);
var state = MutationState.From(result.Document);
var request = new QueryRequest("SELECT d.* FROM default as d WHERE `value` = $1 LIMIT 1")
.AddPositionalParameter(doc.Content.Value)
.ConsistentWith(state);
var queryResult = _bucket.Query<DocumentContent>(request);
Assert.True(queryResult.Success);
Assert.IsNotEmpty(queryResult.Rows);
Assert.AreEqual(doc.Content.Value, queryResult.Rows.First().Value);
}
private static async Task ScanConsistencyAtPlusExample(ICluster cluster, ICouchbaseCollection collection)
{
// tag::atplus[]
// create / update document (mutation)
var upsertResult = await collection.UpsertAsync("doc1", new { name = "Mike AtPlus", type = "user" });
// create mutation state from mutation results
var state = MutationState.From(upsertResult);
// use mutation state with query option
var result = await cluster.QueryAsync <dynamic>(
"SELECT t.* FROM `travel-sample` t WHERE t.type=$1",
options => options.ConsistentWith(state)
.Parameter("user")
);
// end::atplus[]
// check query was successful
if (result.MetaData.Status != QueryStatus.Success)
{
// error
}
// iterate over rows
// NOTE: because the query is using AtPlus, the new user will be indexed, but notice
// the extra step of creating the 'state' object and passing it in as a 'ConsistentWith' option
await foreach (var row in result)
{
// each row is an instance of the Query<T> call (e.g. dynamic or custom type)
var name = row.name; // "Mike AtPlus"
Console.WriteLine($"{name}");
}
}
// end::RequestPlusExample[]
// tag::AtPlusExample[]
private static void AtPlusExample()
{
Console.WriteLine("========= AtPlus");
// get the current count
var result1 = _bucket.Query <dynamic>("SELECT COUNT(1) as airportCount FROM `travel-sample` WHERE type='airport'")
.Rows.First();
Console.WriteLine($"Initial count: {result1.airportCount}");
// insert a new airport
var doc = new Document <dynamic>
{
Id = "ScanConsistency::airport::" + _random.Next(10000),
Content = new
{
type = "airport"
}
};
var insertResult = _bucket.Insert(doc);
// get the count again
var state = MutationState.From(insertResult.Document);
var request = new QueryRequest("SELECT COUNT(1) as airportCount FROM `travel-sample` WHERE type='airport'");
var t = request.ConsistentWith(state);
var result2 = _bucket.Query <dynamic>(t).Rows.First();
Console.WriteLine($"Count after insert with AtPlus: {result2.airportCount}");
}
public void GetFormValues_MultipleTokensSameVBucketId_HighestSequenceNumberIsUsed2()
{
var document1 = new Mock <IDocument <dynamic> >();
document1.Setup(x => x.Token).Returns(new MutationToken("bucket1_name", 102, 22, 9999));
var document2 = new Mock <IDocument <dynamic> >();
document2.Setup(x => x.Token).Returns(new MutationToken("bucket1_name", 102, 11, 8332));
var document3 = new Mock <IDocument <dynamic> >();
document3.Setup(x => x.Token).Returns(new MutationToken("bucket2_name", 133, 23, 333));
var queryRequest = new QueryRequest("SELECT * FROM `bucket1_name`;").
ConsistentWith(MutationState.From(document1.Object, document2.Object, document3.Object)).
#pragma warning disable 618
ScanConsistency(ScanConsistency.AtPlus);
#pragma warning restore 618
var actual = queryRequest.GetFormValues()["scan_vectors"];
var expected = "{\"bucket1_name\":{\"102\":[9999,\"22\"]},\"bucket2_name\":{\"133\":[333,\"23\"]}}";
Assert.AreEqual(expected, JsonConvert.SerializeObject(actual));
}
public void GetFormValues_WhenScanConsistenyIsAtPlus_ScanVectorsIsAddedToFormValues()
{
var document1 = new Mock <IDocument <dynamic> >();
document1.Setup(x => x.Token).Returns(new MutationToken("bucket1_name", 102, 22, 8282));
var document2 = new Mock <IDocument <dynamic> >();
document2.Setup(x => x.Token).Returns(new MutationToken("bucket1_name", 123, 11, 8332));
var document3 = new Mock <IDocument <dynamic> >();
document3.Setup(x => x.Token).Returns(new MutationToken("bucket2_name", 133, 23, 333));
var queryRequest = new QueryRequest("SELECT * FROM `bucket1_name`;").
ConsistentWith(MutationState.From(document1.Object, document2.Object, document3.Object)).
#pragma warning disable 618
ScanConsistency(ScanConsistency.AtPlus);
#pragma warning restore 618
var actual = queryRequest.GetFormValues()["scan_vectors"];
var expected = "{\"bucket1_name\":{\"102\":[8282,\"22\"],\"123\":[8332,\"11\"]},\"bucket2_name\":{\"133\":[333,\"23\"]}}";
Assert.AreEqual(expected, JsonConvert.SerializeObject(actual));
}
public void GetFormValues_ScanVector_CorrectValues()
{
// Arrange
var token1 = new MutationToken("defaultIndex", 1, 9, 12);
var token2 = new MutationToken("defaultIndex", 2, 1, 22);
var state = new MutationState()
.Add(
// ReSharper disable PossibleUnintendedReferenceComparison
Mock.Of <IMutationResult>(m => m.MutationToken == token1),
Mock.Of <IMutationResult>(m => m.MutationToken == token2));
// ReSharper restore PossibleUnintendedReferenceComparison
var options = new SearchOptions()
.ConsistentWith(state);
// Assert
var json = options.ToJson("defaultIndex");
var vectors = json.SelectToken("ctl.consistency.vectors");
var indexVectors = vectors["defaultIndex"];
Assert.Equal(12, indexVectors["1/9"]);
Assert.Equal(22, indexVectors["2/1"]);
}
public void Test_CloneIdUsedAlready()
{
var cts = new CancellationTokenSource();
var mutationState = new MutationState();
mutationState.Add(new MutationResult(0, TimeSpan.FromSeconds(10), new MutationToken("default", 1, 1, 1)));
var options = new QueryOptions().
AdHoc(true).AutoExecute(true).
CancellationToken(cts.Token).
ClientContextId("clientid").
ConsistentWith(mutationState).
FlexIndex(true).
MaxServerParallelism(1).
Metrics(true).
Parameter(1).
Parameter("name", "value").
PipelineBatch(1).
PipelineCap(1).
PreserveExpiry(true).
Profile(QueryProfile.Off).
Raw("foo", "bar").
Readonly(true).
ScanCap(1).
ScanWait(TimeSpan.FromSeconds(10)).
Timeout(TimeSpan.FromMilliseconds(1)).
Statement("SELECT 1;").
ScanCap(1);
var newOptions = options.CloneIfUsedAlready();
var newValues = newOptions.GetFormValues();
var oldValues = options.GetFormValues();
Assert.Equal(newValues.Count, oldValues.Count);
Assert.Equal(newValues["max_parallelism"], oldValues["max_parallelism"]);
Assert.Equal(newValues["statement"], oldValues["statement"]);
Assert.Equal(newValues["timeout"], oldValues["timeout"]);
Assert.Equal(newValues["readonly"], oldValues["readonly"]);
Assert.Equal(newValues["metrics"], oldValues["metrics"]);
Assert.Equal(newValues["$name"], oldValues["$name"]);
Assert.Equal(newValues["args"], oldValues["args"]);
Assert.Equal(newValues["scan_consistency"], oldValues["scan_consistency"]);
Assert.Equal(newValues["scan_vectors"], oldValues["scan_vectors"]);
Assert.Equal(newValues["scan_wait"], oldValues["scan_wait"]);
Assert.Equal(newValues["scan_cap"], oldValues["scan_cap"]);
Assert.Equal(newValues["pipeline_batch"], oldValues["pipeline_batch"]);
Assert.Equal(newValues["pipeline_cap"], oldValues["pipeline_cap"]);
Assert.Equal(newValues["preserve_expiry"], oldValues["preserve_expiry"]);
Assert.Equal(newValues["foo"], oldValues["foo"]);
Assert.Equal(newValues["auto_execute"], oldValues["auto_execute"]);
Assert.Equal(newValues["client_context_id"], oldValues["client_context_id"]);
Assert.Equal(newValues["use_fts"], oldValues["use_fts"]);
}
/// <summary>
/// Requires that the indexes but up to date with a <see cref="MutationState"/> before the query is executed.
/// </summary>
/// <param name="source">Sets consistency requirement for this query. Must be a Couchbase LINQ query.</param>
/// <param name="state"><see cref="MutationState"/> used for conistency controls.</param>
/// <remarks>If called multiple times, the states from the calls are combined.</remarks>
public static IQueryable <T> ConsistentWith <T>(this IQueryable <T> source, MutationState state)
{
if (source == null)
{
throw new ArgumentNullException(nameof(source));
}
return(source.Provider.CreateQuery <T>(
Expression.Call(
QueryExtensionMethods.ConsistentWith.MakeGenericMethod(typeof(T)),
source.Expression,
Expression.Constant(state))));
}
/// <summary>
/// Requires that the indexes but up to date with a <see cref="N1QL.MutationState"/> before the query is executed.
/// </summary>
/// <param name="state"><see cref="N1QL.MutationState"/> used for conistency controls.</param>
/// <remarks>If called multiple times, the states from the calls are combined.</remarks>
public void ConsistentWith(MutationState state)
{
if (state == null)
{
return;
}
if (MutationState == null)
{
MutationState = new MutationState();
}
MutationState.Add(state);
}
public SearchOptions ConsistentWith(MutationState mutationState)
{
#pragma warning disable 618
ScanConsistency(SearchScanConsistency.AtPlus);
#pragma warning restore 618
_scanVectors = new Dictionary <string, Dictionary <string, List <object> > >();
foreach (var token in mutationState)
{
if (_scanVectors.TryGetValue(token.BucketRef, out var vector))
{
var bucketId = token.VBucketId.ToString();
if (vector.TryGetValue(bucketId, out var bucketRef))
{
if ((long)bucketRef.First() < token.SequenceNumber)
{
vector[bucketId] = new List <object>
{
token.SequenceNumber,
token.VBucketUuid.ToString()
};
}
}
else
{
vector.Add(token.VBucketId.ToString(),
new List <object>
{
token.SequenceNumber,
token.VBucketUuid.ToString()
});
}
}
else
{
_scanVectors.Add(token.BucketRef, new Dictionary <string, List <object> >
{
{
token.VBucketId.ToString(),
new List <object>
{
token.SequenceNumber,
token.VBucketUuid.ToString()
}
}
});
}
}
return(this);
}
/// <summary>
/// Requires that the indexes but up to date with a <see cref="MutationState"/> before the query is executed.
/// </summary>
/// <param name="source">Sets consistency requirement for this query. Must be a Couchbase LINQ query.</param>
/// <param name="state"><see cref="MutationState"/> used for conistency controls.</param>
/// <remarks>If called multiple times, the states from the calls are combined.</remarks>
public static IQueryable <T> ConsistentWith <T>(this IQueryable <T> source, MutationState state)
{
if (source == null)
{
throw new ArgumentNullException("source");
}
if (!(source is IBucketQueryExecutorProvider))
{
// do nothing if this isn't a Couchbase LINQ query
return(source);
}
((IBucketQueryExecutorProvider)source).BucketQueryExecutor.ConsistentWith(state);
return(source);
}
internal virtual void AddToMutationState(MutationToken token)
{
if (token == null || !IsTokenSet(token))
{
// No token was returned, so don't add to the mutation state
return;
}
if (MutationState == null)
{
MutationState = new MutationState();
}
MutationState.Add(new TempDocument()
{
Token = token
});
}
public void MutationState_WhenScanConsistencyIsNotAtPlus_ThrowsArgumentException()
{
var document1 = new Mock <IDocument <dynamic> >();
document1.Setup(x => x.Token).Returns(new MutationToken("bucket1_name", 102, 22, 8282));
var document2 = new Mock <IDocument <dynamic> >();
document2.Setup(x => x.Token).Returns(new MutationToken("bucket1_name", 123, 11, 8332));
var document3 = new Mock <IDocument <dynamic> >();
document3.Setup(x => x.Token).Returns(new MutationToken("bucket2_name", 133, 23, 333));
var queryRequest = new QueryRequest("SELECT * FROM `bucket1_name`;").
ConsistentWith(MutationState.From(document1.Object, document2.Object, document3.Object)).
ScanConsistency(ScanConsistency.NotBounded);
Assert.Throws <ArgumentException>(() => queryRequest.GetFormValues());
}
public async Task ConsistentWith_ScanWait()
{
var context = new BucketContext(TestSetup.Bucket);
var upsertResult = await TestSetup.Bucket.DefaultCollection().UpsertAsync("test-mutation", new { a = "a" });
try
{
var mutationState = MutationState.From(upsertResult);
var beers = from b in context.Query <Beer>().ConsistentWith(mutationState, TimeSpan.FromSeconds(10))
select b;
var beer = await beers.FirstAsync();
Console.WriteLine(beer.Name);
}
finally
{
await TestSetup.Bucket.DefaultCollection().RemoveAsync("test-mutation");
}
}
public async Task GetRequestBody_ScanVector_CorrectFormatting()
{
// Arrange
var token1 = new MutationToken("WHAT", 105, 105, 945678);
var token2 = new MutationToken("WHAT", 105, 105, 955555);
var token3 = new MutationToken("WHAT", 210, 210, 12345);
var state = new MutationState()
.Add(
// ReSharper disable PossibleUnintendedReferenceComparison
Mock.Of <IMutationResult>(m => m.MutationToken == token1),
Mock.Of <IMutationResult>(m => m.MutationToken == token2),
Mock.Of <IMutationResult>(m => m.MutationToken == token3));
// ReSharper restore PossibleUnintendedReferenceComparison
var options = new QueryOptions("SELECT * FROM WHAT").ConsistentWith(state);
// Act
using var content = options.GetRequestBody(GetSerializer());
// Assert
var values = await ExtractValuesAsync(content);
var vectors = (JObject)values["scan_vectors"];
var bucketVectors = (JObject)vectors["WHAT"];
var vBucketComponent1 = (JArray)bucketVectors["105"];
Assert.Equal(955555L, vBucketComponent1[0]);
Assert.Equal("105", vBucketComponent1[1]);
var vBucketComponent2 = (JArray)bucketVectors["210"];
Assert.Equal(12345L, vBucketComponent2[0]);
Assert.Equal("210", vBucketComponent2[1]);
}
public void GetFormValues_ScanVector_CorrectValues()
{
// Arrange
var token1 = new MutationToken("WHAT", 105, 105, 945678);
var token2 = new MutationToken("WHAT", 105, 105, 955555);
var token3 = new MutationToken("WHAT", 210, 210, 12345);
var state = new MutationState()
.Add(
// ReSharper disable PossibleUnintendedReferenceComparison
Mock.Of <IMutationResult>(m => m.MutationToken == token1),
Mock.Of <IMutationResult>(m => m.MutationToken == token2),
Mock.Of <IMutationResult>(m => m.MutationToken == token3));
// ReSharper restore PossibleUnintendedReferenceComparison
var options = new QueryOptions("SELECT * FROM WHAT")
.ConsistentWith(state);
// Assert
var values = options.GetFormValues();
var vectors = (Dictionary <string, Dictionary <string, ScanVectorComponent> >)values["scan_vectors"] !;
var bucketVectors = vectors["WHAT"];
var vBucketComponent1 = bucketVectors["105"];
Assert.Equal(955555L, vBucketComponent1.SequenceNumber);
Assert.Equal(105, vBucketComponent1.VBucketUuid);
var vBucketComponent2 = bucketVectors["210"];
Assert.Equal(12345L, vBucketComponent2.SequenceNumber);
Assert.Equal(210, vBucketComponent2.VBucketUuid);
}
public void ConsistentWith(MutationState state)
{
}
/// <summary>
/// Generates a list of random mutation operations. Any unique row, identified by
/// it's key, could have a random number of operations/mutations. However, the
/// target count of numInserts, numUpdates and numDeletes will always be achieved
/// if the entire list of operations is processed.
/// </summary>
/// <param name="table">The table to generate operations for.</param>
/// <param name="numInserts">The number of row mutations to end with an insert.</param>
/// <param name="numUpdates">The number of row mutations to end with an update.</param>
/// <param name="numDeletes">The number of row mutations to end with an delete.</param>
private List <KuduOperation> GenerateMutationOperations(
KuduTable table, int numInserts, int numUpdates, int numDeletes)
{
var results = new List <KuduOperation>();
var unfinished = new List <MutationState>();
int minMutationsBound = 5;
// Generate Operations to initialize all of the row with inserts.
var changeCounts = new List <(RowOperation type, int count)>
{
(RowOperation.Insert, numInserts),
(RowOperation.Update, numUpdates),
(RowOperation.Delete, numDeletes)
};
foreach (var(type, count) in changeCounts)
{
for (int i = 0; i < count; i++)
{
// Generate a random insert.
var insert = table.NewInsert();
_generator.RandomizeRow(insert);
var key = insert.GetInt32(0);
// Add the insert to the results.
results.Add(insert);
// Initialize the unfinished MutationState.
unfinished.Add(new MutationState(key, type, _random.Next(minMutationsBound)));
}
}
// Randomly pull from the unfinished list, mutate it and add that operation to
// the results. If it has been mutated at least the minimum number of times,
// remove it from the unfinished list.
while (unfinished.Count > 0)
{
// Get a random row to mutate.
int index = _random.Next(unfinished.Count);
MutationState state = unfinished[index];
// If the row is done, remove it from unfinished and continue.
if (state.NumMutations >= state.MinMutations && state.CurrentType == state.EndType)
{
unfinished.RemoveAt(index);
continue;
}
// Otherwise, generate an operation to mutate the row based on its current ChangeType.
// insert -> update|delete
// update -> update|delete
// delete -> insert
KuduOperation op;
if (state.CurrentType == RowOperation.Insert || state.CurrentType == RowOperation.Update)
{
op = _random.NextBool() ? table.NewUpdate() : table.NewDelete();
}
else
{
// Must be a delete, so we need an insert next.
op = table.NewInsert();
}
op.SetInt32(0, state.Key);
if (op.Operation != RowOperation.Delete)
{
_generator.RandomizeRow(op, randomizeKeys: false);
}
results.Add(op);
state.CurrentType = op.Operation;
state.NumMutations++;
}
return(results);
}
/// <summary>
/// Initializes a new instance of the <see cref="ConsistentWithClause" /> class.
/// </summary>
/// <param name="mutationState">Mutation state for the query.</param>
/// <param name="scanWait">Time to wait for index scan.</param>
public ConsistentWithClause(MutationState mutationState, TimeSpan?scanWait)
{
MutationState = mutationState;
ScanWait = scanWait;
}
/// <summary>
/// Provides a means of ensuring "read your own writes" or RYOW consistency on the current query.
/// </summary>
#pragma warning disable 618
/// <remarks>Note: <see cref="ScanConsistency"/> will be overwritten to <see cref="N1QL.ScanConsistency.AtPlus"/>.</remarks>
#pragma warning restore 618
/// <param name="mutationState">State of the mutation.</param>
/// <returns>A reference to the current <see cref="IQueryRequest"/> for method chaining.</returns>
public IQueryRequest ConsistentWith(MutationState mutationState)
{
#pragma warning disable 618
ScanConsistency(N1QL.ScanConsistency.AtPlus);
#pragma warning restore 618
_scanVectors = new Dictionary<string, Dictionary<string, List<object>>>();
foreach (var token in mutationState)
{
Dictionary<string, List<object>> vector;
if (_scanVectors.TryGetValue(token.BucketRef, out vector))
{
var bucketId = token.VBucketId.ToString();
List<object> bucketRef;
if (vector.TryGetValue(bucketId, out bucketRef))
{
if ((long)bucketRef.First() < token.SequenceNumber)
{
vector[bucketId] = new List<object>
{
token.SequenceNumber,
token.VBucketUUID.ToString()
};
}
}
else
{
vector.Add(token.VBucketId.ToString(),
new List<object>
{
token.SequenceNumber,
token.VBucketUUID.ToString()
});
}
}
else
{
_scanVectors.Add(token.BucketRef, new Dictionary<string, List<object>>
{
{
token.VBucketId.ToString(),
new List<object>
{
token.SequenceNumber,
token.VBucketUUID.ToString()
}
}
});
}
}
return this;
}
public void ResetMutationState()
{
_mutationState = new MutationState();;
}
/// <summary>
/// Resets the <see cref="MutationState"/> to start a new set of mutations.
/// </summary>
/// <remarks>
/// If you are using an <see cref="BucketContext"/> over and extended period of time,
/// performing a reset regularly is recommend. This will help keep the size of the
/// <see cref="MutationState"/> to a minimum.
/// </remarks>
public void ResetMutationState()
{
MutationState = null;
}
