public void StartsWithFindsAllKeys()
{
for (int i = 0; i < 10; i++)
{
Client.Put(new RiakObject(Bucket, string.Format("time_{0}", i), EmptyBody,
RiakConstants.ContentTypes.ApplicationJson));
}
#pragma warning disable 618
var mr = new RiakMapReduceQuery();
mr.Inputs(Bucket)
.Filter(f => f.StartsWith("time"))
.MapJs(m => m.Source("function (o) { return [1]; }"))
.ReduceJs(r => r.Name("Riak.reduceSum").Keep(true));
#pragma warning restore 618
var result = Client.MapReduce(mr);
result.IsSuccess.ShouldBeTrue();
var mrResult = result.Value;
mrResult.PhaseResults.ShouldNotBeNull();
mrResult.PhaseResults.Count().ShouldEqual(2);
mrResult.PhaseResults.ElementAt(0).Phase.ShouldEqual(0u);
mrResult.PhaseResults.ElementAt(1).Phase.ShouldEqual(1u);
mrResult.PhaseResults.ElementAt(0).Values.Count().ShouldEqual(0);
mrResult.PhaseResults.ElementAt(1).Values.Count().ShouldNotEqual(0);
var values = result.Value.PhaseResults.ElementAt(1).GetObjects<int[]>().First();
values[0].ShouldEqual(10);
}
public static Func<RiakResult<RiakMapReduceResult>> RunMapReduceQuery(
this IRiakClient client, RiakMapReduceQuery req)
{
Func<RiakResult<RiakMapReduceResult>> runMapRedQuery =
() => client.MapReduce(req);
return runMapRedQuery;
}
public void BuildingSimpleMapReduceJobsWithTimeoutProducesTheCorrectCommand()
{
var query = new RiakMapReduceQuery(100200)
.Inputs("animals")
.MapJs(m => m.Source("function(v) { return [v]; }").Keep(true));
var request = query.ToMessage();
request.content_type.ShouldEqual(MrContentType.ToRiakString());
request.request.ShouldEqual(MrJobWithTimeoutText.ToRiakString());
}
public void BuildingSimpleMapReduceJobsWithTheApiProducesByteArrays()
{
var query = new RiakMapReduceQuery()
.Inputs("animals")
.MapJs(m => m.Source("function(v) { return [v]; }").Keep(true));
var request = query.ToMessage();
request.content_type.ShouldEqual(MrContentType.ToRiakString());
request.request.ShouldEqual(MrJobText.ToRiakString());
}
public void SearchingByNameReturnsTheObjectId()
{
#pragma warning disable 612, 618
var mr = new RiakMapReduceQuery().Inputs(new RiakBucketSearchInput(Bucket, "name:Al*"));
#pragma warning restore 612, 618
var result = Client.MapReduce(mr);
result.IsSuccess.ShouldBeTrue(result.ErrorMessage);
var phaseResults = result.Value.PhaseResults.ToList();
phaseResults.Count.ShouldEqual(1);
CheckThatResultContainAllKeys(result);
}
public void LotsOfConcurrentMapRedRequestsShouldWork()
{
var keys = new List<string>();
for (var i = 1; i < 11; i++)
{
var key = "key" + i;
var doc = new RiakObject(MapReduceBucket, key, new { value = i });
keys.Add(key);
var result = Client.Put(doc, new RiakPutOptions { ReturnBody = true });
result.IsSuccess.ShouldBeTrue();
}
var input = new RiakBucketKeyInput();
keys.ForEach(k => input.Add(new RiakObjectId(MapReduceBucket, k)));
var query = new RiakMapReduceQuery()
.Inputs(input)
.MapJs(m => m.Source(@"function(o){return[1];}"))
.ReduceJs(r => r.Name(@"Riak.reduceSum").Keep(true));
query.Compile();
var results = new List<RiakResult<RiakMapReduceResult>>[ThreadCount];
Parallel.For(0, ThreadCount, i =>
{
results[i] = DoMapRed(query);
});
var failures = 0;
foreach (var r in results.SelectMany(l => l))
{
if (r.IsSuccess)
{
var resultValue = JsonConvert.DeserializeObject<int[]>(r.Value.PhaseResults.ElementAt(1).Values.First().FromRiakString())[0];
resultValue.ShouldEqual(10);
//r.Value.PhaseResults.ElementAt(1).GetObject<int[]>()[0].ShouldEqual(10);
}
else
{
// the only acceptable result is that it ran out of retries when
// talking to the cluster (trying to get a connection)
r.ResultCode.ShouldEqual(ResultCode.NoRetries);
++failures;
}
}
Console.WriteLine("Total of {0} out of {1} failed to execute due to connection contention", failures, ThreadCount * ActionCount);
}
public void EqualsFindsOneKey()
{
for (int i = 0; i < 10; i++)
{
Client.Put(new RiakObject(Bucket, string.Format("time_{0}", i), EmptyBody,
RiakConstants.ContentTypes.ApplicationJson));
}
var mr = new RiakMapReduceQuery();
mr.Inputs(Bucket)
#pragma warning disable 618
.Filter(f => f.Equal("time_8"))
#pragma warning restore 618
.MapJs(m => m.Source("function (o) { return [1]; }"))
.ReduceJs(r => r.Name("Riak.reduceSum").Keep(true));
var result = Client.MapReduce(mr);
result.IsSuccess.ShouldBeTrue();
var mrResult = result.Value;
mrResult.PhaseResults.ShouldNotBeNull();
mrResult.PhaseResults.Count().ShouldEqual(2);
mrResult.PhaseResults.ElementAt(0).Phase.ShouldEqual(0u);
mrResult.PhaseResults.ElementAt(1).Phase.ShouldEqual(1u);
mrResult.PhaseResults.ElementAt(0).Values.Count().ShouldEqual(0);
mrResult.PhaseResults.ElementAt(1).Values.Count().ShouldNotEqual(0);
var values = JsonConvert.DeserializeObject<int[]>(mrResult.PhaseResults
.ElementAt(1)
.Values
.First()
.FromRiakString());
values[0].ShouldEqual(1);
}
public void StartsWithAndBetweenReturnASubsetOfAllKeys()
{
for (var i = 0; i < 10; i++)
{
Client.Put(new RiakObject(Bucket, string.Format("time_{0}", i), EmptyBody,
RiakConstants.ContentTypes.ApplicationJson));
}
var mr = new RiakMapReduceQuery();
mr.Inputs(Bucket)
#pragma warning disable 618
.Filter(f => f.And(l => l.StartsWith("time"),
r => r.Tokenize("_", 2)
.StringToInt()
.Between(3, 7, true)))
#pragma warning restore 618
.MapJs(m => m.Source("function (o) { return [1]; }").Keep(false))
.ReduceJs(r => r.Name("Riak.reduceSum").Keep(true));
var result = Client.MapReduce(mr);
result.IsSuccess.ShouldBeTrue();
var mrResult = result.Value;
mrResult.PhaseResults.ShouldNotBeNull();
mrResult.PhaseResults.Count().ShouldEqual(2);
var values = result.Value.PhaseResults.ElementAt(1).GetObjects<int[]>().First();
values[0].ShouldEqual(5);
}
public void KeysReturnsSelectiveListOfKeys()
{
const int keyStart = 10;
const int keyCount = 10;
const int idxStart = 12;
const int idxEnd = 16;
const string keyPrefix = "KeysReturnsSelectiveListOfKeys";
var originalKeys = new List<string>();
for (var i = keyStart; i < keyStart + keyCount; i++)
{
var o = CreateIntKeyedRiakObject(keyPrefix, true, i);
originalKeys.Add(o.Key);
Client.Put(o);
}
var mr = new RiakMapReduceQuery()
.Inputs(RiakIndex.Keys(LegacyBucket, keyPrefix + "_" + idxStart, keyPrefix + "_" + idxEnd));
var result = Client.RunMapReduceQuery(mr).WaitUntil(MapReduceTestHelpers.OnePhaseWithFiveResultsFound);
var keys = result.Value.PhaseResults.SelectMany(x => x.GetObjectIds()).ToList();
result.IsSuccess.ShouldBeTrue(result.ErrorMessage);
keys.Count.ShouldEqual(5);
foreach (var key in keys)
{
key.Bucket.ShouldNotBeNullOrEmpty();
key.Key.ShouldNotBeNullOrEmpty();
originalKeys.Contains(key.Key).ShouldBeTrue();
}
}
public void AllKeysReturnsListOfKeys()
{
var insertedKeys = GenerateIntKeyObjects("AllKeys");
var mr = new RiakMapReduceQuery().Inputs(RiakIndex.AllKeys(Bucket));
var result = Client.RunMapReduceQuery(mr).WaitUntil(MapReduceTestHelpers.OnePhaseWithTenResultsFound);
var queriedKeys = result.Value.PhaseResults.SelectMany(x => x.GetObjectIds()).ToList();
result.IsSuccess.ShouldBeTrue(result.ErrorMessage);
var foundKeys = string.Join("\",\r\n\t\"", queriedKeys);
if (foundKeys.Length > 0)
{
foundKeys = "Found keys: \"" + foundKeys + "\"";
}
CollectionAssert.IsNotEmpty(queriedKeys);
Assert.AreEqual(DefaultKeyCount, queriedKeys.Count());
foreach (var key in queriedKeys)
{
key.Bucket.ShouldNotBeNullOrEmpty();
key.Key.ShouldNotBeNullOrEmpty();
insertedKeys.Contains(key.Key).ShouldBeTrue();
}
}
public void LotsOfConcurrentStreamingMapRedRequestsShouldWork()
{
var keys = new List<string>();
for (var i = 1; i < 11; i++)
{
var key = "key" + i;
var doc = new RiakObject(MapReduceBucket, key, new { value = i });
keys.Add(key);
var result = Client.Put(doc, new RiakPutOptions { ReturnBody = true });
result.IsSuccess.ShouldBeTrue();
}
var input = new RiakBucketKeyInput();
keys.ForEach(k => input.Add(new RiakObjectId(MapReduceBucket, k)));
var query = new RiakMapReduceQuery()
.Inputs(input)
.MapJs(m => m.Source(@"function(o){return[1];}"))
.ReduceJs(r => r.Name(@"Riak.reduceSum").Keep(true));
query.Compile();
var results = new List<RiakMapReduceResultPhase>[ThreadCount];
Parallel.For(0, ThreadCount, i =>
{
results[i] = DoStreamingMapRed(query);
});
var failures = 0;
foreach (var result in results)
{
if (result.Count > 0)
{
var lastResult = result.OrderByDescending(r => r.Phase).First();
var resultValue = JsonConvert.DeserializeObject<int[]>(lastResult.Values.First().FromRiakString());
//var resultValue = JsonConvert.DeserializeObject<int[]>(r.Value.PhaseResults.ElementAt(1).Values.First().FromRiakString())[0];
// due to the speed which things happen at, we can't gaurantee all 10 will be in the result set
resultValue[0].IsAtLeast(5);
//lastResult.GetObject<int[]>()[0].ShouldEqual(10);
}
else
{
++failures;
}
}
Console.WriteLine("Total of {0} out of {1} failed to execute due to connection contention", failures, ThreadCount * ActionCount);
}
public void SearchingComplexQueryWorks()
{
var search = new RiakFluentSearch(Index, "name_s");
// name_s:{integer}Al* AND bio_tsd:awesome AND bio_tsd:an AND (bio_tsd:mathematician OR favorites.album_tsd:Fame)
search.Search(Token.StartsWith(_randomId + "Al"))
.And("bio_tsd", "awesome")
.And("an")
.And("mathematician", s => s.Or("favorites.album_tsd", "Fame"));
var mr = new RiakMapReduceQuery().Inputs(new RiakSearchInput(search));
var result = Client.RunMapReduceQuery(mr).WaitUntil(MapReduceTestHelpers.OnePhaseWithOneResultFound);
result.IsSuccess.ShouldBeTrue(result.ErrorMessage);
var singleResult = result.Value.PhaseResults.First().Values[0].FromRiakString();
var failureMessage = string.Format("Results did not contain \"{0}\". \r\nResult was:\"{1}\"",
PublicKey, singleResult);
singleResult.Contains(PublicKey).ShouldBeTrue(failureMessage);
}
private List<RiakResult<RiakMapReduceResult>> DoMapRed(RiakMapReduceQuery query)
{
var client = Cluster.CreateClient();
var results = ActionCount.Times(() => client.MapReduce(query));
return results.ToList();
}
public void RiakObjectLinksAreTheSameAsLinksRetrievedViaMapReduce()
{
var jeremiah = Client.Get(TestBucket, Jeremiah).Value;
var jLinks = jeremiah.Links;
var input = new RiakBucketKeyInput()
.Add(new RiakObjectId(TestBucket, Jeremiah));
#pragma warning disable 618
var query = new RiakMapReduceQuery().Inputs(input).Link(l => l.AllLinks().Keep(true));
#pragma warning restore 618
var mrResult = Client.MapReduce(query);
mrResult.IsSuccess.ShouldBeTrue();
// TODO: FUTURE - Is *this* chunk of code acceptable?
// This should probably be taken care of in the RiakClient.WalkLinks
var listOfLinks = mrResult.Value.PhaseResults.OrderBy(pr => pr.Phase)
.ElementAt(0).Values
.Select(v => RiakLink.ParseArrayFromJsonString(v.FromRiakString()));
var mrLinks = listOfLinks.SelectMany(l => l).ToList();
mrLinks.Count().ShouldEqual(jLinks.Count);
foreach (var link in jLinks)
{
mrLinks.ShouldContain(link);
}
}
public void BuildingMapReducePhaseWithVaueTypeArgumentProducesCorrectResult()
{
var query = new RiakMapReduceQuery()
.Inputs("animals")
.ReduceJs(c => c.Name("Riak.reduceSlice").Keep(true).Argument("slartibartfast"));
var request = query.ToMessage();
request.request.ShouldEqual(MrJobWithValueTypeArgument.ToRiakString());
}
public void BuildingMapReducePhaseWithObjectArgumentProducesCorrectResult()
{
var query = new RiakMapReduceQuery()
.Inputs("animals")
.ReduceJs(c => c.Name("Riak.reduceSlice").Keep(true).Argument(new { reduce_phase_only_1 = true }));
var request = query.ToMessage();
request.request.ShouldEqual(MrJobWithObjectArgument.ToRiakString());
}
public void QueryingDollarKeyDoesNotAppendBinIndexSuffix()
{
var query = new RiakMapReduceQuery()
.Inputs(RiakIndex.Range(new RiakIndexId("animals", "$key"), "0", "zzzzz"));
var request = query.ToMessage();
var requestString = request.request.FromRiakString();
requestString.Contains("$key").ShouldBeTrue();
requestString.Contains("$key_bin").ShouldBeFalse();
}
private List<RiakMapReduceResultPhase> DoStreamingMapRed(RiakMapReduceQuery query)
{
var client = Cluster.CreateClient();
var results = ActionCount.Times(() =>
{
var streamedResults = client.StreamMapReduce(query);
if (streamedResults.IsSuccess)
{
return streamedResults.Value.PhaseResults;
}
return null;
}).Where(r => r != null).SelectMany(r => r);
return results.ToList();
}
public void SearchingViaFluentSearchObjectWorks()
{
var search = new RiakFluentSearch(Bucket, "name").Search(Token.StartsWith("Al")).Build();
#pragma warning disable 612, 618
var mr = new RiakMapReduceQuery().Inputs(new RiakBucketSearchInput(search));
#pragma warning restore 612, 618
var result = Client.MapReduce(mr);
result.IsSuccess.ShouldBeTrue(result.ErrorMessage);
CheckThatResultContainAllKeys(result);
}
public void SearchingByNameReturnsTheObjectId()
{
var mr = new RiakMapReduceQuery()
.Inputs(new RiakSearchInput(Index, "name_s:" + _randomId + "Al*"));
var result = Client.RunMapReduceQuery(mr).WaitUntil(MapReduceTestHelpers.OnePhaseWithTwoResultsFound);
result.IsSuccess.ShouldBeTrue(result.ErrorMessage);
var phaseResults = result.Value.PhaseResults.ToList();
phaseResults.Count.ShouldEqual(1);
AssertThatResultContainsAllKeys(result);
}
public void LinksAreRetrievedWithAMapReducePhase()
{
#pragma warning disable 618
var query = new RiakMapReduceQuery()
.Inputs(TestBucket)
.Filter(f => f.Matches(Jeremiah))
.Link(l => l.Tag("friends").Bucket(TestBucket).Keep(false))
.ReduceErlang(r => r.ModFun("riak_kv_mapreduce", "reduce_set_union").Keep(true));
#pragma warning restore 618
var result = Client.MapReduce(query);
result.IsSuccess.ShouldBeTrue();
var mrResult = result.Value;
mrResult.PhaseResults.ShouldNotBeNull();
mrResult.PhaseResults.Count().ShouldEqual(2);
}
public void BuildingComplexMapReduceJobsWithTheApiProducesTheCorrectCommand()
{
var query = new RiakMapReduceQuery()
.Inputs("animals")
.MapJs(m => m.Source("function(o) { if (o.key.indexOf('spider') != -1) return [1]; return []; }"))
.ReduceJs(r => r.Name("Riak.reduceSum").Keep(true));
var request = query.ToMessage();
request.request.ShouldEqual(ComplexMrJobText.ToRiakString());
}
public void SearchingViaFluentSearchObjectWorks()
{
var search = new RiakFluentSearch(Index, "name_s").Search(Token.StartsWith(_randomId + "Al")).Build();
var mr = new RiakMapReduceQuery().Inputs(new RiakSearchInput(search));
var result = Client.RunMapReduceQuery(mr).WaitUntil(MapReduceTestHelpers.OnePhaseWithTwoResultsFound);
result.IsSuccess.ShouldBeTrue(result.ErrorMessage);
AssertThatResultContainsAllKeys(result);
}
public void BuildingComplexMapReduceJobsWithFiltersProducesTheCorrectCommand()
{
#pragma warning disable 618
var query = new RiakMapReduceQuery()
.Inputs("animals")
.Filter(f => f.Matches("spider"))
.MapJs(m => m.Source("function(o) { return [1]; }"))
.ReduceJs(r => r.Name("Riak.reduceSum").Keep(true));
#pragma warning restore 618
var request = query.ToMessage();
request.request.ShouldEqual(ComplexMrJobWithFilterText.ToRiakString());
}
public void MapReduceQueriesReturnDataInBatch()
{
var bucket = Guid.NewGuid().ToString();
Client.Batch(batch =>
{
for (var i = 1; i < 11; i++)
{
var doc = new RiakObject(bucket, i.ToString(), new { value = i });
batch.Put(doc).IsSuccess.ShouldBeTrue();
}
var query = new RiakMapReduceQuery()
.Inputs(bucket)
.MapJs(m => m.Source(@"function(o) {return [ 1 ];}"))
.ReduceJs(r => r.Name(@"Riak.reduceSum").Keep(true));
var result = batch.MapReduce(query);
result.IsSuccess.ShouldBeTrue();
var mrRes = result.Value;
mrRes.PhaseResults.ShouldNotBeNull();
mrRes.PhaseResults.Count().ShouldEqual(2);
mrRes.PhaseResults.ElementAt(0).Phase.ShouldEqual(0u);
mrRes.PhaseResults.ElementAt(1).Phase.ShouldEqual(1u);
//mrRes.PhaseResults.ElementAt(0).Values.ShouldBeNull();
foreach (var v in mrRes.PhaseResults.ElementAt(0).Values)
{
v.ShouldBeNull();
}
mrRes.PhaseResults.ElementAt(1).Values.ShouldNotBeNull();
var values = result.Value.PhaseResults.ElementAt(1).GetObjects<int[]>().First();
values[0].ShouldEqual(10);
});
}
public void QueryingByIntIndexReturnsAListOfKeys()
{
GenerateGuidKeyObjects("QueryByIntIndex", (o, i) => o.IntIndex("age").Set(32, 20));
var mr = new RiakMapReduceQuery()
.Inputs(RiakIndex.Match(new RiakIndexId(LegacyBucket, "age"), 32));
var result = Client.MapReduce(mr);
Assert.IsTrue(result.IsSuccess, result.ErrorMessage);
var keys = result.Value.PhaseResults.SelectMany(x => x.GetObjectIds()).ToList();
Assert.GreaterOrEqual(keys.Count(), DefaultKeyCount);
foreach (var key in keys)
{
Assert.True(EnumerableUtil.NotNullOrEmpty(key.Bucket));
Assert.True(EnumerableUtil.NotNullOrEmpty(key.Key));
}
}