public async Task Test_Can_MergeSort()
{
int K = 3;
int N = 100;
using (var db = await OpenTestPartitionAsync())
{
var location = await GetCleanDirectory(db, "Queries", "MergeSort");
// clear!
await db.ClearRangeAsync(location, this.Cancellation);
// create K lists
var lists = Enumerable.Range(0, K).Select(i => location.Partition.ByKey(i)).ToArray();
// lists[0] contains all multiples of K ([0, 0], [K, 1], [2K, 2], ...)
// lists[1] contains all multiples of K, offset by 1 ([1, 0], [K+1, 1], [2K+1, 2], ...)
// lists[k-1] contains all multiples of K, offset by k-1 ([K-1, 0], [2K-1, 1], [3K-1, 2], ...)
// more generally: lists[k][i] = (..., MergeSort, k, (i * K) + k) = (k, i)
for (int k = 0; k < K; k++)
{
using (var tr = db.BeginTransaction(this.Cancellation))
{
for (int i = 0; i < N; i++)
{
tr.Set(lists[k].Keys.Encode((i * K) + k), FdbTuple.EncodeKey(k, i));
}
await tr.CommitAsync();
}
}
// MergeSorting all lists together should produce all integers from 0 to (K*N)-1, in order
// we use the last part of the key for sorting
using (var tr = db.BeginTransaction(this.Cancellation))
{
var merge = tr.MergeSort(
lists.Select(list => FdbKeySelectorPair.Create(list.Keys.ToRange())),
kvp => location.Keys.DecodeLast <int>(kvp.Key)
);
Assert.That(merge, Is.Not.Null);
Assert.That(merge, Is.InstanceOf <FdbMergeSortIterator <KeyValuePair <Slice, Slice>, int, KeyValuePair <Slice, Slice> > >());
var results = await merge.ToListAsync();
Assert.That(results, Is.Not.Null);
Assert.That(results.Count, Is.EqualTo(K * N));
for (int i = 0; i < K * N; i++)
{
Assert.That(location.ExtractKey(results[i].Key), Is.EqualTo(FdbTuple.EncodeKey(i % K, i)));
Assert.That(results[i].Value, Is.EqualTo(FdbTuple.EncodeKey(i % K, i / K)));
}
}
}
}
/// <summary>Returns a Key Selector pair that defines the range of all items contained under this tuple</summary>
public static FdbKeySelectorPair ToSelectorPair([NotNull] this IFdbTuple tuple)
{
if (tuple == null)
{
throw new ArgumentNullException("tuple");
}
return(FdbKeySelectorPair.StartsWith(tuple.ToSlice()));
}
/// <summary>Copy a pair of key selectors into the buffer, and return a new identical pair</summary>
/// <param name="pair">Pair of key selectors to copy to the buffer</param>
/// <returns>Equivalent pair of key selectors that is backed by the buffer.</returns>
public FdbKeySelectorPair InternSelectorPair(FdbKeySelectorPair pair)
{
var begin = Intern(pair.Begin.Key, default(Slice), aligned: true);
var end = Intern(pair.End.Key, default(Slice), aligned: true);
return new FdbKeySelectorPair(
new FdbKeySelector(begin, pair.Begin.OrEqual, pair.Begin.Offset),
new FdbKeySelector(end, pair.End.OrEqual, pair.End.Offset)
);
}
private static async Task BenchMergeSortAsync(IFdbDatabase db, int N, int K, int B, CancellationToken ct)
{
// create multiple lists
var location = db.Partition("MergeSort");
await db.ClearRangeAsync(location, ct);
var sources = Enumerable.Range(0, K).Select(i => 'A' + i).ToArray();
var rnd = new Random();
// insert a number of random number lists
Console.Write("> Inserting " + (K * N).ToString("N0", CultureInfo.InvariantCulture) + " items... ");
foreach (var source in sources)
{
using (var tr = db.BeginTransaction(ct))
{
var list = location.Partition(source);
for (int i = 0; i < N; i++)
{
tr.Set(list.Pack(rnd.Next()), Slice.FromInt32(i));
}
await tr.CommitAsync();
}
}
Console.WriteLine("Done");
// merge/sort them to get only one (hopefully sorted) list
using (var tr = db.BeginTransaction(ct))
{
var mergesort = tr
.MergeSort(
sources.Select(source => FdbKeySelectorPair.StartsWith(location.Pack(source))),
(kvp) => location.UnpackLast <int>(kvp.Key)
)
.Take(B)
.Select(kvp => location.Unpack(kvp.Key));
Console.Write("> MergeSort with limit " + B + "... ");
var sw = Stopwatch.StartNew();
var results = await mergesort.ToListAsync();
sw.Stop();
Console.WriteLine("Done");
Console.WriteLine("Took " + FormatTimeMilli(sw.Elapsed.TotalMilliseconds) + " to merge sort " + results.Count + " results from " + K + " lists of " + N + " items each");
//foreach (var result in results)
//{
// Console.WriteLine(result.Get<int>(-1));
//}
}
}
public FdbRangeQuery <TId> LookupLessThan([NotNull] IFdbReadOnlyTransaction trans, TValue value, bool orEqual, bool reverse = false)
{
var prefix = this.Location.Partial.Keys.Encode(value);
if (orEqual)
{
prefix = FdbKey.Increment(prefix);
}
var space = new FdbKeySelectorPair(
FdbKeySelector.FirstGreaterOrEqual(this.Location.ToRange().Begin),
FdbKeySelector.FirstGreaterThan(prefix)
);
return(trans
.GetRange(space, new FdbRangeOptions {
Reverse = reverse
})
.Select((kvp) => this.Location.Keys.Decode(kvp.Key).Item2));
}
public static FdbQueryRangeExpression RangeStartsWith(Slice prefix, FdbRangeOptions options = null)
{
// starts_with('A') means ['A', B')
return(Range(FdbKeySelectorPair.StartsWith(prefix), options));
}
public static FdbQueryRangeExpression Range(FdbKeySelectorPair range, FdbRangeOptions options = null)
{
return(new FdbQueryRangeExpression(range, options));
}
/// <summary>Return a range of keys</summary>
/// <param name="query">Source database query</param>
/// <param name="range">Pair of key selectors</param>
/// <returns>Query that will return the keys from the specified <paramref name="range"/></returns>
public static IFdbAsyncSequenceQueryable <KeyValuePair <Slice, Slice> > Range(this IFdbDatabaseQueryable query, FdbKeySelectorPair range)
{
if (query == null)
{
throw new ArgumentNullException("query");
}
var expr = FdbQueryExpressions.Range(range);
return(query.Provider.CreateSequenceQuery(expr));
}
public async Task Test_Range_Except()
{
int K = 3;
int N = 100;
using (var db = await OpenTestPartitionAsync())
{
// get a clean new directory
var location = await GetCleanDirectory(db, "Queries", "Except");
// create K lists
var lists = Enumerable.Range(0, K).Select(i => location.Partition.ByKey(i)).ToArray();
// lists[0] contains all multiples of 1
// lists[1] contains all multiples of 2
// lists[k-1] contains all multiples of K
// more generally: lists[k][i] = (..., Intersect, k, i * (k + 1)) = (k, i)
var series = Enumerable.Range(1, K).Select(k => Enumerable.Range(1, N).Select(x => k * x).ToArray()).ToArray();
//foreach(var serie in series)
//{
// Console.WriteLine(String.Join(", ", serie));
//}
for (int k = 0; k < K; k++)
{
//Console.WriteLine("> k = " + k);
using (var tr = db.BeginTransaction(this.Cancellation))
{
for (int i = 0; i < N; i++)
{
var key = lists[k].Keys.Encode(series[k][i]);
var value = FdbTuple.EncodeKey(k, i);
//Console.WriteLine("> " + key + " = " + value);
tr.Set(key, value);
}
await tr.CommitAsync();
}
}
// Intersect all lists together should produce all integers that are prime numbers
IEnumerable <int> xs = series[0];
for (int i = 1; i < K; i++)
{
xs = xs.Except(series[i]);
}
var expected = xs.ToArray();
Log("Expected: {0}", String.Join(", ", expected));
using (var tr = db.BeginTransaction(this.Cancellation))
{
var merge = tr.Except(
lists.Select(list => FdbKeySelectorPair.Create(list.Keys.ToRange())),
kvp => location.Keys.DecodeLast <int>(kvp.Key)
);
Assert.That(merge, Is.Not.Null);
Assert.That(merge, Is.InstanceOf <FdbExceptIterator <KeyValuePair <Slice, Slice>, int, KeyValuePair <Slice, Slice> > >());
var results = await merge.ToListAsync();
Assert.That(results, Is.Not.Null);
Assert.That(results.Count, Is.EqualTo(expected.Length));
for (int i = 0; i < results.Count; i++)
{
Assert.That(location.Keys.DecodeLast <int>(results[i].Key), Is.EqualTo(expected[i]));
}
}
}
}
internal FdbQueryRangeExpression(FdbKeySelectorPair range, FdbRangeOptions options)
{
this.Range = range;
this.Options = options;
}
