/// <summary>
/// Simulate a student that is really indecisive
/// </summary>
public async Task RunProducer(IFdbDatabase db, CancellationToken ct)
{
int cnt = 0;
var rnd = new Random();
this.TimeLine.Start();
while (!ct.IsCancellationRequested)
{
int k = cnt++;
Slice taskId = TuPack.EncodeKey(this.Id.GetHashCode(), k);
string msg = "Message #" + k + " from producer " + this.Id + " (" + DateTime.UtcNow.ToString("O") + ")";
var latency = Stopwatch.StartNew();
await this.WorkerPool.ScheduleTaskAsync(db, taskId, Slice.FromString(msg), ct).ConfigureAwait(false);
latency.Stop();
Console.Write(k.ToString("N0") + "\r");
this.TimeLine.Add(latency.Elapsed.TotalMilliseconds);
TimeSpan delay = TimeSpan.FromTicks(rnd.Next((int)this.DelayMin.Ticks, (int)this.DelayMax.Ticks));
await Task.Delay(delay, ct).ConfigureAwait(false);
}
this.TimeLine.Stop();
ct.ThrowIfCancellationRequested();
}
public void Test_KeyRange_Test()
{
const int BEFORE = -1, INSIDE = 0, AFTER = +1;
KeyRange range;
// range: [ "A", "Z" )
range = KeyRange.Create(Slice.FromByteString("A"), Slice.FromByteString("Z"));
// Excluding the end: < "Z"
Assert.That(range.Test(Slice.FromByteString("\x00"), endIncluded: false), Is.EqualTo(BEFORE));
Assert.That(range.Test(Slice.FromByteString("@"), endIncluded: false), Is.EqualTo(BEFORE));
Assert.That(range.Test(Slice.FromByteString("A"), endIncluded: false), Is.EqualTo(INSIDE));
Assert.That(range.Test(Slice.FromByteString("Z"), endIncluded: false), Is.EqualTo(AFTER));
Assert.That(range.Test(Slice.FromByteString("Z\x00"), endIncluded: false), Is.EqualTo(AFTER));
Assert.That(range.Test(Slice.FromByteString("\xFF"), endIncluded: false), Is.EqualTo(AFTER));
// Including the end: <= "Z"
Assert.That(range.Test(Slice.FromByteString("\x00"), endIncluded: true), Is.EqualTo(BEFORE));
Assert.That(range.Test(Slice.FromByteString("@"), endIncluded: true), Is.EqualTo(BEFORE));
Assert.That(range.Test(Slice.FromByteString("A"), endIncluded: true), Is.EqualTo(INSIDE));
Assert.That(range.Test(Slice.FromByteString("Z"), endIncluded: true), Is.EqualTo(INSIDE));
Assert.That(range.Test(Slice.FromByteString("Z\x00"), endIncluded: true), Is.EqualTo(AFTER));
Assert.That(range.Test(Slice.FromByteString("\xFF"), endIncluded: true), Is.EqualTo(AFTER));
range = KeyRange.Create(TuPack.EncodeKey("A"), TuPack.EncodeKey("Z"));
Assert.That(range.Test(TuPack.EncodeKey("@")), Is.EqualTo((BEFORE)));
Assert.That(range.Test(TuPack.EncodeKey("A")), Is.EqualTo((INSIDE)));
Assert.That(range.Test(TuPack.EncodeKey("Z")), Is.EqualTo((AFTER)));
Assert.That(range.Test(TuPack.EncodeKey("Z"), endIncluded: true), Is.EqualTo(INSIDE));
}
public async Task Test_Vector_Fast()
{
using (var db = await OpenTestPartitionAsync())
{
var location = db.Root["ranked_set"];
await CleanLocation(db, location);
var rankedSet = new FdbRankedSet(location);
await db.WriteAsync(tr => rankedSet.OpenAsync(tr), this.Cancellation);
Log(await rankedSet.ReadAsync(db, (tr, state) => PrintRankedSet(state, tr), this.Cancellation));
Log();
var rnd = new Random();
var sw = Stopwatch.StartNew();
for (int i = 0; i < 100; i++)
{
//Log("Inserting " + i);
await db.WriteAsync(async tr =>
{
var state = await rankedSet.Resolve(tr);
await state.InsertAsync(tr, TuPack.EncodeKey(rnd.Next()));
}, this.Cancellation);
}
sw.Stop();
Log($"Done in {sw.Elapsed.TotalSeconds:N3} sec");
#if FULL_DEBUG
await DumpSubspace(db, location);
#endif
Log(await rankedSet.ReadAsync(db, (tr, state) => PrintRankedSet(state, tr), this.Cancellation));
}
}
public void Test_Simple_SelfTerms_Codecs()
{
// encodes a key using 3 parts: (x, y, z) => ordered_key
string x = "abc";
long y = 123;
Guid z = Guid.NewGuid();
var first = TupleCodec <string> .Default;
var second = TupleCodec <long> .Default;
var third = TupleCodec <Guid> .Default;
var writer = default(SliceWriter);
first.EncodeOrderedSelfTerm(ref writer, x);
second.EncodeOrderedSelfTerm(ref writer, y);
third.EncodeOrderedSelfTerm(ref writer, z);
var data = writer.ToSlice();
Assert.That(data, Is.EqualTo(TuPack.EncodeKey(x, y, z)));
var reader = new SliceReader(data);
Assert.That(first.DecodeOrderedSelfTerm(ref reader), Is.EqualTo(x));
Assert.That(second.DecodeOrderedSelfTerm(ref reader), Is.EqualTo(y));
Assert.That(third.DecodeOrderedSelfTerm(ref reader), Is.EqualTo(z));
Assert.That(reader.HasMore, Is.False);
}
public void Test_Subspace_With_Tuple_Prefix()
{
var subspace = KeySubspace.CreateDynamic(TuPack.EncodeKey("hello"));
Assert.That(subspace.GetPrefix().ToString(), Is.EqualTo("<02>hello<00>"));
Assert.That(subspace.Copy(), Is.Not.SameAs(subspace));
Assert.That(subspace.Copy().GetPrefix(), Is.EqualTo(subspace.GetPrefix()));
// concat(Slice) should append the slice to the tuple prefix directly
Assert.That(subspace[Slice.FromInt32(0x01020304)].ToString(), Is.EqualTo("<02>hello<00><04><03><02><01>"));
Assert.That(subspace[Slice.FromStringAscii("world")].ToString(), Is.EqualTo("<02>hello<00>world"));
// pack(...) should use tuple serialization
Assert.That(subspace.Keys.Encode(123).ToString(), Is.EqualTo("<02>hello<00><15>{"));
Assert.That(subspace.Keys.Encode("world").ToString(), Is.EqualTo("<02>hello<00><02>world<00>"));
// even though the subspace prefix is a tuple, appending to it will only return the new items
var k = subspace.Keys.Pack(("world", 123, false));
Assert.That(k.ToString(), Is.EqualTo("<02>hello<00><02>world<00><15>{<14>"));
// if we unpack the key with the binary prefix, we should get a valid tuple
var t2 = subspace.Keys.Unpack(k);
Assert.That(t2, Is.Not.Null);
Assert.That(t2.Count, Is.EqualTo(3));
Assert.That(t2.Get <string>(0), Is.EqualTo("world"));
Assert.That(t2.Get <int>(1), Is.EqualTo(123));
Assert.That(t2.Get <bool>(2), Is.False);
}
public async Task Test_Vector_Fast()
{
using (var db = await OpenTestPartitionAsync())
{
var location = await GetCleanDirectory(db, "ranked_set");
var vector = new FdbRankedSet(location);
await db.ReadWriteAsync(async (tr) =>
{
await vector.OpenAsync(tr);
await PrintRankedSet(vector, tr);
}, this.Cancellation);
Log();
var rnd = new Random();
var sw = Stopwatch.StartNew();
for (int i = 0; i < 100; i++)
{
Console.Write("\rInserting " + i);
await db.ReadWriteAsync((tr) => vector.InsertAsync(tr, TuPack.EncodeKey(rnd.Next())), this.Cancellation);
}
sw.Stop();
Log("\rDone in {0:N3} sec", sw.Elapsed.TotalSeconds);
await db.ReadAsync((tr) => PrintRankedSet(vector, tr), this.Cancellation);
}
}
public async Task Test_Can_MergeSort()
{
int K = 3;
int N = 100;
using (var db = await OpenTestPartitionAsync())
{
var location = await GetCleanDirectory(db, "Queries", "MergeSort");
// clear!
await db.ClearRangeAsync(location, this.Cancellation);
// create K lists
var lists = Enumerable.Range(0, K).Select(i => location.Partition.ByKey(i)).ToArray();
// lists[0] contains all multiples of K ([0, 0], [K, 1], [2K, 2], ...)
// lists[1] contains all multiples of K, offset by 1 ([1, 0], [K+1, 1], [2K+1, 2], ...)
// lists[k-1] contains all multiples of K, offset by k-1 ([K-1, 0], [2K-1, 1], [3K-1, 2], ...)
// more generally: lists[k][i] = (..., MergeSort, k, (i * K) + k) = (k, i)
for (int k = 0; k < K; k++)
{
using (var tr = db.BeginTransaction(this.Cancellation))
{
for (int i = 0; i < N; i++)
{
tr.Set(lists[k].Keys.Encode((i * K) + k), TuPack.EncodeKey(k, i));
}
await tr.CommitAsync();
}
}
// MergeSorting all lists together should produce all integers from 0 to (K*N)-1, in order
// we use the last part of the key for sorting
using (var tr = db.BeginTransaction(this.Cancellation))
{
var merge = tr.MergeSort(
lists.Select(list => KeySelectorPair.Create(list.Keys.ToRange())),
kvp => location.Keys.DecodeLast <int>(kvp.Key)
);
Assert.That(merge, Is.Not.Null);
Assert.That(merge, Is.InstanceOf <MergeSortAsyncIterator <KeyValuePair <Slice, Slice>, int, KeyValuePair <Slice, Slice> > >());
var results = await merge.ToListAsync();
Assert.That(results, Is.Not.Null);
Assert.That(results.Count, Is.EqualTo(K * N));
for (int i = 0; i < K * N; i++)
{
Assert.That(location.ExtractKey(results[i].Key), Is.EqualTo(TuPack.EncodeKey(i % K, i)));
Assert.That(results[i].Value, Is.EqualTo(TuPack.EncodeKey(i % K, i / K)));
}
}
}
}
public void Test_FdbKey_PrettyPrint()
{
// verify that the pretty printing of keys produce a user friendly output
Assert.That(FdbKey.Dump(Slice.Nil), Is.EqualTo("<null>"));
Assert.That(FdbKey.Dump(Slice.Empty), Is.EqualTo("<empty>"));
Assert.That(FdbKey.Dump(Slice.FromByte(0)), Is.EqualTo("<00>"));
Assert.That(FdbKey.Dump(Slice.FromByte(255)), Is.EqualTo("<FF>"));
Assert.That(FdbKey.Dump(new byte[] { 0, 1, 2, 3, 4, 5, 6, 7 }.AsSlice()), Is.EqualTo("<00><01><02><03><04><05><06><07>"));
Assert.That(FdbKey.Dump(new byte[] { 255, 254, 253, 252, 251, 250, 249, 248 }.AsSlice()), Is.EqualTo("<FF><FE><FD><FC><FB><FA><F9><F8>"));
Assert.That(FdbKey.Dump(Slice.FromString("hello")), Is.EqualTo("hello"));
Assert.That(FdbKey.Dump(Slice.FromString("héllø")), Is.EqualTo("h<C3><A9>ll<C3><B8>"));
// tuples should be decoded properly
Assert.That(FdbKey.Dump(TuPack.EncodeKey(123)), Is.EqualTo("(123,)"), "Singleton tuples should end with a ','");
Assert.That(FdbKey.Dump(TuPack.EncodeKey(Slice.FromByteString("hello"))), Is.EqualTo("(`hello`,)"), "ASCII strings should use single back quotes");
Assert.That(FdbKey.Dump(TuPack.EncodeKey("héllø")), Is.EqualTo("(\"héllø\",)"), "Unicode strings should use double quotes");
Assert.That(FdbKey.Dump(TuPack.EncodeKey(new byte[] { 1, 2, 3 }.AsSlice())), Is.EqualTo("(`<01><02><03>`,)"));
Assert.That(FdbKey.Dump(TuPack.EncodeKey(123, 456)), Is.EqualTo("(123, 456)"), "Elements should be separated with a space, and not end up with ','");
Assert.That(FdbKey.Dump(TuPack.EncodeKey(default(object), true, false)), Is.EqualTo("(null, true, false)"), "Booleans should be displayed as numbers, and null should be in lowercase"); //note: even though it's tempting to using Python's "Nil", it's not very ".NETty"
//note: the string representation of double is not identical between NetFx and .NET Core! So we cannot used a constant literal here
Assert.That(FdbKey.Dump(TuPack.EncodeKey(1.0d, Math.PI, Math.E)), Is.EqualTo("(1, " + Math.PI.ToString("R", CultureInfo.InvariantCulture) + ", " + Math.E.ToString("R", CultureInfo.InvariantCulture) + ")"), "Doubles should used dot and have full precision (17 digits)");
Assert.That(FdbKey.Dump(TuPack.EncodeKey(1.0f, (float)Math.PI, (float)Math.E)), Is.EqualTo("(1, " + ((float)Math.PI).ToString("R", CultureInfo.InvariantCulture) + ", " + ((float)Math.E).ToString("R", CultureInfo.InvariantCulture) + ")"), "Singles should used dot and have full precision (10 digits)");
var guid = Guid.NewGuid();
Assert.That(FdbKey.Dump(TuPack.EncodeKey(guid)), Is.EqualTo($"({guid:B},)"), "GUIDs should be displayed as a string literal, surrounded by {{...}}, and without quotes");
var uuid128 = Uuid128.NewUuid();
Assert.That(FdbKey.Dump(TuPack.EncodeKey(uuid128)), Is.EqualTo($"({uuid128:B},)"), "Uuid128s should be displayed as a string literal, surrounded by {{...}}, and without quotes");
var uuid64 = Uuid64.NewUuid();
Assert.That(FdbKey.Dump(TuPack.EncodeKey(uuid64)), Is.EqualTo($"({uuid64:B},)"), "Uuid64s should be displayed as a string literal, surrounded by {{...}}, and without quotes");
// ranges should be decoded when possible
var key = TuPack.ToRange(STuple.Create("hello"));
// "<02>hello<00><00>" .. "<02>hello<00><FF>"
Assert.That(FdbKey.PrettyPrint(key.Begin, FdbKey.PrettyPrintMode.Begin), Is.EqualTo("(\"hello\",).<00>"));
Assert.That(FdbKey.PrettyPrint(key.End, FdbKey.PrettyPrintMode.End), Is.EqualTo("(\"hello\",).<FF>"));
key = KeyRange.StartsWith(TuPack.EncodeKey("hello"));
// "<02>hello<00>" .. "<02>hello<01>"
Assert.That(FdbKey.PrettyPrint(key.Begin, FdbKey.PrettyPrintMode.Begin), Is.EqualTo("(\"hello\",)"));
Assert.That(FdbKey.PrettyPrint(key.End, FdbKey.PrettyPrintMode.End), Is.EqualTo("(\"hello\",) + 1"));
var t = TuPack.EncodeKey(123);
Assert.That(FdbKey.PrettyPrint(t, FdbKey.PrettyPrintMode.Single), Is.EqualTo("(123,)"));
Assert.That(FdbKey.PrettyPrint(TuPack.ToRange(t).Begin, FdbKey.PrettyPrintMode.Begin), Is.EqualTo("(123,).<00>"));
Assert.That(FdbKey.PrettyPrint(TuPack.ToRange(t).End, FdbKey.PrettyPrintMode.End), Is.EqualTo("(123,).<FF>"));
}
private static async Task HelloWorld(CancellationToken ct)
{
// Connect to the "DB" database on the local cluster
using (var db = await Fdb.OpenAsync())
{
// Writes some data in to the database
using (var tr = db.BeginTransaction(ct))
{
tr.Set(TuPack.EncodeKey("Test", 123), Slice.FromString("Hello World!"));
tr.Set(TuPack.EncodeKey("Test", 456), Slice.FromInt64(DateTime.UtcNow.Ticks));
}
}
}
public void Test_Simple_String_Codec()
{
var codec = TupleCodec <string> .Default;
Assert.That(codec, Is.Not.Null);
Assert.That(codec.EncodeOrdered("héllø Wörld"), Is.EqualTo(TuPack.EncodeKey("héllø Wörld")));
Assert.That(codec.EncodeOrdered(String.Empty), Is.EqualTo(TuPack.EncodeKey("")));
Assert.That(codec.EncodeOrdered(null), Is.EqualTo(TuPack.EncodeKey(default(string))));
Assert.That(codec.DecodeOrdered(TuPack.EncodeKey("héllø Wörld")), Is.EqualTo("héllø Wörld"));
Assert.That(codec.DecodeOrdered(TuPack.EncodeKey(String.Empty)), Is.EqualTo(""));
Assert.That(codec.DecodeOrdered(TuPack.EncodeKey(default(string))), Is.Null);
}
public void Test_Simple_Integer_Codec()
{
var codec = TupleCodec <int> .Default;
Assert.That(codec, Is.Not.Null);
Assert.That(codec.EncodeOrdered(0), Is.EqualTo(TuPack.EncodeKey(0)));
Assert.That(codec.EncodeOrdered(123), Is.EqualTo(TuPack.EncodeKey(123)));
Assert.That(codec.EncodeOrdered(123456), Is.EqualTo(TuPack.EncodeKey(123456)));
Assert.That(codec.DecodeOrdered(TuPack.EncodeKey(0)), Is.EqualTo(0));
Assert.That(codec.DecodeOrdered(TuPack.EncodeKey(123)), Is.EqualTo(123));
Assert.That(codec.DecodeOrdered(TuPack.EncodeKey(123456)), Is.EqualTo(123456));
}
public void Test_Tuple_Composite_Encoder()
{
// encodes a key using 3 parts: (x, y, z) => ordered_key
string x = "abc";
long y = 123;
Guid z = Guid.NewGuid();
var encoder = TuPack.Encoding.GetKeyEncoder <string, long, Guid>();
Assert.That(encoder, Is.Not.Null);
// full key encoding
// note: EncodeKey(...) is just a shortcurt for packing all items in a tuple, and EncodeComposite(..., count = 3)
var data = encoder.EncodeKey(x, y, z);
Assert.That(data, Is.EqualTo(TuPack.EncodeKey(x, y, z)));
var items = encoder.DecodeKey(data);
Assert.That(items.Item1, Is.EqualTo(x));
Assert.That(items.Item2, Is.EqualTo(y));
Assert.That(items.Item3, Is.EqualTo(z));
// partial key encoding
data = encoder.EncodeKeyParts(2, items);
Assert.That(data, Is.EqualTo(TuPack.EncodeKey(x, y)));
items = encoder.DecodeKeyParts(2, TuPack.EncodeKey(x, y));
Assert.That(items.Item1, Is.EqualTo(x));
Assert.That(items.Item2, Is.EqualTo(y));
Assert.That(items.Item3, Is.EqualTo(default(Guid)));
data = encoder.EncodeKeyParts(1, items);
Assert.That(data, Is.EqualTo(TuPack.EncodeKey(x)));
items = encoder.DecodeKeyParts(1, TuPack.EncodeKey(x));
Assert.That(items.Item1, Is.EqualTo(x));
Assert.That(items.Item2, Is.EqualTo(default(long)));
Assert.That(items.Item3, Is.EqualTo(default(Guid)));
// should fail if number of items to encode is out of range
Assert.That(() => { encoder.EncodeKeyParts(4, items); }, Throws.InstanceOf <ArgumentOutOfRangeException>());
Assert.That(() => { encoder.EncodeKeyParts(0, items); }, Throws.InstanceOf <ArgumentOutOfRangeException>());
}
public void Test_FdbQueryTransformExpression()
{
var expr = FdbQueryExpressions.Transform(
FdbQueryExpressions.RangeStartsWith(TuPack.EncodeKey("Hello", "World")),
(kvp) => kvp.Value.ToUnicode()
);
Log(expr);
Assert.That(expr, Is.Not.Null);
Assert.That(expr.Source, Is.Not.Null.And.InstanceOf <FdbQueryRangeExpression>());
Assert.That(expr.Transform, Is.Not.Null);
Assert.That(expr.Type, Is.EqualTo(typeof(IAsyncEnumerable <string>)));
Assert.That(expr.ElementType, Is.EqualTo(typeof(string)));
Log(FdbQueryExpressions.ExplainSequence(expr));
}
public void Test_FdbQueryFilterExpression()
{
var expr = FdbQueryExpressions.Filter(
FdbQueryExpressions.RangeStartsWith(TuPack.EncodeKey("Hello", "World")),
(kvp) => kvp.Value.ToInt32() % 2 == 0
);
Log(expr);
Assert.That(expr, Is.Not.Null);
Assert.That(expr.Source, Is.Not.Null.And.InstanceOf <FdbQueryRangeExpression>());
Assert.That(expr.Filter, Is.Not.Null);
Assert.That(expr.Type, Is.EqualTo(typeof(IAsyncEnumerable <KeyValuePair <Slice, Slice> >)));
Assert.That(expr.ElementType, Is.EqualTo(typeof(KeyValuePair <Slice, Slice>)));
Log(FdbQueryExpressions.ExplainSequence(expr));
}
public async Task Test_Can_Open_Database_With_Non_Empty_GlobalSpace()
{
// using a tuple prefix
using (var db = await Fdb.OpenAsync(new FdbConnectionOptions {
GlobalSpace = KeySubspace.FromKey(TuPack.EncodeKey("test"))
}, this.Cancellation))
{
Assert.That(db, Is.Not.Null);
Assert.That(db.GlobalSpace, Is.Not.Null);
Assert.That(db.GlobalSpace.GetPrefix().ToString(), Is.EqualTo("<02>test<00>"));
var subspace = db.Partition.ByKey("hello");
Assert.That(subspace.GetPrefix().ToString(), Is.EqualTo("<02>test<00><02>hello<00>"));
// keys inside the global space are valid
Assert.That(db.IsKeyValid(TuPack.EncodeKey("test", 123)), Is.True);
// keys outside the global space are invalid
Assert.That(db.IsKeyValid(Slice.FromByte(42)), Is.False);
}
// using a random binary prefix
using (var db = await Fdb.OpenAsync(new FdbConnectionOptions {
GlobalSpace = KeySubspace.FromKey(new byte[] { 42, 255, 0, 90 }.AsSlice())
}, this.Cancellation))
{
Assert.That(db, Is.Not.Null);
Assert.That(db.GlobalSpace, Is.Not.Null);
Assert.That(db.GlobalSpace.GetPrefix().ToString(), Is.EqualTo("*<FF><00>Z"));
var subspace = db.Partition.ByKey("hello");
Assert.That(subspace.GetPrefix().ToString(), Is.EqualTo("*<FF><00>Z<02>hello<00>"));
// keys inside the global space are valid
Assert.That(db.IsKeyValid(Slice.Unescape("*<FF><00>Z123")), Is.True);
// keys outside the global space are invalid
Assert.That(db.IsKeyValid(Slice.FromByte(123)), Is.False);
Assert.That(db.IsKeyValid(Slice.Unescape("*<FF>")), Is.False);
}
}
/// <summary>Append a pair encoded keys to the prefix of the current location</summary> /// <typeparam name="T1">Type of the first key</typeparam> /// <typeparam name="T2">Type of the second key</typeparam> /// <param name="item1">Key that will be appended first to the current location's binary prefix</param> /// <param name="item2">Key that will be appended last to the current location's binary prefix</param> /// <returns>A new subspace location with an additional binary suffix</returns> public DynamicKeySubspaceLocation ByKey <T1, T2>(T1 item1, T2 item2) => new DynamicKeySubspaceLocation(GetSafePath(), TuPack.EncodeKey <T1, T2>(item1, item2), TuPack.Encoding.GetDynamicKeyEncoder());
public async Task Test_Range_Except()
{
int K = 3;
int N = 100;
using (var db = await OpenTestPartitionAsync())
{
// get a clean new directory
var location = await GetCleanDirectory(db, "Queries", "Except");
// create K lists
var lists = Enumerable.Range(0, K).Select(i => location.Partition.ByKey(i)).ToArray();
// lists[0] contains all multiples of 1
// lists[1] contains all multiples of 2
// lists[k-1] contains all multiples of K
// more generally: lists[k][i] = (..., Intersect, k, i * (k + 1)) = (k, i)
var series = Enumerable.Range(1, K).Select(k => Enumerable.Range(1, N).Select(x => k * x).ToArray()).ToArray();
//foreach(var serie in series)
//{
// Log(String.Join(", ", serie));
//}
for (int k = 0; k < K; k++)
{
//Log("> k = " + k);
using (var tr = db.BeginTransaction(this.Cancellation))
{
for (int i = 0; i < N; i++)
{
var key = lists[k].Keys.Encode(series[k][i]);
var value = TuPack.EncodeKey(k, i);
//Log("> " + key + " = " + value);
tr.Set(key, value);
}
await tr.CommitAsync();
}
}
// Intersect all lists together should produce all integers that are prime numbers
IEnumerable <int> xs = series[0];
for (int i = 1; i < K; i++)
{
xs = xs.Except(series[i]);
}
var expected = xs.ToArray();
Log("Expected: {0}", String.Join(", ", expected));
using (var tr = db.BeginTransaction(this.Cancellation))
{
var merge = tr.Except(
lists.Select(list => KeySelectorPair.Create(list.Keys.ToRange())),
kvp => location.Keys.DecodeLast <int>(kvp.Key)
);
Assert.That(merge, Is.Not.Null);
Assert.That(merge, Is.InstanceOf <ExceptAsyncIterator <KeyValuePair <Slice, Slice>, int, KeyValuePair <Slice, Slice> > >());
var results = await merge.ToListAsync();
Assert.That(results, Is.Not.Null);
Assert.That(results.Count, Is.EqualTo(expected.Length));
for (int i = 0; i < results.Count; i++)
{
Assert.That(location.Keys.DecodeLast <int>(results[i].Key), Is.EqualTo(expected[i]));
}
}
}
}
public async Task Test_FdbMap_With_Custom_Key_Encoder()
{
// Use a table as a backing store for the rules of a Poor Man's firewall, where each keys are the IPEndPoint (tcp only!), and the values are "pass" or "block"
// Encode IPEndPoint as the (IP, Port,) encoded with the Tuple codec
// note: there is a much simpler way or creating composite keys, this is just a quick and dirty test!
var keyEncoder = new KeyEncoder <IPEndPoint>(
(ipe) => ipe == null ? Slice.Empty : TuPack.EncodeKey(ipe.Address, ipe.Port),
(packed) =>
{
if (packed.IsNullOrEmpty)
{
return(default(IPEndPoint));
}
var t = TuPack.Unpack(packed);
return(new IPEndPoint(t.Get <IPAddress>(0), t.Get <int>(1)));
}
);
var rules = new Dictionary <IPEndPoint, string>()
{
{ new IPEndPoint(IPAddress.Parse("172.16.12.34"), 6667), "block" },
{ new IPEndPoint(IPAddress.Parse("192.168.34.56"), 80), "pass" },
{ new IPEndPoint(IPAddress.Parse("192.168.34.56"), 443), "pass" }
};
using (var db = await OpenTestPartitionAsync())
{
var location = db.Root["Collections"]["Maps"];
await CleanLocation(db, location);
var mapHosts = new FdbMap <IPEndPoint, string>(location.ByKey("Hosts").AsTyped <IPEndPoint>(keyEncoder), BinaryEncoding.StringEncoder);
// import all the rules
await mapHosts.WriteAsync(db, (tr, hosts) =>
{
foreach (var rule in rules)
{
hosts.Set(tr, rule.Key, rule.Value);
}
}, this.Cancellation);
#if FULL_DEBUG
await DumpSubspace(db, location);
#endif
// test the rules
await mapHosts.ReadAsync(db, async (tr, hosts) =>
{
var value = await hosts.GetAsync(tr, new IPEndPoint(IPAddress.Parse("172.16.12.34"), 6667));
Assert.That(value, Is.EqualTo("block"));
value = await hosts.GetAsync(tr, new IPEndPoint(IPAddress.Parse("192.168.34.56"), 443));
Assert.That(value, Is.EqualTo("pass"));
var baz = new IPEndPoint(IPAddress.Parse("172.16.12.34"), 80);
Assert.That(async() => await hosts.GetAsync(tr, baz), Throws.InstanceOf <KeyNotFoundException>());
var opt = await hosts.TryGetAsync(tr, baz);
Assert.That(opt.HasValue, Is.False);
}, this.Cancellation);
}
}
private static async Task MainAsync(CancellationToken ct)
{
// change the path to the native lib if not default
if (NATIVE_PATH != null)
{
Fdb.Options.SetNativeLibPath(NATIVE_PATH);
}
// uncomment this to enable network thread tracing
// FdbCore.TracePath = Path.Combine(Path.GetTempPath(), "fdb");
int apiVersion = Fdb.GetMaxApiVersion();
Console.WriteLine("Max API Version: " + apiVersion);
try
{
Console.WriteLine("Starting network thread...");
Fdb.Start(); // this will select API version 21
Console.WriteLine("> Up and running");
Console.WriteLine("Connecting to local cluster...");
using (var cluster = await Fdb.CreateClusterAsync(CLUSTER_FILE, ct))
{
Console.WriteLine("> Connected!");
Console.WriteLine("Opening database 'DB'...");
using (var db = await cluster.OpenDatabaseAsync(DB_NAME, KeySubspace.FromKey(TuPack.EncodeKey(SUBSPACE)), false, ct))
{
Console.WriteLine("> Connected to db '{0}'", db.Name);
// get coordinators
var cf = await Fdb.System.GetCoordinatorsAsync(db, ct);
Console.WriteLine("Coordinators: " + cf.ToString());
// clear everything
using (var tr = db.BeginTransaction(ct))
{
Console.WriteLine("Clearing subspace " + db.GlobalSpace + " ...");
tr.ClearRange(db.GlobalSpace);
await tr.CommitAsync();
Console.WriteLine("> Database cleared");
}
Console.WriteLine("----------");
await TestSimpleTransactionAsync(db, ct);
Console.WriteLine("----------");
await BenchInsertSmallKeysAsync(db, N, 16, ct); // some guid
await BenchInsertSmallKeysAsync(db, N, 60 * 4, ct); // one Int32 per minutes, over an hour
await BenchInsertSmallKeysAsync(db, N, 512, ct); // small JSON payload
////await BenchInsertSmallKeysAsync(db, N, 4096, ct); // typical small cunk size
////await BenchInsertSmallKeysAsync(db, N / 10, 65536, ct); // typical medium chunk size
//await BenchInsertSmallKeysAsync(db, 1, 100000, ct); // Maximum value size (as of beta 1)
////// insert keys in parrallel
await BenchConcurrentInsert(db, 1, 100, 512, ct);
await BenchConcurrentInsert(db, 1, 1000, 512, ct);
await BenchConcurrentInsert(db, 1, 10000, 512, ct);
await BenchConcurrentInsert(db, 1, N, 16, ct);
await BenchConcurrentInsert(db, 2, N, 16, ct);
await BenchConcurrentInsert(db, 4, N, 16, ct);
await BenchConcurrentInsert(db, 8, N, 16, ct);
await BenchConcurrentInsert(db, 16, N, 16, ct);
//await BenchSerialWriteAsync(db, N, ct);
//await BenchSerialReadAsync(db, N, ct);
//await BenchConcurrentReadAsync(db, N, ct);
//await BenchClearAsync(db, N, ct);
await BenchUpdateSameKeyLotsOfTimesAsync(db, 1000, ct);
await BenchUpdateLotsOfKeysAsync(db, 1000, ct);
await BenchBulkInsertThenBulkReadAsync(db, 100 * 1000, 50, 128, ct);
await BenchBulkInsertThenBulkReadAsync(db, 100 * 1000, 128, 50, ct);
////await BenchBulkInsertThenBulkReadAsync(db, 1 * 1000 * 1000, 50, 128, ct);
await BenchMergeSortAsync(db, 100, 3, 20, ct);
await BenchMergeSortAsync(db, 1000, 10, 100, ct);
await BenchMergeSortAsync(db, 100, 100, 100, ct);
await BenchMergeSortAsync(db, 100, 1000, 100, ct);
Console.WriteLine("time to say goodbye...");
}
}
}
finally
{
Console.WriteLine("### DONE ###");
Fdb.Stop();
}
#if DEBUG
Console.ReadLine();
#endif
}