public async Task Test_Vector_Fast() { using (var db = await OpenTestPartitionAsync()) { var location = await GetCleanDirectory(db, "ranked_set"); var vector = new FdbRankedSet(location); await db.ReadWriteAsync(async (tr) => { await vector.OpenAsync(tr); await PrintRankedSet(vector, tr); }, this.Cancellation); Log(); var rnd = new Random(); var sw = Stopwatch.StartNew(); for (int i = 0; i < 100; i++) { Console.Write("\rInserting " + i); await db.ReadWriteAsync((tr) => vector.InsertAsync(tr, TuPack.EncodeKey(rnd.Next())), this.Cancellation); } sw.Stop(); Log("\rDone in {0:N3} sec", sw.Elapsed.TotalSeconds); await db.ReadAsync((tr) => PrintRankedSet(vector, tr), this.Cancellation); } }
public void Test_Subspace_With_Tuple_Prefix() { var subspace = KeySubspace.CreateDynamic(TuPack.EncodeKey("hello")); Assert.That(subspace.GetPrefix().ToString(), Is.EqualTo("<02>hello<00>")); Assert.That(subspace.Copy(), Is.Not.SameAs(subspace)); Assert.That(subspace.Copy().GetPrefix(), Is.EqualTo(subspace.GetPrefix())); // concat(Slice) should append the slice to the tuple prefix directly Assert.That(subspace[Slice.FromInt32(0x01020304)].ToString(), Is.EqualTo("<02>hello<00><04><03><02><01>")); Assert.That(subspace[Slice.FromStringAscii("world")].ToString(), Is.EqualTo("<02>hello<00>world")); // pack(...) should use tuple serialization Assert.That(subspace.Keys.Encode(123).ToString(), Is.EqualTo("<02>hello<00><15>{")); Assert.That(subspace.Keys.Encode("world").ToString(), Is.EqualTo("<02>hello<00><02>world<00>")); // even though the subspace prefix is a tuple, appending to it will only return the new items var k = subspace.Keys.Pack(("world", 123, false)); Assert.That(k.ToString(), Is.EqualTo("<02>hello<00><02>world<00><15>{<14>")); // if we unpack the key with the binary prefix, we should get a valid tuple var t2 = subspace.Keys.Unpack(k); Assert.That(t2, Is.Not.Null); Assert.That(t2.Count, Is.EqualTo(3)); Assert.That(t2.Get <string>(0), Is.EqualTo("world")); Assert.That(t2.Get <int>(1), Is.EqualTo(123)); Assert.That(t2.Get <bool>(2), Is.False); }
public void Test_Simple_SelfTerms_Codecs() { // encodes a key using 3 parts: (x, y, z) => ordered_key string x = "abc"; long y = 123; Guid z = Guid.NewGuid(); var first = TupleCodec <string> .Default; var second = TupleCodec <long> .Default; var third = TupleCodec <Guid> .Default; var writer = default(SliceWriter); first.EncodeOrderedSelfTerm(ref writer, x); second.EncodeOrderedSelfTerm(ref writer, y); third.EncodeOrderedSelfTerm(ref writer, z); var data = writer.ToSlice(); Assert.That(data, Is.EqualTo(TuPack.EncodeKey(x, y, z))); var reader = new SliceReader(data); Assert.That(first.DecodeOrderedSelfTerm(ref reader), Is.EqualTo(x)); Assert.That(second.DecodeOrderedSelfTerm(ref reader), Is.EqualTo(y)); Assert.That(third.DecodeOrderedSelfTerm(ref reader), Is.EqualTo(z)); Assert.That(reader.HasMore, Is.False); }
public void Test_KeyRange_Test() { const int BEFORE = -1, INSIDE = 0, AFTER = +1; KeyRange range; // range: [ "A", "Z" ) range = KeyRange.Create(Slice.FromByteString("A"), Slice.FromByteString("Z")); // Excluding the end: < "Z" Assert.That(range.Test(Slice.FromByteString("\x00"), endIncluded: false), Is.EqualTo(BEFORE)); Assert.That(range.Test(Slice.FromByteString("@"), endIncluded: false), Is.EqualTo(BEFORE)); Assert.That(range.Test(Slice.FromByteString("A"), endIncluded: false), Is.EqualTo(INSIDE)); Assert.That(range.Test(Slice.FromByteString("Z"), endIncluded: false), Is.EqualTo(AFTER)); Assert.That(range.Test(Slice.FromByteString("Z\x00"), endIncluded: false), Is.EqualTo(AFTER)); Assert.That(range.Test(Slice.FromByteString("\xFF"), endIncluded: false), Is.EqualTo(AFTER)); // Including the end: <= "Z" Assert.That(range.Test(Slice.FromByteString("\x00"), endIncluded: true), Is.EqualTo(BEFORE)); Assert.That(range.Test(Slice.FromByteString("@"), endIncluded: true), Is.EqualTo(BEFORE)); Assert.That(range.Test(Slice.FromByteString("A"), endIncluded: true), Is.EqualTo(INSIDE)); Assert.That(range.Test(Slice.FromByteString("Z"), endIncluded: true), Is.EqualTo(INSIDE)); Assert.That(range.Test(Slice.FromByteString("Z\x00"), endIncluded: true), Is.EqualTo(AFTER)); Assert.That(range.Test(Slice.FromByteString("\xFF"), endIncluded: true), Is.EqualTo(AFTER)); range = KeyRange.Create(TuPack.EncodeKey("A"), TuPack.EncodeKey("Z")); Assert.That(range.Test(TuPack.EncodeKey("@")), Is.EqualTo((BEFORE))); Assert.That(range.Test(TuPack.EncodeKey("A")), Is.EqualTo((INSIDE))); Assert.That(range.Test(TuPack.EncodeKey("Z")), Is.EqualTo((AFTER))); Assert.That(range.Test(TuPack.EncodeKey("Z"), endIncluded: true), Is.EqualTo(INSIDE)); }
public async Task Test_Vector_Fast() { using (var db = await OpenTestPartitionAsync()) { var location = db.Root["ranked_set"]; await CleanLocation(db, location); var rankedSet = new FdbRankedSet(location); await db.WriteAsync(tr => rankedSet.OpenAsync(tr), this.Cancellation); Log(await rankedSet.ReadAsync(db, (tr, state) => PrintRankedSet(state, tr), this.Cancellation)); Log(); var rnd = new Random(); var sw = Stopwatch.StartNew(); for (int i = 0; i < 100; i++) { //Log("Inserting " + i); await db.WriteAsync(async tr => { var state = await rankedSet.Resolve(tr); await state.InsertAsync(tr, TuPack.EncodeKey(rnd.Next())); }, this.Cancellation); } sw.Stop(); Log($"Done in {sw.Elapsed.TotalSeconds:N3} sec"); #if FULL_DEBUG await DumpSubspace(db, location); #endif Log(await rankedSet.ReadAsync(db, (tr, state) => PrintRankedSet(state, tr), this.Cancellation)); } }
/// <summary>Return one or more fields of an hashset</summary> /// <param name="trans">Transaction that will be used for this request</param> /// <param name="id">Unique identifier of the hashset</param> /// <param name="fields">List of the fields to read</param> /// <returns>Dictionary containing the values of the selected fields, or <see cref="Slice.Empty"/> if that particular field does not exist.</returns> public async Task <IDictionary <string, Slice> > GetAsync(IFdbReadOnlyTransaction trans, IVarTuple id, params string[] fields) { if (trans == null) { throw new ArgumentNullException(nameof(trans)); } if (id == null) { throw new ArgumentNullException(nameof(id)); } if (fields == null) { throw new ArgumentNullException(nameof(fields)); } var keys = TuPack.EncodePrefixedKeys(GetKey(id), fields); var values = await trans.GetValuesAsync(keys).ConfigureAwait(false); Contract.Debug.Assert(values != null && values.Length == fields.Length); var results = new Dictionary <string, Slice>(values.Length, StringComparer.OrdinalIgnoreCase); for (int i = 0; i < fields.Length; i++) { results[fields[i]] = values[i]; } return(results); }
/// <summary> /// Simulate a student that is really indecisive /// </summary> public async Task RunProducer(IFdbDatabase db, CancellationToken ct) { int cnt = 0; var rnd = new Random(); this.TimeLine.Start(); while (!ct.IsCancellationRequested) { int k = cnt++; Slice taskId = TuPack.EncodeKey(this.Id.GetHashCode(), k); string msg = "Message #" + k + " from producer " + this.Id + " (" + DateTime.UtcNow.ToString("O") + ")"; var latency = Stopwatch.StartNew(); await this.WorkerPool.ScheduleTaskAsync(db, taskId, Slice.FromString(msg), ct).ConfigureAwait(false); latency.Stop(); Console.Write(k.ToString("N0") + "\r"); this.TimeLine.Add(latency.Elapsed.TotalMilliseconds); TimeSpan delay = TimeSpan.FromTicks(rnd.Next((int)this.DelayMin.Ticks, (int)this.DelayMax.Ticks)); await Task.Delay(delay, ct).ConfigureAwait(false); } this.TimeLine.Stop(); ct.ThrowIfCancellationRequested(); }
public async Task Test_Can_MergeSort() { int K = 3; int N = 100; using (var db = await OpenTestPartitionAsync()) { var location = await GetCleanDirectory(db, "Queries", "MergeSort"); // clear! await db.ClearRangeAsync(location, this.Cancellation); // create K lists var lists = Enumerable.Range(0, K).Select(i => location.Partition.ByKey(i)).ToArray(); // lists[0] contains all multiples of K ([0, 0], [K, 1], [2K, 2], ...) // lists[1] contains all multiples of K, offset by 1 ([1, 0], [K+1, 1], [2K+1, 2], ...) // lists[k-1] contains all multiples of K, offset by k-1 ([K-1, 0], [2K-1, 1], [3K-1, 2], ...) // more generally: lists[k][i] = (..., MergeSort, k, (i * K) + k) = (k, i) for (int k = 0; k < K; k++) { using (var tr = db.BeginTransaction(this.Cancellation)) { for (int i = 0; i < N; i++) { tr.Set(lists[k].Keys.Encode((i * K) + k), TuPack.EncodeKey(k, i)); } await tr.CommitAsync(); } } // MergeSorting all lists together should produce all integers from 0 to (K*N)-1, in order // we use the last part of the key for sorting using (var tr = db.BeginTransaction(this.Cancellation)) { var merge = tr.MergeSort( lists.Select(list => KeySelectorPair.Create(list.Keys.ToRange())), kvp => location.Keys.DecodeLast <int>(kvp.Key) ); Assert.That(merge, Is.Not.Null); Assert.That(merge, Is.InstanceOf <MergeSortAsyncIterator <KeyValuePair <Slice, Slice>, int, KeyValuePair <Slice, Slice> > >()); var results = await merge.ToListAsync(); Assert.That(results, Is.Not.Null); Assert.That(results.Count, Is.EqualTo(K * N)); for (int i = 0; i < K * N; i++) { Assert.That(location.ExtractKey(results[i].Key), Is.EqualTo(TuPack.EncodeKey(i % K, i))); Assert.That(results[i].Value, Is.EqualTo(TuPack.EncodeKey(i % K, i / K))); } } } }
private static void DumpIndex <TKey, TVal>(string label, MemoryIndex <TKey> index, Func <TKey, int, TVal> orderBy, IComparer <TVal> comparer = null, bool heatMaps = false) { comparer = comparer ?? Comparer <TVal> .Default; int total = index.Statistics.Values.Sum(); long totalLegacy = 0; int[] map = new int[100]; double r = (double)(map.Length - 1) / total; Log("__{0}__", label); Log("| Indexed Value | Count | Total % | Words | Lit% | 1-Bits | Word% | Bitmap | ratio % | Legacy | ratio % |" + (heatMaps ? " HeatMap |" : "")); Log("|:------------------------|-------:|--------:|------:|-------:|-------:|-------:|---------:|--------:|----------:|--------:|" + (heatMaps ? ":-----------------------------------------------------------------------|" : "")); foreach (var kv in index.Values.OrderBy((kv) => orderBy(kv.Key, index.Count(kv.Key)), comparer)) { var t = STuple.Create(kv.Key); var tk = TuPack.Pack(t); kv.Value.GetStatistics(out int bits, out int words, out int literals, out int _, out double ratio); long legacyIndexSize = 0; // size estimate of a regular FDB index (..., "Value", GUID) = "" Array.Clear(map, 0, map.Length); foreach (var p in kv.Value.GetView()) { map[(int)(r * p)]++; legacyIndexSize += 3 + tk.Count + 17; } totalLegacy += legacyIndexSize; int bytes = kv.Value.ToSlice().Count; Log(string.Format( CultureInfo.InvariantCulture, "| {0,-24}| {1,6:N0} | {2,6:N2}% | {3,5:N0} | {4,5:N1}% | {5,6:N0} | {6,6:N2} | {7,8:N0} | {8,6:N2}% | {9,9:N0} | {10,6:N2}% |" + (heatMaps ? " `{11}` |" : ""), /*0*/ t, /*1*/ index.Count(kv.Key), /*2*/ 100.0 * index.Frequency(kv.Key), /*3*/ words, /*4*/ (100.0 * literals) / words, /*5*/ bits, /*6*/ 1.0 * bits / words, /*7*/ bytes, /*8*/ 100.0 * ratio, /*9*/ legacyIndexSize, /*A*/ (100.0 * bytes) / legacyIndexSize, /*B*/ heatMaps ? MakeHeatMap(map) : "" )); } Log(string.Format( CultureInfo.InvariantCulture, "> {0:N0} distinct value(s), {1:N0} document(s), {2:N0} bitmap bytes, {3:N0} legacy bytes", index.Values.Count, total, index.Values.Values.Sum(x => x.ToSlice().Count), totalLegacy )); }
public void Test_FdbKey_PrettyPrint() { // verify that the pretty printing of keys produce a user friendly output Assert.That(FdbKey.Dump(Slice.Nil), Is.EqualTo("<null>")); Assert.That(FdbKey.Dump(Slice.Empty), Is.EqualTo("<empty>")); Assert.That(FdbKey.Dump(Slice.FromByte(0)), Is.EqualTo("<00>")); Assert.That(FdbKey.Dump(Slice.FromByte(255)), Is.EqualTo("<FF>")); Assert.That(FdbKey.Dump(new byte[] { 0, 1, 2, 3, 4, 5, 6, 7 }.AsSlice()), Is.EqualTo("<00><01><02><03><04><05><06><07>")); Assert.That(FdbKey.Dump(new byte[] { 255, 254, 253, 252, 251, 250, 249, 248 }.AsSlice()), Is.EqualTo("<FF><FE><FD><FC><FB><FA><F9><F8>")); Assert.That(FdbKey.Dump(Slice.FromString("hello")), Is.EqualTo("hello")); Assert.That(FdbKey.Dump(Slice.FromString("héllø")), Is.EqualTo("h<C3><A9>ll<C3><B8>")); // tuples should be decoded properly Assert.That(FdbKey.Dump(TuPack.EncodeKey(123)), Is.EqualTo("(123,)"), "Singleton tuples should end with a ','"); Assert.That(FdbKey.Dump(TuPack.EncodeKey(Slice.FromByteString("hello"))), Is.EqualTo("(`hello`,)"), "ASCII strings should use single back quotes"); Assert.That(FdbKey.Dump(TuPack.EncodeKey("héllø")), Is.EqualTo("(\"héllø\",)"), "Unicode strings should use double quotes"); Assert.That(FdbKey.Dump(TuPack.EncodeKey(new byte[] { 1, 2, 3 }.AsSlice())), Is.EqualTo("(`<01><02><03>`,)")); Assert.That(FdbKey.Dump(TuPack.EncodeKey(123, 456)), Is.EqualTo("(123, 456)"), "Elements should be separated with a space, and not end up with ','"); Assert.That(FdbKey.Dump(TuPack.EncodeKey(default(object), true, false)), Is.EqualTo("(null, true, false)"), "Booleans should be displayed as numbers, and null should be in lowercase"); //note: even though it's tempting to using Python's "Nil", it's not very ".NETty" //note: the string representation of double is not identical between NetFx and .NET Core! So we cannot used a constant literal here Assert.That(FdbKey.Dump(TuPack.EncodeKey(1.0d, Math.PI, Math.E)), Is.EqualTo("(1, " + Math.PI.ToString("R", CultureInfo.InvariantCulture) + ", " + Math.E.ToString("R", CultureInfo.InvariantCulture) + ")"), "Doubles should used dot and have full precision (17 digits)"); Assert.That(FdbKey.Dump(TuPack.EncodeKey(1.0f, (float)Math.PI, (float)Math.E)), Is.EqualTo("(1, " + ((float)Math.PI).ToString("R", CultureInfo.InvariantCulture) + ", " + ((float)Math.E).ToString("R", CultureInfo.InvariantCulture) + ")"), "Singles should used dot and have full precision (10 digits)"); var guid = Guid.NewGuid(); Assert.That(FdbKey.Dump(TuPack.EncodeKey(guid)), Is.EqualTo($"({guid:B},)"), "GUIDs should be displayed as a string literal, surrounded by {{...}}, and without quotes"); var uuid128 = Uuid128.NewUuid(); Assert.That(FdbKey.Dump(TuPack.EncodeKey(uuid128)), Is.EqualTo($"({uuid128:B},)"), "Uuid128s should be displayed as a string literal, surrounded by {{...}}, and without quotes"); var uuid64 = Uuid64.NewUuid(); Assert.That(FdbKey.Dump(TuPack.EncodeKey(uuid64)), Is.EqualTo($"({uuid64:B},)"), "Uuid64s should be displayed as a string literal, surrounded by {{...}}, and without quotes"); // ranges should be decoded when possible var key = TuPack.ToRange(STuple.Create("hello")); // "<02>hello<00><00>" .. "<02>hello<00><FF>" Assert.That(FdbKey.PrettyPrint(key.Begin, FdbKey.PrettyPrintMode.Begin), Is.EqualTo("(\"hello\",).<00>")); Assert.That(FdbKey.PrettyPrint(key.End, FdbKey.PrettyPrintMode.End), Is.EqualTo("(\"hello\",).<FF>")); key = KeyRange.StartsWith(TuPack.EncodeKey("hello")); // "<02>hello<00>" .. "<02>hello<01>" Assert.That(FdbKey.PrettyPrint(key.Begin, FdbKey.PrettyPrintMode.Begin), Is.EqualTo("(\"hello\",)")); Assert.That(FdbKey.PrettyPrint(key.End, FdbKey.PrettyPrintMode.End), Is.EqualTo("(\"hello\",) + 1")); var t = TuPack.EncodeKey(123); Assert.That(FdbKey.PrettyPrint(t, FdbKey.PrettyPrintMode.Single), Is.EqualTo("(123,)")); Assert.That(FdbKey.PrettyPrint(TuPack.ToRange(t).Begin, FdbKey.PrettyPrintMode.Begin), Is.EqualTo("(123,).<00>")); Assert.That(FdbKey.PrettyPrint(TuPack.ToRange(t).End, FdbKey.PrettyPrintMode.End), Is.EqualTo("(123,).<FF>")); }
private static async Task HelloWorld(CancellationToken ct) { // Connect to the "DB" database on the local cluster using (var db = await Fdb.OpenAsync()) { // Writes some data in to the database using (var tr = db.BeginTransaction(ct)) { tr.Set(TuPack.EncodeKey("Test", 123), Slice.FromString("Hello World!")); tr.Set(TuPack.EncodeKey("Test", 456), Slice.FromInt64(DateTime.UtcNow.Ticks)); } } }
public void Test_Simple_String_Codec() { var codec = TupleCodec <string> .Default; Assert.That(codec, Is.Not.Null); Assert.That(codec.EncodeOrdered("héllø Wörld"), Is.EqualTo(TuPack.EncodeKey("héllø Wörld"))); Assert.That(codec.EncodeOrdered(String.Empty), Is.EqualTo(TuPack.EncodeKey(""))); Assert.That(codec.EncodeOrdered(null), Is.EqualTo(TuPack.EncodeKey(default(string)))); Assert.That(codec.DecodeOrdered(TuPack.EncodeKey("héllø Wörld")), Is.EqualTo("héllø Wörld")); Assert.That(codec.DecodeOrdered(TuPack.EncodeKey(String.Empty)), Is.EqualTo("")); Assert.That(codec.DecodeOrdered(TuPack.EncodeKey(default(string))), Is.Null); }
public void Test_Simple_Integer_Codec() { var codec = TupleCodec <int> .Default; Assert.That(codec, Is.Not.Null); Assert.That(codec.EncodeOrdered(0), Is.EqualTo(TuPack.EncodeKey(0))); Assert.That(codec.EncodeOrdered(123), Is.EqualTo(TuPack.EncodeKey(123))); Assert.That(codec.EncodeOrdered(123456), Is.EqualTo(TuPack.EncodeKey(123456))); Assert.That(codec.DecodeOrdered(TuPack.EncodeKey(0)), Is.EqualTo(0)); Assert.That(codec.DecodeOrdered(TuPack.EncodeKey(123)), Is.EqualTo(123)); Assert.That(codec.DecodeOrdered(TuPack.EncodeKey(123456)), Is.EqualTo(123456)); }
private static Slice MakeKey(IKeySubspace folder, object key) { if (key is IVarTuple t) { return(folder[TuPack.Pack(t)]); } else if (key is string s) { return(folder[Slice.FromStringUtf8(s)]); } else { throw new FormatException("Unsupported key type: " + key); } }
public void Test_FdbQueryRangeExpression() { var expr = FdbQueryExpressions.Range( KeySelectorPair.Create(TuPack.ToKeyRange("Foo")) ); Log(expr); Assert.That(expr, Is.Not.Null); Assert.That(expr.Range.Begin.Key.ToString(), Is.EqualTo("<02>Foo<00><00>")); Assert.That(expr.Range.End.Key.ToString(), Is.EqualTo("<02>Foo<00><FF>")); Assert.That(expr.Type, Is.EqualTo(typeof(IAsyncEnumerable <KeyValuePair <Slice, Slice> >))); Assert.That(expr.ElementType, Is.EqualTo(typeof(KeyValuePair <Slice, Slice>))); Log(FdbQueryExpressions.ExplainSequence(expr)); }
public KeyValuePair <IVarTuple, Slice>[] Split(List <KeyValuePair <string, IVarTuple> > document) { if (document == null) { throw new ArgumentNullException(nameof(document)); } return(document // don't include the id .Where(kvp => !m_keyComparer.Equals(kvp.Key, this.IdName)) // convert into tuples .Select(kvp => new KeyValuePair <IVarTuple, Slice>( STuple.Create(kvp.Key), TuPack.Pack(kvp.Value) )) .ToArray()); }
public void Test_Tuple_Composite_Encoder() { // encodes a key using 3 parts: (x, y, z) => ordered_key string x = "abc"; long y = 123; Guid z = Guid.NewGuid(); var encoder = TuPack.Encoding.GetKeyEncoder <string, long, Guid>(); Assert.That(encoder, Is.Not.Null); // full key encoding // note: EncodeKey(...) is just a shortcurt for packing all items in a tuple, and EncodeComposite(..., count = 3) var data = encoder.EncodeKey(x, y, z); Assert.That(data, Is.EqualTo(TuPack.EncodeKey(x, y, z))); var items = encoder.DecodeKey(data); Assert.That(items.Item1, Is.EqualTo(x)); Assert.That(items.Item2, Is.EqualTo(y)); Assert.That(items.Item3, Is.EqualTo(z)); // partial key encoding data = encoder.EncodeKeyParts(2, items); Assert.That(data, Is.EqualTo(TuPack.EncodeKey(x, y))); items = encoder.DecodeKeyParts(2, TuPack.EncodeKey(x, y)); Assert.That(items.Item1, Is.EqualTo(x)); Assert.That(items.Item2, Is.EqualTo(y)); Assert.That(items.Item3, Is.EqualTo(default(Guid))); data = encoder.EncodeKeyParts(1, items); Assert.That(data, Is.EqualTo(TuPack.EncodeKey(x))); items = encoder.DecodeKeyParts(1, TuPack.EncodeKey(x)); Assert.That(items.Item1, Is.EqualTo(x)); Assert.That(items.Item2, Is.EqualTo(default(long))); Assert.That(items.Item3, Is.EqualTo(default(Guid))); // should fail if number of items to encode is out of range Assert.That(() => { encoder.EncodeKeyParts(4, items); }, Throws.InstanceOf <ArgumentOutOfRangeException>()); Assert.That(() => { encoder.EncodeKeyParts(0, items); }, Throws.InstanceOf <ArgumentOutOfRangeException>()); }
public List <KeyValuePair <string, IVarTuple> > Build(KeyValuePair <IVarTuple, Slice>[] parts) { if (parts == null) { throw new ArgumentNullException(nameof(parts)); } var list = new List <KeyValuePair <string, IVarTuple> >(parts.Length); foreach (var part in parts) { list.Add(new KeyValuePair <string, IVarTuple>( part.Key.Last <string>(), TuPack.Unpack(part.Value) )); } return(list); }
public void Test_FdbQueryTransformExpression() { var expr = FdbQueryExpressions.Transform( FdbQueryExpressions.RangeStartsWith(TuPack.EncodeKey("Hello", "World")), (kvp) => kvp.Value.ToUnicode() ); Log(expr); Assert.That(expr, Is.Not.Null); Assert.That(expr.Source, Is.Not.Null.And.InstanceOf <FdbQueryRangeExpression>()); Assert.That(expr.Transform, Is.Not.Null); Assert.That(expr.Type, Is.EqualTo(typeof(IAsyncEnumerable <string>))); Assert.That(expr.ElementType, Is.EqualTo(typeof(string))); Log(FdbQueryExpressions.ExplainSequence(expr)); }
public void Test_FdbQueryFilterExpression() { var expr = FdbQueryExpressions.Filter( FdbQueryExpressions.RangeStartsWith(TuPack.EncodeKey("Hello", "World")), (kvp) => kvp.Value.ToInt32() % 2 == 0 ); Log(expr); Assert.That(expr, Is.Not.Null); Assert.That(expr.Source, Is.Not.Null.And.InstanceOf <FdbQueryRangeExpression>()); Assert.That(expr.Filter, Is.Not.Null); Assert.That(expr.Type, Is.EqualTo(typeof(IAsyncEnumerable <KeyValuePair <Slice, Slice> >))); Assert.That(expr.ElementType, Is.EqualTo(typeof(KeyValuePair <Slice, Slice>))); Log(FdbQueryExpressions.ExplainSequence(expr)); }
public async Task Test_Can_Open_Database_With_Non_Empty_GlobalSpace() { // using a tuple prefix using (var db = await Fdb.OpenAsync(new FdbConnectionOptions { GlobalSpace = KeySubspace.FromKey(TuPack.EncodeKey("test")) }, this.Cancellation)) { Assert.That(db, Is.Not.Null); Assert.That(db.GlobalSpace, Is.Not.Null); Assert.That(db.GlobalSpace.GetPrefix().ToString(), Is.EqualTo("<02>test<00>")); var subspace = db.Partition.ByKey("hello"); Assert.That(subspace.GetPrefix().ToString(), Is.EqualTo("<02>test<00><02>hello<00>")); // keys inside the global space are valid Assert.That(db.IsKeyValid(TuPack.EncodeKey("test", 123)), Is.True); // keys outside the global space are invalid Assert.That(db.IsKeyValid(Slice.FromByte(42)), Is.False); } // using a random binary prefix using (var db = await Fdb.OpenAsync(new FdbConnectionOptions { GlobalSpace = KeySubspace.FromKey(new byte[] { 42, 255, 0, 90 }.AsSlice()) }, this.Cancellation)) { Assert.That(db, Is.Not.Null); Assert.That(db.GlobalSpace, Is.Not.Null); Assert.That(db.GlobalSpace.GetPrefix().ToString(), Is.EqualTo("*<FF><00>Z")); var subspace = db.Partition.ByKey("hello"); Assert.That(subspace.GetPrefix().ToString(), Is.EqualTo("*<FF><00>Z<02>hello<00>")); // keys inside the global space are valid Assert.That(db.IsKeyValid(Slice.Unescape("*<FF><00>Z123")), Is.True); // keys outside the global space are invalid Assert.That(db.IsKeyValid(Slice.FromByte(123)), Is.False); Assert.That(db.IsKeyValid(Slice.Unescape("*<FF>")), Is.False); } }
/// <summary>Return the list the names of all fields of an hashset</summary> /// <param name="trans">Transaction that will be used for this request</param> /// <param name="id">Unique identifier of the hashset</param> /// <returns>List of all fields. If the list is empty, the hashset does not exist</returns> public Task <List <string> > GetKeys(IFdbReadOnlyTransaction trans, IVarTuple id) { //note: As of Beta2, FDB does not have a fdb_get_range that only return the keys. That means that we will have to also read the values from the db, in order to just get the names of the fields :( //TODO: find a way to optimize this ? if (trans == null) { throw new ArgumentNullException(nameof(trans)); } if (id == null) { throw new ArgumentNullException(nameof(id)); } var prefix = GetKey(id); return(trans .GetRange(KeyRange.StartsWith(prefix)) .Select((kvp) => ParseFieldKey(TuPack.Unpack(kvp.Key))) .ToListAsync()); }
public static async Task DumpSubspace(IFdbReadOnlyTransaction tr, IKeySubspace subspace) { Assert.That(tr, Is.Not.Null); Assert.That(subspace, Is.Not.Null); FdbTest.Log($"Dumping content of {subspace} at {subspace.GetPrefix():K}:"); int count = 0; await tr .GetRange(KeyRange.StartsWith(subspace.GetPrefix())) .ForEachAsync((kvp) => { var key = subspace.ExtractKey(kvp.Key, boundCheck: true); ++count; string keyDump; try { // attempts decoding it as a tuple keyDump = TuPack.Unpack(key).ToString() !; } catch (Exception) { // not a tuple, dump as bytes keyDump = "'" + key.ToString() + "'"; } FdbTest.Log("- " + keyDump + " = " + kvp.Value.ToString()); }); if (count == 0) { FdbTest.Log("> empty !"); } else { FdbTest.Log("> Found " + count + " values"); } }
public T DecodeKeyLast <T>(Slice packed) { return(TuPack.DecodeLast <T>(packed)); }
public static async Task Get(string[] path, IVarTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct) { if (path == null || path.Length == 0) { Program.Error(log, "Cannot read keys of the Root Partition."); return; } if (extras.Count == 0) { Program.Error(log, "You must specify a key of range of keys!"); return; } var folder = await db.Directory.TryOpenAsync(db, path, ct : ct); if (folder == null) { Program.Error(log, "The directory does not exist anymore"); return; } if (folder.Layer == FdbDirectoryPartition.LayerId) { Program.Error(log, "Cannot clear the content of a Directory Partition!"); return; } object key = extras[0]; Slice k = MakeKey(folder, key); Program.Comment(log, "# Reading key: " + k.ToString("K")); Slice v = await db.ReadWriteAsync(tr => tr.GetAsync(k), ct); if (v.IsNull) { Program.StdOut(log, "# Key does not exist in the database.", ConsoleColor.Red); return; } if (v.IsEmpty) { Program.StdOut(log, "# Key exists but is empty.", ConsoleColor.Gray); return; } Program.StdOut(log, $"# Size: {v.Count:N0}", ConsoleColor.Gray); string format = extras.Count > 1 ? extras.Get <string>(1) : null; switch (format) { case "--text": case "--json": case "--utf8": { Program.StdOut(log, v.ToStringUtf8(), ConsoleColor.Gray); break; } case "--hex": case "--hexa": { Program.StdOut(log, v.ToHexaString(), ConsoleColor.White); break; } case "--dump": { var sb = new StringBuilder(v.Count * 3 + (v.Count >> 4) * 2 + 16); for (int i = 0; i < v.Count; i += 16) { sb.AppendLine(v.Substring(i, 16).ToHexaString(' ')); } Program.StdOut(log, sb.ToString(), ConsoleColor.White); break; } case "--int": { if (v.Count <= 8) { long he = v.ToInt64BE(); long le = v.ToInt64(); Program.StdOut(log, $"BE: {he:X016} ({he:N0})", ConsoleColor.White); Program.StdOut(log, $"LE: {le:X016} ({le:N0})", ConsoleColor.White); } else { Program.StdOut(log, $"Value is too large ({v.Count} bytes)", ConsoleColor.DarkRed); Program.StdOut(log, v.ToHexaString(' '), ConsoleColor.Gray); } break; } case "--uint": { if (v.Count <= 8) { ulong he = v.ToUInt64BE(); ulong le = v.ToUInt64(); Program.StdOut(log, $"BE: {he:X016} ({he:N0})", ConsoleColor.White); Program.StdOut(log, $"LE: {le:X016} ({le:N0})", ConsoleColor.White); } else { Program.StdOut(log, $"Value is too large ({v.Count} bytes)", ConsoleColor.DarkRed); Program.StdOut(log, v.ToHexaString(' '), ConsoleColor.Gray); } break; } case "--tuple": { try { var t = TuPack.Unpack(v); Program.StdOut(log, t.ToString(), ConsoleColor.Gray); } catch (Exception e) { Program.Error(log, "Key value does not seem to be a valid Tuple: " + e.Message); Program.StdOut(log, v.ToHexaString(' '), ConsoleColor.Gray); } break; } default: { Program.StdOut(log, v.ToString("V"), ConsoleColor.White); break; } } }
public static FdbQueryRangeExpression RangeStartsWith(IVarTuple tuple, FdbRangeOptions options = null) { return(RangeStartsWith(TuPack.Pack(tuple), options)); }
public override T DecodeOrdered(Slice input) { return(TuPack.DecodeKey <T>(input)); }
public async Task Test_FdbMap_With_Custom_Key_Encoder() { // Use a table as a backing store for the rules of a Poor Man's firewall, where each keys are the IPEndPoint (tcp only!), and the values are "pass" or "block" // Encode IPEndPoint as the (IP, Port,) encoded with the Tuple codec // note: there is a much simpler way or creating composite keys, this is just a quick and dirty test! var keyEncoder = new KeyEncoder <IPEndPoint>( (ipe) => ipe == null ? Slice.Empty : TuPack.EncodeKey(ipe.Address, ipe.Port), (packed) => { if (packed.IsNullOrEmpty) { return(default(IPEndPoint)); } var t = TuPack.Unpack(packed); return(new IPEndPoint(t.Get <IPAddress>(0), t.Get <int>(1))); } ); var rules = new Dictionary <IPEndPoint, string>() { { new IPEndPoint(IPAddress.Parse("172.16.12.34"), 6667), "block" }, { new IPEndPoint(IPAddress.Parse("192.168.34.56"), 80), "pass" }, { new IPEndPoint(IPAddress.Parse("192.168.34.56"), 443), "pass" } }; using (var db = await OpenTestPartitionAsync()) { var location = db.Root["Collections"]["Maps"]; await CleanLocation(db, location); var mapHosts = new FdbMap <IPEndPoint, string>(location.ByKey("Hosts").AsTyped <IPEndPoint>(keyEncoder), BinaryEncoding.StringEncoder); // import all the rules await mapHosts.WriteAsync(db, (tr, hosts) => { foreach (var rule in rules) { hosts.Set(tr, rule.Key, rule.Value); } }, this.Cancellation); #if FULL_DEBUG await DumpSubspace(db, location); #endif // test the rules await mapHosts.ReadAsync(db, async (tr, hosts) => { var value = await hosts.GetAsync(tr, new IPEndPoint(IPAddress.Parse("172.16.12.34"), 6667)); Assert.That(value, Is.EqualTo("block")); value = await hosts.GetAsync(tr, new IPEndPoint(IPAddress.Parse("192.168.34.56"), 443)); Assert.That(value, Is.EqualTo("pass")); var baz = new IPEndPoint(IPAddress.Parse("172.16.12.34"), 80); Assert.That(async() => await hosts.GetAsync(tr, baz), Throws.InstanceOf <KeyNotFoundException>()); var opt = await hosts.TryGetAsync(tr, baz); Assert.That(opt.HasValue, Is.False); }, this.Cancellation); } }
public IVarTuple UnpackKey(Slice packed) { return(TuPack.Unpack(packed)); }
public T DecodeKeyFirst <T>(Slice packed) { return(TuPack.DecodeFirst <T>(packed)); }