public async Task Test_Vector_Fast()
{
using (var db = await OpenTestPartitionAsync())
{
var location = await GetCleanDirectory(db, "ranked_set");
var vector = new FdbRankedSet(location);
await db.ReadWriteAsync(async (tr) =>
{
await vector.OpenAsync(tr);
await PrintRankedSet(vector, tr);
}, this.Cancellation);
Log();
var rnd = new Random();
var sw = Stopwatch.StartNew();
for (int i = 0; i < 100; i++)
{
Console.Write("\rInserting " + i);
await db.ReadWriteAsync((tr) => vector.InsertAsync(tr, TuPack.EncodeKey(rnd.Next())), this.Cancellation);
}
sw.Stop();
Log("\rDone in {0:N3} sec", sw.Elapsed.TotalSeconds);
await db.ReadAsync((tr) => PrintRankedSet(vector, tr), this.Cancellation);
}
}
public void Test_Subspace_With_Tuple_Prefix()
{
var subspace = KeySubspace.CreateDynamic(TuPack.EncodeKey("hello"));
Assert.That(subspace.GetPrefix().ToString(), Is.EqualTo("<02>hello<00>"));
Assert.That(subspace.Copy(), Is.Not.SameAs(subspace));
Assert.That(subspace.Copy().GetPrefix(), Is.EqualTo(subspace.GetPrefix()));
// concat(Slice) should append the slice to the tuple prefix directly
Assert.That(subspace[Slice.FromInt32(0x01020304)].ToString(), Is.EqualTo("<02>hello<00><04><03><02><01>"));
Assert.That(subspace[Slice.FromStringAscii("world")].ToString(), Is.EqualTo("<02>hello<00>world"));
// pack(...) should use tuple serialization
Assert.That(subspace.Keys.Encode(123).ToString(), Is.EqualTo("<02>hello<00><15>{"));
Assert.That(subspace.Keys.Encode("world").ToString(), Is.EqualTo("<02>hello<00><02>world<00>"));
// even though the subspace prefix is a tuple, appending to it will only return the new items
var k = subspace.Keys.Pack(("world", 123, false));
Assert.That(k.ToString(), Is.EqualTo("<02>hello<00><02>world<00><15>{<14>"));
// if we unpack the key with the binary prefix, we should get a valid tuple
var t2 = subspace.Keys.Unpack(k);
Assert.That(t2, Is.Not.Null);
Assert.That(t2.Count, Is.EqualTo(3));
Assert.That(t2.Get <string>(0), Is.EqualTo("world"));
Assert.That(t2.Get <int>(1), Is.EqualTo(123));
Assert.That(t2.Get <bool>(2), Is.False);
}
public void Test_Simple_SelfTerms_Codecs()
{
// encodes a key using 3 parts: (x, y, z) => ordered_key
string x = "abc";
long y = 123;
Guid z = Guid.NewGuid();
var first = TupleCodec <string> .Default;
var second = TupleCodec <long> .Default;
var third = TupleCodec <Guid> .Default;
var writer = default(SliceWriter);
first.EncodeOrderedSelfTerm(ref writer, x);
second.EncodeOrderedSelfTerm(ref writer, y);
third.EncodeOrderedSelfTerm(ref writer, z);
var data = writer.ToSlice();
Assert.That(data, Is.EqualTo(TuPack.EncodeKey(x, y, z)));
var reader = new SliceReader(data);
Assert.That(first.DecodeOrderedSelfTerm(ref reader), Is.EqualTo(x));
Assert.That(second.DecodeOrderedSelfTerm(ref reader), Is.EqualTo(y));
Assert.That(third.DecodeOrderedSelfTerm(ref reader), Is.EqualTo(z));
Assert.That(reader.HasMore, Is.False);
}
public void Test_KeyRange_Test()
{
const int BEFORE = -1, INSIDE = 0, AFTER = +1;
KeyRange range;
// range: [ "A", "Z" )
range = KeyRange.Create(Slice.FromByteString("A"), Slice.FromByteString("Z"));
// Excluding the end: < "Z"
Assert.That(range.Test(Slice.FromByteString("\x00"), endIncluded: false), Is.EqualTo(BEFORE));
Assert.That(range.Test(Slice.FromByteString("@"), endIncluded: false), Is.EqualTo(BEFORE));
Assert.That(range.Test(Slice.FromByteString("A"), endIncluded: false), Is.EqualTo(INSIDE));
Assert.That(range.Test(Slice.FromByteString("Z"), endIncluded: false), Is.EqualTo(AFTER));
Assert.That(range.Test(Slice.FromByteString("Z\x00"), endIncluded: false), Is.EqualTo(AFTER));
Assert.That(range.Test(Slice.FromByteString("\xFF"), endIncluded: false), Is.EqualTo(AFTER));
// Including the end: <= "Z"
Assert.That(range.Test(Slice.FromByteString("\x00"), endIncluded: true), Is.EqualTo(BEFORE));
Assert.That(range.Test(Slice.FromByteString("@"), endIncluded: true), Is.EqualTo(BEFORE));
Assert.That(range.Test(Slice.FromByteString("A"), endIncluded: true), Is.EqualTo(INSIDE));
Assert.That(range.Test(Slice.FromByteString("Z"), endIncluded: true), Is.EqualTo(INSIDE));
Assert.That(range.Test(Slice.FromByteString("Z\x00"), endIncluded: true), Is.EqualTo(AFTER));
Assert.That(range.Test(Slice.FromByteString("\xFF"), endIncluded: true), Is.EqualTo(AFTER));
range = KeyRange.Create(TuPack.EncodeKey("A"), TuPack.EncodeKey("Z"));
Assert.That(range.Test(TuPack.EncodeKey("@")), Is.EqualTo((BEFORE)));
Assert.That(range.Test(TuPack.EncodeKey("A")), Is.EqualTo((INSIDE)));
Assert.That(range.Test(TuPack.EncodeKey("Z")), Is.EqualTo((AFTER)));
Assert.That(range.Test(TuPack.EncodeKey("Z"), endIncluded: true), Is.EqualTo(INSIDE));
}
public async Task Test_Vector_Fast()
{
using (var db = await OpenTestPartitionAsync())
{
var location = db.Root["ranked_set"];
await CleanLocation(db, location);
var rankedSet = new FdbRankedSet(location);
await db.WriteAsync(tr => rankedSet.OpenAsync(tr), this.Cancellation);
Log(await rankedSet.ReadAsync(db, (tr, state) => PrintRankedSet(state, tr), this.Cancellation));
Log();
var rnd = new Random();
var sw = Stopwatch.StartNew();
for (int i = 0; i < 100; i++)
{
//Log("Inserting " + i);
await db.WriteAsync(async tr =>
{
var state = await rankedSet.Resolve(tr);
await state.InsertAsync(tr, TuPack.EncodeKey(rnd.Next()));
}, this.Cancellation);
}
sw.Stop();
Log($"Done in {sw.Elapsed.TotalSeconds:N3} sec");
#if FULL_DEBUG
await DumpSubspace(db, location);
#endif
Log(await rankedSet.ReadAsync(db, (tr, state) => PrintRankedSet(state, tr), this.Cancellation));
}
}
/// <summary>Return one or more fields of an hashset</summary>
/// <param name="trans">Transaction that will be used for this request</param>
/// <param name="id">Unique identifier of the hashset</param>
/// <param name="fields">List of the fields to read</param>
/// <returns>Dictionary containing the values of the selected fields, or <see cref="Slice.Empty"/> if that particular field does not exist.</returns>
public async Task <IDictionary <string, Slice> > GetAsync(IFdbReadOnlyTransaction trans, IVarTuple id, params string[] fields)
{
if (trans == null)
{
throw new ArgumentNullException(nameof(trans));
}
if (id == null)
{
throw new ArgumentNullException(nameof(id));
}
if (fields == null)
{
throw new ArgumentNullException(nameof(fields));
}
var keys = TuPack.EncodePrefixedKeys(GetKey(id), fields);
var values = await trans.GetValuesAsync(keys).ConfigureAwait(false);
Contract.Debug.Assert(values != null && values.Length == fields.Length);
var results = new Dictionary <string, Slice>(values.Length, StringComparer.OrdinalIgnoreCase);
for (int i = 0; i < fields.Length; i++)
{
results[fields[i]] = values[i];
}
return(results);
}
/// <summary>
/// Simulate a student that is really indecisive
/// </summary>
public async Task RunProducer(IFdbDatabase db, CancellationToken ct)
{
int cnt = 0;
var rnd = new Random();
this.TimeLine.Start();
while (!ct.IsCancellationRequested)
{
int k = cnt++;
Slice taskId = TuPack.EncodeKey(this.Id.GetHashCode(), k);
string msg = "Message #" + k + " from producer " + this.Id + " (" + DateTime.UtcNow.ToString("O") + ")";
var latency = Stopwatch.StartNew();
await this.WorkerPool.ScheduleTaskAsync(db, taskId, Slice.FromString(msg), ct).ConfigureAwait(false);
latency.Stop();
Console.Write(k.ToString("N0") + "\r");
this.TimeLine.Add(latency.Elapsed.TotalMilliseconds);
TimeSpan delay = TimeSpan.FromTicks(rnd.Next((int)this.DelayMin.Ticks, (int)this.DelayMax.Ticks));
await Task.Delay(delay, ct).ConfigureAwait(false);
}
this.TimeLine.Stop();
ct.ThrowIfCancellationRequested();
}
public async Task Test_Can_MergeSort()
{
int K = 3;
int N = 100;
using (var db = await OpenTestPartitionAsync())
{
var location = await GetCleanDirectory(db, "Queries", "MergeSort");
// clear!
await db.ClearRangeAsync(location, this.Cancellation);
// create K lists
var lists = Enumerable.Range(0, K).Select(i => location.Partition.ByKey(i)).ToArray();
// lists[0] contains all multiples of K ([0, 0], [K, 1], [2K, 2], ...)
// lists[1] contains all multiples of K, offset by 1 ([1, 0], [K+1, 1], [2K+1, 2], ...)
// lists[k-1] contains all multiples of K, offset by k-1 ([K-1, 0], [2K-1, 1], [3K-1, 2], ...)
// more generally: lists[k][i] = (..., MergeSort, k, (i * K) + k) = (k, i)
for (int k = 0; k < K; k++)
{
using (var tr = db.BeginTransaction(this.Cancellation))
{
for (int i = 0; i < N; i++)
{
tr.Set(lists[k].Keys.Encode((i * K) + k), TuPack.EncodeKey(k, i));
}
await tr.CommitAsync();
}
}
// MergeSorting all lists together should produce all integers from 0 to (K*N)-1, in order
// we use the last part of the key for sorting
using (var tr = db.BeginTransaction(this.Cancellation))
{
var merge = tr.MergeSort(
lists.Select(list => KeySelectorPair.Create(list.Keys.ToRange())),
kvp => location.Keys.DecodeLast <int>(kvp.Key)
);
Assert.That(merge, Is.Not.Null);
Assert.That(merge, Is.InstanceOf <MergeSortAsyncIterator <KeyValuePair <Slice, Slice>, int, KeyValuePair <Slice, Slice> > >());
var results = await merge.ToListAsync();
Assert.That(results, Is.Not.Null);
Assert.That(results.Count, Is.EqualTo(K * N));
for (int i = 0; i < K * N; i++)
{
Assert.That(location.ExtractKey(results[i].Key), Is.EqualTo(TuPack.EncodeKey(i % K, i)));
Assert.That(results[i].Value, Is.EqualTo(TuPack.EncodeKey(i % K, i / K)));
}
}
}
}
private static void DumpIndex <TKey, TVal>(string label, MemoryIndex <TKey> index, Func <TKey, int, TVal> orderBy, IComparer <TVal> comparer = null, bool heatMaps = false)
{
comparer = comparer ?? Comparer <TVal> .Default;
int total = index.Statistics.Values.Sum();
long totalLegacy = 0;
int[] map = new int[100];
double r = (double)(map.Length - 1) / total;
Log("__{0}__", label);
Log("| Indexed Value | Count | Total % | Words | Lit% | 1-Bits | Word% | Bitmap | ratio % | Legacy | ratio % |" + (heatMaps ? " HeatMap |" : ""));
Log("|:------------------------|-------:|--------:|------:|-------:|-------:|-------:|---------:|--------:|----------:|--------:|" + (heatMaps ? ":-----------------------------------------------------------------------|" : ""));
foreach (var kv in index.Values.OrderBy((kv) => orderBy(kv.Key, index.Count(kv.Key)), comparer))
{
var t = STuple.Create(kv.Key);
var tk = TuPack.Pack(t);
kv.Value.GetStatistics(out int bits, out int words, out int literals, out int _, out double ratio);
long legacyIndexSize = 0; // size estimate of a regular FDB index (..., "Value", GUID) = ""
Array.Clear(map, 0, map.Length);
foreach (var p in kv.Value.GetView())
{
map[(int)(r * p)]++;
legacyIndexSize += 3 + tk.Count + 17;
}
totalLegacy += legacyIndexSize;
int bytes = kv.Value.ToSlice().Count;
Log(string.Format(
CultureInfo.InvariantCulture,
"| {0,-24}| {1,6:N0} | {2,6:N2}% | {3,5:N0} | {4,5:N1}% | {5,6:N0} | {6,6:N2} | {7,8:N0} | {8,6:N2}% | {9,9:N0} | {10,6:N2}% |" + (heatMaps ? " `{11}` |" : ""),
/*0*/ t,
/*1*/ index.Count(kv.Key),
/*2*/ 100.0 * index.Frequency(kv.Key),
/*3*/ words,
/*4*/ (100.0 * literals) / words,
/*5*/ bits,
/*6*/ 1.0 * bits / words,
/*7*/ bytes,
/*8*/ 100.0 * ratio,
/*9*/ legacyIndexSize,
/*A*/ (100.0 * bytes) / legacyIndexSize,
/*B*/ heatMaps ? MakeHeatMap(map) : ""
));
}
Log(string.Format(
CultureInfo.InvariantCulture,
"> {0:N0} distinct value(s), {1:N0} document(s), {2:N0} bitmap bytes, {3:N0} legacy bytes",
index.Values.Count,
total,
index.Values.Values.Sum(x => x.ToSlice().Count),
totalLegacy
));
}
public void Test_FdbKey_PrettyPrint()
{
// verify that the pretty printing of keys produce a user friendly output
Assert.That(FdbKey.Dump(Slice.Nil), Is.EqualTo("<null>"));
Assert.That(FdbKey.Dump(Slice.Empty), Is.EqualTo("<empty>"));
Assert.That(FdbKey.Dump(Slice.FromByte(0)), Is.EqualTo("<00>"));
Assert.That(FdbKey.Dump(Slice.FromByte(255)), Is.EqualTo("<FF>"));
Assert.That(FdbKey.Dump(new byte[] { 0, 1, 2, 3, 4, 5, 6, 7 }.AsSlice()), Is.EqualTo("<00><01><02><03><04><05><06><07>"));
Assert.That(FdbKey.Dump(new byte[] { 255, 254, 253, 252, 251, 250, 249, 248 }.AsSlice()), Is.EqualTo("<FF><FE><FD><FC><FB><FA><F9><F8>"));
Assert.That(FdbKey.Dump(Slice.FromString("hello")), Is.EqualTo("hello"));
Assert.That(FdbKey.Dump(Slice.FromString("héllø")), Is.EqualTo("h<C3><A9>ll<C3><B8>"));
// tuples should be decoded properly
Assert.That(FdbKey.Dump(TuPack.EncodeKey(123)), Is.EqualTo("(123,)"), "Singleton tuples should end with a ','");
Assert.That(FdbKey.Dump(TuPack.EncodeKey(Slice.FromByteString("hello"))), Is.EqualTo("(`hello`,)"), "ASCII strings should use single back quotes");
Assert.That(FdbKey.Dump(TuPack.EncodeKey("héllø")), Is.EqualTo("(\"héllø\",)"), "Unicode strings should use double quotes");
Assert.That(FdbKey.Dump(TuPack.EncodeKey(new byte[] { 1, 2, 3 }.AsSlice())), Is.EqualTo("(`<01><02><03>`,)"));
Assert.That(FdbKey.Dump(TuPack.EncodeKey(123, 456)), Is.EqualTo("(123, 456)"), "Elements should be separated with a space, and not end up with ','");
Assert.That(FdbKey.Dump(TuPack.EncodeKey(default(object), true, false)), Is.EqualTo("(null, true, false)"), "Booleans should be displayed as numbers, and null should be in lowercase"); //note: even though it's tempting to using Python's "Nil", it's not very ".NETty"
//note: the string representation of double is not identical between NetFx and .NET Core! So we cannot used a constant literal here
Assert.That(FdbKey.Dump(TuPack.EncodeKey(1.0d, Math.PI, Math.E)), Is.EqualTo("(1, " + Math.PI.ToString("R", CultureInfo.InvariantCulture) + ", " + Math.E.ToString("R", CultureInfo.InvariantCulture) + ")"), "Doubles should used dot and have full precision (17 digits)");
Assert.That(FdbKey.Dump(TuPack.EncodeKey(1.0f, (float)Math.PI, (float)Math.E)), Is.EqualTo("(1, " + ((float)Math.PI).ToString("R", CultureInfo.InvariantCulture) + ", " + ((float)Math.E).ToString("R", CultureInfo.InvariantCulture) + ")"), "Singles should used dot and have full precision (10 digits)");
var guid = Guid.NewGuid();
Assert.That(FdbKey.Dump(TuPack.EncodeKey(guid)), Is.EqualTo($"({guid:B},)"), "GUIDs should be displayed as a string literal, surrounded by {{...}}, and without quotes");
var uuid128 = Uuid128.NewUuid();
Assert.That(FdbKey.Dump(TuPack.EncodeKey(uuid128)), Is.EqualTo($"({uuid128:B},)"), "Uuid128s should be displayed as a string literal, surrounded by {{...}}, and without quotes");
var uuid64 = Uuid64.NewUuid();
Assert.That(FdbKey.Dump(TuPack.EncodeKey(uuid64)), Is.EqualTo($"({uuid64:B},)"), "Uuid64s should be displayed as a string literal, surrounded by {{...}}, and without quotes");
// ranges should be decoded when possible
var key = TuPack.ToRange(STuple.Create("hello"));
// "<02>hello<00><00>" .. "<02>hello<00><FF>"
Assert.That(FdbKey.PrettyPrint(key.Begin, FdbKey.PrettyPrintMode.Begin), Is.EqualTo("(\"hello\",).<00>"));
Assert.That(FdbKey.PrettyPrint(key.End, FdbKey.PrettyPrintMode.End), Is.EqualTo("(\"hello\",).<FF>"));
key = KeyRange.StartsWith(TuPack.EncodeKey("hello"));
// "<02>hello<00>" .. "<02>hello<01>"
Assert.That(FdbKey.PrettyPrint(key.Begin, FdbKey.PrettyPrintMode.Begin), Is.EqualTo("(\"hello\",)"));
Assert.That(FdbKey.PrettyPrint(key.End, FdbKey.PrettyPrintMode.End), Is.EqualTo("(\"hello\",) + 1"));
var t = TuPack.EncodeKey(123);
Assert.That(FdbKey.PrettyPrint(t, FdbKey.PrettyPrintMode.Single), Is.EqualTo("(123,)"));
Assert.That(FdbKey.PrettyPrint(TuPack.ToRange(t).Begin, FdbKey.PrettyPrintMode.Begin), Is.EqualTo("(123,).<00>"));
Assert.That(FdbKey.PrettyPrint(TuPack.ToRange(t).End, FdbKey.PrettyPrintMode.End), Is.EqualTo("(123,).<FF>"));
}
private static async Task HelloWorld(CancellationToken ct)
{
// Connect to the "DB" database on the local cluster
using (var db = await Fdb.OpenAsync())
{
// Writes some data in to the database
using (var tr = db.BeginTransaction(ct))
{
tr.Set(TuPack.EncodeKey("Test", 123), Slice.FromString("Hello World!"));
tr.Set(TuPack.EncodeKey("Test", 456), Slice.FromInt64(DateTime.UtcNow.Ticks));
}
}
}
public void Test_Simple_String_Codec()
{
var codec = TupleCodec <string> .Default;
Assert.That(codec, Is.Not.Null);
Assert.That(codec.EncodeOrdered("héllø Wörld"), Is.EqualTo(TuPack.EncodeKey("héllø Wörld")));
Assert.That(codec.EncodeOrdered(String.Empty), Is.EqualTo(TuPack.EncodeKey("")));
Assert.That(codec.EncodeOrdered(null), Is.EqualTo(TuPack.EncodeKey(default(string))));
Assert.That(codec.DecodeOrdered(TuPack.EncodeKey("héllø Wörld")), Is.EqualTo("héllø Wörld"));
Assert.That(codec.DecodeOrdered(TuPack.EncodeKey(String.Empty)), Is.EqualTo(""));
Assert.That(codec.DecodeOrdered(TuPack.EncodeKey(default(string))), Is.Null);
}
public void Test_Simple_Integer_Codec()
{
var codec = TupleCodec <int> .Default;
Assert.That(codec, Is.Not.Null);
Assert.That(codec.EncodeOrdered(0), Is.EqualTo(TuPack.EncodeKey(0)));
Assert.That(codec.EncodeOrdered(123), Is.EqualTo(TuPack.EncodeKey(123)));
Assert.That(codec.EncodeOrdered(123456), Is.EqualTo(TuPack.EncodeKey(123456)));
Assert.That(codec.DecodeOrdered(TuPack.EncodeKey(0)), Is.EqualTo(0));
Assert.That(codec.DecodeOrdered(TuPack.EncodeKey(123)), Is.EqualTo(123));
Assert.That(codec.DecodeOrdered(TuPack.EncodeKey(123456)), Is.EqualTo(123456));
}
private static Slice MakeKey(IKeySubspace folder, object key)
{
if (key is IVarTuple t)
{
return(folder[TuPack.Pack(t)]);
}
else if (key is string s)
{
return(folder[Slice.FromStringUtf8(s)]);
}
else
{
throw new FormatException("Unsupported key type: " + key);
}
}
public void Test_FdbQueryRangeExpression()
{
var expr = FdbQueryExpressions.Range(
KeySelectorPair.Create(TuPack.ToKeyRange("Foo"))
);
Log(expr);
Assert.That(expr, Is.Not.Null);
Assert.That(expr.Range.Begin.Key.ToString(), Is.EqualTo("<02>Foo<00><00>"));
Assert.That(expr.Range.End.Key.ToString(), Is.EqualTo("<02>Foo<00><FF>"));
Assert.That(expr.Type, Is.EqualTo(typeof(IAsyncEnumerable <KeyValuePair <Slice, Slice> >)));
Assert.That(expr.ElementType, Is.EqualTo(typeof(KeyValuePair <Slice, Slice>)));
Log(FdbQueryExpressions.ExplainSequence(expr));
}
public KeyValuePair <IVarTuple, Slice>[] Split(List <KeyValuePair <string, IVarTuple> > document)
{
if (document == null)
{
throw new ArgumentNullException(nameof(document));
}
return(document
// don't include the id
.Where(kvp => !m_keyComparer.Equals(kvp.Key, this.IdName))
// convert into tuples
.Select(kvp => new KeyValuePair <IVarTuple, Slice>(
STuple.Create(kvp.Key),
TuPack.Pack(kvp.Value)
))
.ToArray());
}
public void Test_Tuple_Composite_Encoder()
{
// encodes a key using 3 parts: (x, y, z) => ordered_key
string x = "abc";
long y = 123;
Guid z = Guid.NewGuid();
var encoder = TuPack.Encoding.GetKeyEncoder <string, long, Guid>();
Assert.That(encoder, Is.Not.Null);
// full key encoding
// note: EncodeKey(...) is just a shortcurt for packing all items in a tuple, and EncodeComposite(..., count = 3)
var data = encoder.EncodeKey(x, y, z);
Assert.That(data, Is.EqualTo(TuPack.EncodeKey(x, y, z)));
var items = encoder.DecodeKey(data);
Assert.That(items.Item1, Is.EqualTo(x));
Assert.That(items.Item2, Is.EqualTo(y));
Assert.That(items.Item3, Is.EqualTo(z));
// partial key encoding
data = encoder.EncodeKeyParts(2, items);
Assert.That(data, Is.EqualTo(TuPack.EncodeKey(x, y)));
items = encoder.DecodeKeyParts(2, TuPack.EncodeKey(x, y));
Assert.That(items.Item1, Is.EqualTo(x));
Assert.That(items.Item2, Is.EqualTo(y));
Assert.That(items.Item3, Is.EqualTo(default(Guid)));
data = encoder.EncodeKeyParts(1, items);
Assert.That(data, Is.EqualTo(TuPack.EncodeKey(x)));
items = encoder.DecodeKeyParts(1, TuPack.EncodeKey(x));
Assert.That(items.Item1, Is.EqualTo(x));
Assert.That(items.Item2, Is.EqualTo(default(long)));
Assert.That(items.Item3, Is.EqualTo(default(Guid)));
// should fail if number of items to encode is out of range
Assert.That(() => { encoder.EncodeKeyParts(4, items); }, Throws.InstanceOf <ArgumentOutOfRangeException>());
Assert.That(() => { encoder.EncodeKeyParts(0, items); }, Throws.InstanceOf <ArgumentOutOfRangeException>());
}
public List <KeyValuePair <string, IVarTuple> > Build(KeyValuePair <IVarTuple, Slice>[] parts)
{
if (parts == null)
{
throw new ArgumentNullException(nameof(parts));
}
var list = new List <KeyValuePair <string, IVarTuple> >(parts.Length);
foreach (var part in parts)
{
list.Add(new KeyValuePair <string, IVarTuple>(
part.Key.Last <string>(),
TuPack.Unpack(part.Value)
));
}
return(list);
}
public void Test_FdbQueryTransformExpression()
{
var expr = FdbQueryExpressions.Transform(
FdbQueryExpressions.RangeStartsWith(TuPack.EncodeKey("Hello", "World")),
(kvp) => kvp.Value.ToUnicode()
);
Log(expr);
Assert.That(expr, Is.Not.Null);
Assert.That(expr.Source, Is.Not.Null.And.InstanceOf <FdbQueryRangeExpression>());
Assert.That(expr.Transform, Is.Not.Null);
Assert.That(expr.Type, Is.EqualTo(typeof(IAsyncEnumerable <string>)));
Assert.That(expr.ElementType, Is.EqualTo(typeof(string)));
Log(FdbQueryExpressions.ExplainSequence(expr));
}
public void Test_FdbQueryFilterExpression()
{
var expr = FdbQueryExpressions.Filter(
FdbQueryExpressions.RangeStartsWith(TuPack.EncodeKey("Hello", "World")),
(kvp) => kvp.Value.ToInt32() % 2 == 0
);
Log(expr);
Assert.That(expr, Is.Not.Null);
Assert.That(expr.Source, Is.Not.Null.And.InstanceOf <FdbQueryRangeExpression>());
Assert.That(expr.Filter, Is.Not.Null);
Assert.That(expr.Type, Is.EqualTo(typeof(IAsyncEnumerable <KeyValuePair <Slice, Slice> >)));
Assert.That(expr.ElementType, Is.EqualTo(typeof(KeyValuePair <Slice, Slice>)));
Log(FdbQueryExpressions.ExplainSequence(expr));
}
public async Task Test_Can_Open_Database_With_Non_Empty_GlobalSpace()
{
// using a tuple prefix
using (var db = await Fdb.OpenAsync(new FdbConnectionOptions {
GlobalSpace = KeySubspace.FromKey(TuPack.EncodeKey("test"))
}, this.Cancellation))
{
Assert.That(db, Is.Not.Null);
Assert.That(db.GlobalSpace, Is.Not.Null);
Assert.That(db.GlobalSpace.GetPrefix().ToString(), Is.EqualTo("<02>test<00>"));
var subspace = db.Partition.ByKey("hello");
Assert.That(subspace.GetPrefix().ToString(), Is.EqualTo("<02>test<00><02>hello<00>"));
// keys inside the global space are valid
Assert.That(db.IsKeyValid(TuPack.EncodeKey("test", 123)), Is.True);
// keys outside the global space are invalid
Assert.That(db.IsKeyValid(Slice.FromByte(42)), Is.False);
}
// using a random binary prefix
using (var db = await Fdb.OpenAsync(new FdbConnectionOptions {
GlobalSpace = KeySubspace.FromKey(new byte[] { 42, 255, 0, 90 }.AsSlice())
}, this.Cancellation))
{
Assert.That(db, Is.Not.Null);
Assert.That(db.GlobalSpace, Is.Not.Null);
Assert.That(db.GlobalSpace.GetPrefix().ToString(), Is.EqualTo("*<FF><00>Z"));
var subspace = db.Partition.ByKey("hello");
Assert.That(subspace.GetPrefix().ToString(), Is.EqualTo("*<FF><00>Z<02>hello<00>"));
// keys inside the global space are valid
Assert.That(db.IsKeyValid(Slice.Unescape("*<FF><00>Z123")), Is.True);
// keys outside the global space are invalid
Assert.That(db.IsKeyValid(Slice.FromByte(123)), Is.False);
Assert.That(db.IsKeyValid(Slice.Unescape("*<FF>")), Is.False);
}
}
/// <summary>Return the list the names of all fields of an hashset</summary>
/// <param name="trans">Transaction that will be used for this request</param>
/// <param name="id">Unique identifier of the hashset</param>
/// <returns>List of all fields. If the list is empty, the hashset does not exist</returns>
public Task <List <string> > GetKeys(IFdbReadOnlyTransaction trans, IVarTuple id)
{
//note: As of Beta2, FDB does not have a fdb_get_range that only return the keys. That means that we will have to also read the values from the db, in order to just get the names of the fields :(
//TODO: find a way to optimize this ?
if (trans == null)
{
throw new ArgumentNullException(nameof(trans));
}
if (id == null)
{
throw new ArgumentNullException(nameof(id));
}
var prefix = GetKey(id);
return(trans
.GetRange(KeyRange.StartsWith(prefix))
.Select((kvp) => ParseFieldKey(TuPack.Unpack(kvp.Key)))
.ToListAsync());
}
public static async Task DumpSubspace(IFdbReadOnlyTransaction tr, IKeySubspace subspace)
{
Assert.That(tr, Is.Not.Null);
Assert.That(subspace, Is.Not.Null);
FdbTest.Log($"Dumping content of {subspace} at {subspace.GetPrefix():K}:");
int count = 0;
await tr
.GetRange(KeyRange.StartsWith(subspace.GetPrefix()))
.ForEachAsync((kvp) =>
{
var key = subspace.ExtractKey(kvp.Key, boundCheck: true);
++count;
string keyDump;
try
{
// attempts decoding it as a tuple
keyDump = TuPack.Unpack(key).ToString() !;
}
catch (Exception)
{
// not a tuple, dump as bytes
keyDump = "'" + key.ToString() + "'";
}
FdbTest.Log("- " + keyDump + " = " + kvp.Value.ToString());
});
if (count == 0)
{
FdbTest.Log("> empty !");
}
else
{
FdbTest.Log("> Found " + count + " values");
}
}
public T DecodeKeyLast <T>(Slice packed)
{
return(TuPack.DecodeLast <T>(packed));
}
public static async Task Get(string[] path, IVarTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
if (path == null || path.Length == 0)
{
Program.Error(log, "Cannot read keys of the Root Partition.");
return;
}
if (extras.Count == 0)
{
Program.Error(log, "You must specify a key of range of keys!");
return;
}
var folder = await db.Directory.TryOpenAsync(db, path, ct : ct);
if (folder == null)
{
Program.Error(log, "The directory does not exist anymore");
return;
}
if (folder.Layer == FdbDirectoryPartition.LayerId)
{
Program.Error(log, "Cannot clear the content of a Directory Partition!");
return;
}
object key = extras[0];
Slice k = MakeKey(folder, key);
Program.Comment(log, "# Reading key: " + k.ToString("K"));
Slice v = await db.ReadWriteAsync(tr => tr.GetAsync(k), ct);
if (v.IsNull)
{
Program.StdOut(log, "# Key does not exist in the database.", ConsoleColor.Red);
return;
}
if (v.IsEmpty)
{
Program.StdOut(log, "# Key exists but is empty.", ConsoleColor.Gray);
return;
}
Program.StdOut(log, $"# Size: {v.Count:N0}", ConsoleColor.Gray);
string format = extras.Count > 1 ? extras.Get <string>(1) : null;
switch (format)
{
case "--text":
case "--json":
case "--utf8":
{
Program.StdOut(log, v.ToStringUtf8(), ConsoleColor.Gray);
break;
}
case "--hex":
case "--hexa":
{
Program.StdOut(log, v.ToHexaString(), ConsoleColor.White);
break;
}
case "--dump":
{
var sb = new StringBuilder(v.Count * 3 + (v.Count >> 4) * 2 + 16);
for (int i = 0; i < v.Count; i += 16)
{
sb.AppendLine(v.Substring(i, 16).ToHexaString(' '));
}
Program.StdOut(log, sb.ToString(), ConsoleColor.White);
break;
}
case "--int":
{
if (v.Count <= 8)
{
long he = v.ToInt64BE();
long le = v.ToInt64();
Program.StdOut(log, $"BE: {he:X016} ({he:N0})", ConsoleColor.White);
Program.StdOut(log, $"LE: {le:X016} ({le:N0})", ConsoleColor.White);
}
else
{
Program.StdOut(log, $"Value is too large ({v.Count} bytes)", ConsoleColor.DarkRed);
Program.StdOut(log, v.ToHexaString(' '), ConsoleColor.Gray);
}
break;
}
case "--uint":
{
if (v.Count <= 8)
{
ulong he = v.ToUInt64BE();
ulong le = v.ToUInt64();
Program.StdOut(log, $"BE: {he:X016} ({he:N0})", ConsoleColor.White);
Program.StdOut(log, $"LE: {le:X016} ({le:N0})", ConsoleColor.White);
}
else
{
Program.StdOut(log, $"Value is too large ({v.Count} bytes)", ConsoleColor.DarkRed);
Program.StdOut(log, v.ToHexaString(' '), ConsoleColor.Gray);
}
break;
}
case "--tuple":
{
try
{
var t = TuPack.Unpack(v);
Program.StdOut(log, t.ToString(), ConsoleColor.Gray);
}
catch (Exception e)
{
Program.Error(log, "Key value does not seem to be a valid Tuple: " + e.Message);
Program.StdOut(log, v.ToHexaString(' '), ConsoleColor.Gray);
}
break;
}
default:
{
Program.StdOut(log, v.ToString("V"), ConsoleColor.White);
break;
}
}
}
public static FdbQueryRangeExpression RangeStartsWith(IVarTuple tuple, FdbRangeOptions options = null)
{
return(RangeStartsWith(TuPack.Pack(tuple), options));
}
public override T DecodeOrdered(Slice input)
{
return(TuPack.DecodeKey <T>(input));
}
public async Task Test_FdbMap_With_Custom_Key_Encoder()
{
// Use a table as a backing store for the rules of a Poor Man's firewall, where each keys are the IPEndPoint (tcp only!), and the values are "pass" or "block"
// Encode IPEndPoint as the (IP, Port,) encoded with the Tuple codec
// note: there is a much simpler way or creating composite keys, this is just a quick and dirty test!
var keyEncoder = new KeyEncoder <IPEndPoint>(
(ipe) => ipe == null ? Slice.Empty : TuPack.EncodeKey(ipe.Address, ipe.Port),
(packed) =>
{
if (packed.IsNullOrEmpty)
{
return(default(IPEndPoint));
}
var t = TuPack.Unpack(packed);
return(new IPEndPoint(t.Get <IPAddress>(0), t.Get <int>(1)));
}
);
var rules = new Dictionary <IPEndPoint, string>()
{
{ new IPEndPoint(IPAddress.Parse("172.16.12.34"), 6667), "block" },
{ new IPEndPoint(IPAddress.Parse("192.168.34.56"), 80), "pass" },
{ new IPEndPoint(IPAddress.Parse("192.168.34.56"), 443), "pass" }
};
using (var db = await OpenTestPartitionAsync())
{
var location = db.Root["Collections"]["Maps"];
await CleanLocation(db, location);
var mapHosts = new FdbMap <IPEndPoint, string>(location.ByKey("Hosts").AsTyped <IPEndPoint>(keyEncoder), BinaryEncoding.StringEncoder);
// import all the rules
await mapHosts.WriteAsync(db, (tr, hosts) =>
{
foreach (var rule in rules)
{
hosts.Set(tr, rule.Key, rule.Value);
}
}, this.Cancellation);
#if FULL_DEBUG
await DumpSubspace(db, location);
#endif
// test the rules
await mapHosts.ReadAsync(db, async (tr, hosts) =>
{
var value = await hosts.GetAsync(tr, new IPEndPoint(IPAddress.Parse("172.16.12.34"), 6667));
Assert.That(value, Is.EqualTo("block"));
value = await hosts.GetAsync(tr, new IPEndPoint(IPAddress.Parse("192.168.34.56"), 443));
Assert.That(value, Is.EqualTo("pass"));
var baz = new IPEndPoint(IPAddress.Parse("172.16.12.34"), 80);
Assert.That(async() => await hosts.GetAsync(tr, baz), Throws.InstanceOf <KeyNotFoundException>());
var opt = await hosts.TryGetAsync(tr, baz);
Assert.That(opt.HasValue, Is.False);
}, this.Cancellation);
}
}
public IVarTuple UnpackKey(Slice packed)
{
return(TuPack.Unpack(packed));
}
public T DecodeKeyFirst <T>(Slice packed)
{
return(TuPack.DecodeFirst <T>(packed));
}
