public List <KeyValuePair <string, IVarTuple> > Build(KeyValuePair <IVarTuple, Slice>[] parts)
{
if (parts == null)
{
throw new ArgumentNullException(nameof(parts));
}
var list = new List <KeyValuePair <string, IVarTuple> >(parts.Length);
foreach (var part in parts)
{
list.Add(new KeyValuePair <string, IVarTuple>(
part.Key.Last <string>(),
TuPack.Unpack(part.Value)
));
}
return(list);
}
/// <summary>Return the list the names of all fields of an hashset</summary>
/// <param name="trans">Transaction that will be used for this request</param>
/// <param name="id">Unique identifier of the hashset</param>
/// <returns>List of all fields. If the list is empty, the hashset does not exist</returns>
public Task <List <string> > GetKeys(IFdbReadOnlyTransaction trans, IVarTuple id)
{
//note: As of Beta2, FDB does not have a fdb_get_range that only return the keys. That means that we will have to also read the values from the db, in order to just get the names of the fields :(
//TODO: find a way to optimize this ?
if (trans == null)
{
throw new ArgumentNullException(nameof(trans));
}
if (id == null)
{
throw new ArgumentNullException(nameof(id));
}
var prefix = GetKey(id);
return(trans
.GetRange(KeyRange.StartsWith(prefix))
.Select((kvp) => ParseFieldKey(TuPack.Unpack(kvp.Key)))
.ToListAsync());
}
public static async Task DumpSubspace(IFdbReadOnlyTransaction tr, IKeySubspace subspace)
{
Assert.That(tr, Is.Not.Null);
Assert.That(subspace, Is.Not.Null);
FdbTest.Log($"Dumping content of {subspace} at {subspace.GetPrefix():K}:");
int count = 0;
await tr
.GetRange(KeyRange.StartsWith(subspace.GetPrefix()))
.ForEachAsync((kvp) =>
{
var key = subspace.ExtractKey(kvp.Key, boundCheck: true);
++count;
string keyDump;
try
{
// attempts decoding it as a tuple
keyDump = TuPack.Unpack(key).ToString() !;
}
catch (Exception)
{
// not a tuple, dump as bytes
keyDump = "'" + key.ToString() + "'";
}
FdbTest.Log("- " + keyDump + " = " + kvp.Value.ToString());
});
if (count == 0)
{
FdbTest.Log("> empty !");
}
else
{
FdbTest.Log("> Found " + count + " values");
}
}
public static async Task Get(string[] path, IVarTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
if (path == null || path.Length == 0)
{
Program.Error(log, "Cannot read keys of the Root Partition.");
return;
}
if (extras.Count == 0)
{
Program.Error(log, "You must specify a key of range of keys!");
return;
}
var folder = await db.Directory.TryOpenAsync(db, path, ct : ct);
if (folder == null)
{
Program.Error(log, "The directory does not exist anymore");
return;
}
if (folder.Layer == FdbDirectoryPartition.LayerId)
{
Program.Error(log, "Cannot clear the content of a Directory Partition!");
return;
}
object key = extras[0];
Slice k = MakeKey(folder, key);
Program.Comment(log, "# Reading key: " + k.ToString("K"));
Slice v = await db.ReadWriteAsync(tr => tr.GetAsync(k), ct);
if (v.IsNull)
{
Program.StdOut(log, "# Key does not exist in the database.", ConsoleColor.Red);
return;
}
if (v.IsEmpty)
{
Program.StdOut(log, "# Key exists but is empty.", ConsoleColor.Gray);
return;
}
Program.StdOut(log, $"# Size: {v.Count:N0}", ConsoleColor.Gray);
string format = extras.Count > 1 ? extras.Get <string>(1) : null;
switch (format)
{
case "--text":
case "--json":
case "--utf8":
{
Program.StdOut(log, v.ToStringUtf8(), ConsoleColor.Gray);
break;
}
case "--hex":
case "--hexa":
{
Program.StdOut(log, v.ToHexaString(), ConsoleColor.White);
break;
}
case "--dump":
{
var sb = new StringBuilder(v.Count * 3 + (v.Count >> 4) * 2 + 16);
for (int i = 0; i < v.Count; i += 16)
{
sb.AppendLine(v.Substring(i, 16).ToHexaString(' '));
}
Program.StdOut(log, sb.ToString(), ConsoleColor.White);
break;
}
case "--int":
{
if (v.Count <= 8)
{
long he = v.ToInt64BE();
long le = v.ToInt64();
Program.StdOut(log, $"BE: {he:X016} ({he:N0})", ConsoleColor.White);
Program.StdOut(log, $"LE: {le:X016} ({le:N0})", ConsoleColor.White);
}
else
{
Program.StdOut(log, $"Value is too large ({v.Count} bytes)", ConsoleColor.DarkRed);
Program.StdOut(log, v.ToHexaString(' '), ConsoleColor.Gray);
}
break;
}
case "--uint":
{
if (v.Count <= 8)
{
ulong he = v.ToUInt64BE();
ulong le = v.ToUInt64();
Program.StdOut(log, $"BE: {he:X016} ({he:N0})", ConsoleColor.White);
Program.StdOut(log, $"LE: {le:X016} ({le:N0})", ConsoleColor.White);
}
else
{
Program.StdOut(log, $"Value is too large ({v.Count} bytes)", ConsoleColor.DarkRed);
Program.StdOut(log, v.ToHexaString(' '), ConsoleColor.Gray);
}
break;
}
case "--tuple":
{
try
{
var t = TuPack.Unpack(v);
Program.StdOut(log, t.ToString(), ConsoleColor.Gray);
}
catch (Exception e)
{
Program.Error(log, "Key value does not seem to be a valid Tuple: " + e.Message);
Program.StdOut(log, v.ToHexaString(' '), ConsoleColor.Gray);
}
break;
}
default:
{
Program.StdOut(log, v.ToString("V"), ConsoleColor.White);
break;
}
}
}
public async Task Test_FdbMap_With_Custom_Key_Encoder()
{
// Use a table as a backing store for the rules of a Poor Man's firewall, where each keys are the IPEndPoint (tcp only!), and the values are "pass" or "block"
// Encode IPEndPoint as the (IP, Port,) encoded with the Tuple codec
// note: there is a much simpler way or creating composite keys, this is just a quick and dirty test!
var keyEncoder = new KeyEncoder <IPEndPoint>(
(ipe) => ipe == null ? Slice.Empty : TuPack.EncodeKey(ipe.Address, ipe.Port),
(packed) =>
{
if (packed.IsNullOrEmpty)
{
return(default(IPEndPoint));
}
var t = TuPack.Unpack(packed);
return(new IPEndPoint(t.Get <IPAddress>(0), t.Get <int>(1)));
}
);
var rules = new Dictionary <IPEndPoint, string>()
{
{ new IPEndPoint(IPAddress.Parse("172.16.12.34"), 6667), "block" },
{ new IPEndPoint(IPAddress.Parse("192.168.34.56"), 80), "pass" },
{ new IPEndPoint(IPAddress.Parse("192.168.34.56"), 443), "pass" }
};
using (var db = await OpenTestPartitionAsync())
{
var location = db.Root["Collections"]["Maps"];
await CleanLocation(db, location);
var mapHosts = new FdbMap <IPEndPoint, string>(location.ByKey("Hosts").AsTyped <IPEndPoint>(keyEncoder), BinaryEncoding.StringEncoder);
// import all the rules
await mapHosts.WriteAsync(db, (tr, hosts) =>
{
foreach (var rule in rules)
{
hosts.Set(tr, rule.Key, rule.Value);
}
}, this.Cancellation);
#if FULL_DEBUG
await DumpSubspace(db, location);
#endif
// test the rules
await mapHosts.ReadAsync(db, async (tr, hosts) =>
{
var value = await hosts.GetAsync(tr, new IPEndPoint(IPAddress.Parse("172.16.12.34"), 6667));
Assert.That(value, Is.EqualTo("block"));
value = await hosts.GetAsync(tr, new IPEndPoint(IPAddress.Parse("192.168.34.56"), 443));
Assert.That(value, Is.EqualTo("pass"));
var baz = new IPEndPoint(IPAddress.Parse("172.16.12.34"), 80);
Assert.That(async() => await hosts.GetAsync(tr, baz), Throws.InstanceOf <KeyNotFoundException>());
var opt = await hosts.TryGetAsync(tr, baz);
Assert.That(opt.HasValue, Is.False);
}, this.Cancellation);
}
}
public IVarTuple UnpackKey(Slice packed)
{
return(TuPack.Unpack(packed));
}
public async Task Test_Range_Except_Composite_Key()
{
using (var db = await OpenTestPartitionAsync())
{
// get a clean new directory
var location = await GetCleanDirectory(db, "Queries", "ExceptComposite");
// Items contains a list of all ("user", id) that were created
var locItems = (await location.CreateOrOpenAsync(db, "Items", this.Cancellation)).AsTyped <string, int>();
// Processed contain the list of all ("user", id) that were processed
var locProcessed = (await location.CreateOrOpenAsync(db, "Processed", this.Cancellation)).AsTyped <string, int>();
// the goal is to have a query that returns the list of all unprocessed items (ie: in Items but not in Processed)
await db.WriteAsync((tr) =>
{
// Items
tr.Set(locItems.Keys["userA", 10093], Slice.Empty);
tr.Set(locItems.Keys["userA", 19238], Slice.Empty);
tr.Set(locItems.Keys["userB", 20003], Slice.Empty);
// Processed
tr.Set(locProcessed.Keys["userA", 19238], Slice.Empty);
}, this.Cancellation);
// the query (Items ∩ Processed) should return (userA, 10093) and (userB, 20003)
// First Method: pass in a list of key ranges, and merge on the (Slice, Slice) pairs
Trace.WriteLine("Method 1:");
var results = await db.QueryAsync((tr) =>
{
var query = tr.Except(
new[] { locItems.Keys.ToRange(), locProcessed.Keys.ToRange() },
(kv) => TuPack.Unpack(kv.Key).Substring(-2), // note: keys come from any of the two ranges, so we must only keep the last 2 elements of the tuple
TupleComparisons.Composite <string, int>() // compares t[0] as a string, and t[1] as an int
);
// problem: Except() still returns the original (Slice,Slice) pairs from the first range,
// meaning that we still need to unpack agin the key (this time knowing the location)
return(query.Select(kv => locItems.Keys.Decode(kv.Key)));
}, this.Cancellation);
foreach (var r in results)
{
Trace.WriteLine(r);
}
Assert.That(results.Count, Is.EqualTo(2));
Assert.That(results[0], Is.EqualTo(("userA", 10093)));
Assert.That(results[1], Is.EqualTo(("userB", 20003)));
// Second Method: pre-parse the queries, and merge on the results directly
Trace.WriteLine("Method 2:");
results = await db.QueryAsync((tr) =>
{
var items = tr
.GetRange(locItems.Keys.ToRange())
.Select(kv => locItems.Keys.Decode(kv.Key));
var processed = tr
.GetRange(locProcessed.Keys.ToRange())
.Select(kv => locProcessed.Keys.Decode(kv.Key));
// items and processed are lists of (string, int) tuples, we can compare them directly
var query = items.Except(processed, TupleComparisons.Composite <string, int>());
// query is already a list of tuples, nothing more to do
return(query);
}, this.Cancellation);
foreach (var r in results)
{
Trace.WriteLine(r);
}
Assert.That(results.Count, Is.EqualTo(2));
Assert.That(results[0], Is.EqualTo(("userA", 10093)));
Assert.That(results[1], Is.EqualTo(("userB", 20003)));
}
}
public static string PrettyPrint(Slice key, PrettyPrintMode mode)
{
if (key.Count > 1)
{
byte c = key[0];
//OPTIMIZE: maybe we need a lookup table
if (c <= 28 || c == 32 || c == 33 || c == 48 || c == 49 || c >= 254)
{ // it could be a tuple...
try
{
ITuple tuple = null;
string suffix = null;
bool skip = false;
try
{
switch (mode)
{
case PrettyPrintMode.End:
{ // the last byte will either be FF, or incremented
// for tuples, the really bad cases are for byte[]/strings (which normally end with 00)
// => pack(("string",))+\xFF => <02>string<00><FF>
// => string(("string",)) => <02>string<01>
switch (key[-1])
{
case 0xFF:
{
//***README*** if you break under here, see README in the last catch() block
tuple = TuPack.Unpack(key[0, -1]);
suffix = ".<FF>";
break;
}
case 0x01:
{
var tmp = key[0, -1] + (byte)0;
//***README*** if you break under here, see README in the last catch() block
tuple = TuPack.Unpack(tmp);
suffix = " + 1";
break;
}
}
break;
}
case PrettyPrintMode.Begin:
{ // the last byte will usually be 00
// We can't really know if the tuple ended with NULL (serialized to <00>) or if a <00> was added,
// but since the ToRange() on tuples add a <00> we can bet on the fact that it is not part of the tuple itself.
// except maybe if we have "00 FF 00" which would be the expected form of a string that ends with a <00>
if (key.Count > 2 && key[-1] == 0 && key[-2] != 0xFF)
{
//***README*** if you break under here, see README in the last catch() block
tuple = TuPack.Unpack(key[0, -1]);
suffix = ".<00>";
}
break;
}
}
}
catch (Exception e)
{
suffix = null;
skip = !(e is FormatException || e is ArgumentOutOfRangeException);
}
if (tuple == null && !skip)
{ // attempt a regular decoding
//***README*** if you break under here, see README in the last catch() block
tuple = TuPack.Unpack(key);
}
if (tuple != null)
{
return(tuple.ToString() + suffix);
}
}
catch (Exception)
{
//README: If Visual Studio is breaking inside some Tuple parsing method somewhere inside this try/catch,
// this is because your debugger is configured to automatically break on thrown exceptions of type FormatException, ArgumentException, or InvalidOperation.
// Unfortunately, there isn't much you can do except unchecking "break when this exception type is thrown". If you know a way to disable locally this behaviour, please fix this!
// => only other option would be to redesign the parsing of tuples as a TryParseXXX() that does not throw, OR to have a VerifyTuple() methods that only checks for validity....
}
}
}
return(Slice.Dump(key));
}