/// <summary>Creates a new directory</summary>
public static async Task CreateDirectory(string[] path, IFdbTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
if (log == null)
{
log = Console.Out;
}
string layer = extras.Count > 0 ? extras.Get <string>(0) : null;
log.WriteLine("# Creating directory {0} with layer '{1}'", String.Join("/", path), layer);
var folder = await db.Directory.TryOpenAsync(path, cancellationToken : ct);
if (folder != null)
{
log.WriteLine("- Directory {0} already exists!", string.Join("/", path));
return;
}
folder = await db.Directory.TryCreateAsync(path, Slice.FromString(layer), cancellationToken : ct);
log.WriteLine("- Created under {0} [{1}]", FdbKey.Dump(folder.Key), folder.Key.ToHexaString(' '));
// look if there is already stuff under there
var stuff = await db.ReadAsync((tr) => tr.GetRange(folder.ToRange()).FirstOrDefaultAsync(), cancellationToken : ct);
if (stuff.Key.IsPresent)
{
log.WriteLine("CAUTION: There is already some data under {0} !");
log.WriteLine(" {0} = {1}", FdbKey.Dump(stuff.Key), stuff.Value.ToAsciiOrHexaString());
}
}
/// <summary>Creates a new directory</summary>
public static async Task CreateDirectory(string[] path, IVarTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
if (log == null)
{
log = Console.Out;
}
string layer = extras.Count > 0 ? extras.Get <string>(0) : null;
log.WriteLine($"# Creating directory {String.Join("/", path)} with layer '{layer}'");
var folder = await db.Directory.TryOpenAsync(db, path, ct : ct);
if (folder != null)
{
log.WriteLine($"- Directory {string.Join("/", path)} already exists!");
return;
}
folder = await db.Directory.TryCreateAsync(db, path, Slice.FromString(layer), ct : ct);
log.WriteLine($"- Created under {FdbKey.Dump(folder.GetPrefix())} [{folder.GetPrefix().ToHexaString(' ')}]");
// look if there is already stuff under there
var stuff = await db.ReadAsync((tr) => tr.GetRange(folder.Keys.ToRange()).FirstOrDefaultAsync(), ct : ct);
if (stuff.Key.IsPresent)
{
Program.Error(log, $"CAUTION: There is already some data under {string.Join("/", path)} !");
Program.Error(log, $" {FdbKey.Dump(stuff.Key)} = {stuff.Value:V}");
}
}
/// <summary>Counts the number of keys inside a directory</summary>
public static async Task Count(string[] path, IVarTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
// look if there is something under there
var folder = (await TryOpenCurrentDirectoryAsync(path, db, ct)) as FdbDirectorySubspace;
if (folder == null)
{
Program.Error(log, $"# Directory {String.Join("/", path)} does not exist");
return;
}
var copy = folder.Copy();
log.WriteLine($"# Counting keys under {FdbKey.Dump(copy.GetPrefix())} ...");
var progress = new Progress <(long Count, Slice Current)>((state) =>
{
log.Write($"\r# Found {state.Count:N0} keys...");
});
long count = await Fdb.System.EstimateCountAsync(db, copy.ToRange(), progress, ct);
log.Write("\r");
Program.Success(log, $"Found {count:N0} keys in {folder.FullName}");
}
/// <summary>Shows the first few keys of a directory</summary>
public static async Task Show(string[] path, IVarTuple extras, bool reverse, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
int count = 20;
if (extras.Count > 0)
{
int x = extras.Get <int>(0);
if (x > 0)
{
count = x;
}
}
if (path == null || path.Length == 0)
{
Program.Error(log, "Cannot directory list the content of Root Partition.");
return;
}
// look if there is something under there
var folder = await db.Directory.TryOpenAsync(db, path, ct : ct);
if (folder != null)
{
if (folder.Layer == FdbDirectoryPartition.LayerId)
{
Program.Error(log, "Cannot list the content of a Directory Partition!");
return;
}
Program.Comment(log, $"# Content of {FdbKey.Dump(folder.GetPrefix())} [{folder.GetPrefix().ToHexaString(' ')}]");
var keys = await db.QueryAsync((tr) =>
{
var query = tr.GetRange(folder.Keys.ToRange());
return(reverse
? query.Reverse().Take(count)
: query.Take(count + 1));
}, ct : ct);
if (keys.Count > 0)
{
if (reverse)
{
keys.Reverse();
}
foreach (var key in keys.Take(count))
{
log.WriteLine($"...{FdbKey.Dump(folder.ExtractKey(key.Key))} = {key.Value:V}");
}
if (!reverse && keys.Count == count + 1)
{
log.WriteLine("... more");
}
}
else
{
log.WriteLine("Folder is empty");
}
}
}
public static async Task Shards(string[] path, ITuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
var ranges = await Fdb.System.GetChunksAsync(db, FdbKey.MinValue, FdbKey.MaxValue, ct);
Console.WriteLine("Found {0} shards in the whole cluster", ranges.Count);
// look if there is something under there
var folder = (await TryOpenCurrentDirectoryAsync(path, db, ct)) as FdbDirectorySubspace;
if (folder != null)
{
var r = KeyRange.StartsWith(FdbSubspace.Copy(folder).Key);
Console.WriteLine("Searching for shards that intersect with /{0} ...", String.Join("/", path));
ranges = await Fdb.System.GetChunksAsync(db, r, ct);
Console.WriteLine("Found {0} ranges intersecting {1}:", ranges.Count, r);
var last = Slice.Empty;
foreach (var range in ranges)
{
Console.Write("> " + FdbKey.Dump(range.Begin) + " ...");
long count = await Fdb.System.EstimateCountAsync(db, range, ct);
Console.WriteLine(" {0:N0}", count);
last = range.End;
//TODO: we can probably get more details on this shard looking in the system keyspace (where it is, how many replicas, ...)
}
Console.WriteLine("> ... " + FdbKey.Dump(last));
}
//Console.WriteLine("Found " + ranges.Count + " shards in the cluster");
//TODO: shards that intersect the current directory
}
public static async Task Shards(string[] path, IVarTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
var ranges = await Fdb.System.GetChunksAsync(db, FdbKey.MinValue, FdbKey.MaxValue, ct);
Console.WriteLine($"Found {ranges.Count} shards in the whole cluster");
// look if there is something under there
if ((await TryOpenCurrentDirectoryAsync(path, db, ct)) is FdbDirectorySubspace folder)
{
var r = KeyRange.StartsWith(folder.Copy().GetPrefix());
Console.WriteLine($"Searching for shards that intersect with /{String.Join("/", path)} ...");
ranges = await Fdb.System.GetChunksAsync(db, r, ct);
Console.WriteLine($"> Found {ranges.Count} ranges intersecting {r}:");
var last = Slice.Empty;
foreach (var range in ranges)
{
Console.Write($"> {FdbKey.Dump(range.Begin)} ...");
long count = await Fdb.System.EstimateCountAsync(db, range, ct);
Console.WriteLine($" {count:N0}");
last = range.End;
//TODO: we can probably get more details on this shard looking in the system keyspace (where it is, how many replicas, ...)
}
Console.WriteLine($"> ... {FdbKey.Dump(last)}");
}
//Console.WriteLine("Found " + ranges.Count + " shards in the cluster");
//TODO: shards that intersect the current directory
}
protected virtual string ResolveKey(Slice key, Func <Slice, string> resolver)
{
if (resolver == null)
{
return(FdbKey.Dump(key));
}
return(resolver(key));
}
public static async Task Dump(string[] path, string output, IVarTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
// look if there is something under there
var folder = await db.Directory.TryOpenAsync(db, path, ct : ct);
if (folder == null)
{
Program.Comment(log, $"# Directory {String.Join("/", path)} does not exist");
return;
}
//TODO: if file already exists, ask confirmation before overriding?
output = Path.GetFullPath(output);
File.Delete(output);
using (var fs = new FileStream(output, FileMode.Create, FileAccess.Write, FileShare.Read, 64 * 1024))
using (var sw = new StreamWriter(fs, Encoding.UTF8, 32 * 1024))
{
Program.Comment(log, $"# Dumping content of {FdbKey.Dump(folder.GetPrefix())} [{folder.GetPrefix().ToHexaString(' ')}] to {output}");
long bytes = 0;
var kr = new[] { '|', '/', '-', '\\' };
int p = 0;
var count = await Fdb.Bulk.ExportAsync(
db,
folder.ToRange(),
(batch, offset, _ct) =>
{
if (log == Console.Out)
{
log.Write($"\r{kr[(p++) % kr.Length]} {bytes:N0} bytes");
}
foreach (var kv in batch)
{
bytes += kv.Key.Count + kv.Value.Count;
sw.WriteLine("{0} = {1:V}", FdbKey.Dump(folder.ExtractKey(kv.Key)), kv.Value);
}
ct.ThrowIfCancellationRequested();
return(sw.FlushAsync());
},
ct
);
if (log == Console.Out)
{
log.Write("\r");
}
if (count > 0)
{
Program.Success(log, $"Found {count:N0} keys ({bytes:N0} bytes)");
}
else
{
Program.StdOut(log, "Folder is empty.");
}
}
}
public void Test_FdbKey_PrettyPrint()
{
// verify that the pretty printing of keys produce a user friendly output
Assert.That(FdbKey.Dump(Slice.Nil), Is.EqualTo("<null>"));
Assert.That(FdbKey.Dump(Slice.Empty), Is.EqualTo("<empty>"));
Assert.That(FdbKey.Dump(Slice.FromByte(0)), Is.EqualTo("<00>"));
Assert.That(FdbKey.Dump(Slice.FromByte(255)), Is.EqualTo("<FF>"));
Assert.That(FdbKey.Dump(new byte[] { 0, 1, 2, 3, 4, 5, 6, 7 }.AsSlice()), Is.EqualTo("<00><01><02><03><04><05><06><07>"));
Assert.That(FdbKey.Dump(new byte[] { 255, 254, 253, 252, 251, 250, 249, 248 }.AsSlice()), Is.EqualTo("<FF><FE><FD><FC><FB><FA><F9><F8>"));
Assert.That(FdbKey.Dump(Slice.FromString("hello")), Is.EqualTo("hello"));
Assert.That(FdbKey.Dump(Slice.FromString("héllø")), Is.EqualTo("h<C3><A9>ll<C3><B8>"));
// tuples should be decoded properly
Assert.That(FdbKey.Dump(TuPack.EncodeKey(123)), Is.EqualTo("(123,)"), "Singleton tuples should end with a ','");
Assert.That(FdbKey.Dump(TuPack.EncodeKey(Slice.FromByteString("hello"))), Is.EqualTo("(`hello`,)"), "ASCII strings should use single back quotes");
Assert.That(FdbKey.Dump(TuPack.EncodeKey("héllø")), Is.EqualTo("(\"héllø\",)"), "Unicode strings should use double quotes");
Assert.That(FdbKey.Dump(TuPack.EncodeKey(new byte[] { 1, 2, 3 }.AsSlice())), Is.EqualTo("(`<01><02><03>`,)"));
Assert.That(FdbKey.Dump(TuPack.EncodeKey(123, 456)), Is.EqualTo("(123, 456)"), "Elements should be separated with a space, and not end up with ','");
Assert.That(FdbKey.Dump(TuPack.EncodeKey(default(object), true, false)), Is.EqualTo("(null, true, false)"), "Booleans should be displayed as numbers, and null should be in lowercase"); //note: even though it's tempting to using Python's "Nil", it's not very ".NETty"
//note: the string representation of double is not identical between NetFx and .NET Core! So we cannot used a constant literal here
Assert.That(FdbKey.Dump(TuPack.EncodeKey(1.0d, Math.PI, Math.E)), Is.EqualTo("(1, " + Math.PI.ToString("R", CultureInfo.InvariantCulture) + ", " + Math.E.ToString("R", CultureInfo.InvariantCulture) + ")"), "Doubles should used dot and have full precision (17 digits)");
Assert.That(FdbKey.Dump(TuPack.EncodeKey(1.0f, (float)Math.PI, (float)Math.E)), Is.EqualTo("(1, " + ((float)Math.PI).ToString("R", CultureInfo.InvariantCulture) + ", " + ((float)Math.E).ToString("R", CultureInfo.InvariantCulture) + ")"), "Singles should used dot and have full precision (10 digits)");
var guid = Guid.NewGuid();
Assert.That(FdbKey.Dump(TuPack.EncodeKey(guid)), Is.EqualTo($"({guid:B},)"), "GUIDs should be displayed as a string literal, surrounded by {{...}}, and without quotes");
var uuid128 = Uuid128.NewUuid();
Assert.That(FdbKey.Dump(TuPack.EncodeKey(uuid128)), Is.EqualTo($"({uuid128:B},)"), "Uuid128s should be displayed as a string literal, surrounded by {{...}}, and without quotes");
var uuid64 = Uuid64.NewUuid();
Assert.That(FdbKey.Dump(TuPack.EncodeKey(uuid64)), Is.EqualTo($"({uuid64:B},)"), "Uuid64s should be displayed as a string literal, surrounded by {{...}}, and without quotes");
// ranges should be decoded when possible
var key = TuPack.ToRange(STuple.Create("hello"));
// "<02>hello<00><00>" .. "<02>hello<00><FF>"
Assert.That(FdbKey.PrettyPrint(key.Begin, FdbKey.PrettyPrintMode.Begin), Is.EqualTo("(\"hello\",).<00>"));
Assert.That(FdbKey.PrettyPrint(key.End, FdbKey.PrettyPrintMode.End), Is.EqualTo("(\"hello\",).<FF>"));
key = KeyRange.StartsWith(TuPack.EncodeKey("hello"));
// "<02>hello<00>" .. "<02>hello<01>"
Assert.That(FdbKey.PrettyPrint(key.Begin, FdbKey.PrettyPrintMode.Begin), Is.EqualTo("(\"hello\",)"));
Assert.That(FdbKey.PrettyPrint(key.End, FdbKey.PrettyPrintMode.End), Is.EqualTo("(\"hello\",) + 1"));
var t = TuPack.EncodeKey(123);
Assert.That(FdbKey.PrettyPrint(t, FdbKey.PrettyPrintMode.Single), Is.EqualTo("(123,)"));
Assert.That(FdbKey.PrettyPrint(TuPack.ToRange(t).Begin, FdbKey.PrettyPrintMode.Begin), Is.EqualTo("(123,).<00>"));
Assert.That(FdbKey.PrettyPrint(TuPack.ToRange(t).End, FdbKey.PrettyPrintMode.End), Is.EqualTo("(123,).<FF>"));
}
public void Test_FdbKey_PrettyPrint()
{
// verify that the pretty printing of keys produce a user friendly output
Assert.That(FdbKey.Dump(Slice.Nil), Is.EqualTo("<null>"));
Assert.That(FdbKey.Dump(Slice.Empty), Is.EqualTo("<empty>"));
Assert.That(FdbKey.Dump(Slice.FromByte(0)), Is.EqualTo("<00>"));
Assert.That(FdbKey.Dump(Slice.FromByte(255)), Is.EqualTo("<FF>"));
Assert.That(FdbKey.Dump(Slice.Create(new byte[] { 0, 1, 2, 3, 4, 5, 6, 7 })), Is.EqualTo("<00><01><02><03><04><05><06><07>"));
Assert.That(FdbKey.Dump(Slice.Create(new byte[] { 255, 254, 253, 252, 251, 250, 249, 248 })), Is.EqualTo("<FF><FE><FD><FC><FB><FA><F9><F8>"));
Assert.That(FdbKey.Dump(Slice.FromString("hello")), Is.EqualTo("hello"));
Assert.That(FdbKey.Dump(Slice.FromString("héllø")), Is.EqualTo("h<C3><A9>ll<C3><B8>"));
// tuples should be decoded properly
Assert.That(FdbKey.Dump(FdbTuple.Pack(123)), Is.EqualTo("(123,)"), "Singleton tuples should end with a ','");
Assert.That(FdbKey.Dump(FdbTuple.Pack(Slice.FromAscii("hello"))), Is.EqualTo("('hello',)"), "ASCII strings should use single quotes");
Assert.That(FdbKey.Dump(FdbTuple.Pack("héllø")), Is.EqualTo("(\"héllø\",)"), "Unicode strings should use double quotes");
Assert.That(FdbKey.Dump(FdbTuple.Pack(Slice.Create(new byte[] { 1, 2, 3 }))), Is.EqualTo("(<01 02 03>,)"));
Assert.That(FdbKey.Dump(FdbTuple.Pack(123, 456)), Is.EqualTo("(123, 456)"), "Elements should be separated with a space, and not end up with ','");
Assert.That(FdbKey.Dump(FdbTuple.Pack(true, false, default(object))), Is.EqualTo("(1, 0, null)"), "Booleans should be displayed as numbers, and null should be in lowercase"); //note: even though it's tempting to using Python's "Nil", it's not very ".NETty"
Assert.That(FdbKey.Dump(FdbTuple.Pack(1.0d, Math.PI, Math.E)), Is.EqualTo("(1, 3.1415926535897931, 2.7182818284590451)"), "Doubles should used dot and have full precision (17 digits)");
Assert.That(FdbKey.Dump(FdbTuple.Pack(1.0f, (float)Math.PI, (float)Math.E)), Is.EqualTo("(1, 3.14159274, 2.71828175)"), "Singles should used dot and have full precision (10 digits)");
var guid = Guid.NewGuid();
Assert.That(FdbKey.Dump(FdbTuple.Pack(guid)), Is.EqualTo(String.Format("({0},)", guid.ToString("D"))), "GUIDs should be displayed as a string literal, without quotes");
var uuid128 = Uuid128.NewUuid();
Assert.That(FdbKey.Dump(FdbTuple.Pack(uuid128)), Is.EqualTo(String.Format("({0},)", uuid128.ToString("D"))), "Uuid128s should be displayed as a string literal, without quotes");
var uuid64 = Uuid64.NewUuid();
Assert.That(FdbKey.Dump(FdbTuple.Pack(uuid64)), Is.EqualTo(String.Format("({0},)", uuid64.ToString("D"))), "Uuid64s should be displayed as a string literal, without quotes");
// ranges should be decoded when possible
var key = FdbTuple.ToRange(FdbTuple.Create("hello"));
// "<02>hello<00><00>" .. "<02>hello<00><FF>"
Assert.That(FdbKey.PrettyPrint(key.Begin, FdbKey.PrettyPrintMode.Begin), Is.EqualTo("(\"hello\",).<00>"));
Assert.That(FdbKey.PrettyPrint(key.End, FdbKey.PrettyPrintMode.End), Is.EqualTo("(\"hello\",).<FF>"));
key = FdbKeyRange.StartsWith(FdbTuple.Pack("hello"));
// "<02>hello<00>" .. "<02>hello<01>"
Assert.That(FdbKey.PrettyPrint(key.Begin, FdbKey.PrettyPrintMode.Begin), Is.EqualTo("(\"hello\",)"));
Assert.That(FdbKey.PrettyPrint(key.End, FdbKey.PrettyPrintMode.End), Is.EqualTo("(\"hello\",) + 1"));
var t = FdbTuple.Pack(123);
Assert.That(FdbKey.PrettyPrint(t, FdbKey.PrettyPrintMode.Single), Is.EqualTo("(123,)"));
Assert.That(FdbKey.PrettyPrint(FdbTuple.ToRange(t).Begin, FdbKey.PrettyPrintMode.Begin), Is.EqualTo("(123,).<00>"));
Assert.That(FdbKey.PrettyPrint(FdbTuple.ToRange(t).End, FdbKey.PrettyPrintMode.End), Is.EqualTo("(123,).<FF>"));
}
/// <summary>Shows the first few keys of a directory</summary>
public static async Task Show(string[] path, IFdbTuple extras, bool reverse, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
int count = 20;
if (extras.Count > 0)
{
int x = extras.Get <int>(0);
if (x > 0)
{
count = x;
}
}
// look if there is something under there
var folder = await db.Directory.TryOpenAsync(path, cancellationToken : ct);
if (folder != null)
{
log.WriteLine("# Content of {0} [{1}]", FdbKey.Dump(folder.Key), folder.Key.ToHexaString(' '));
var keys = await db.QueryAsync((tr) =>
{
var query = tr.GetRange(folder.ToRange());
return(reverse
? query.Reverse().Take(count)
: query.Take(count + 1));
}, cancellationToken : ct);
if (keys.Count > 0)
{
if (reverse)
{
keys.Reverse();
}
foreach (var key in keys.Take(count))
{
log.WriteLine("...{0} = {1}", FdbKey.Dump(folder.Extract(key.Key)), key.Value.ToAsciiOrHexaString());
}
if (!reverse && keys.Count == count + 1)
{
log.WriteLine("... more");
}
}
else
{
log.WriteLine(" no content found");
}
}
}
/// <summary>Creates a new directory</summary>
public static async Task CreateDirectory(FdbPath path, IVarTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
if (log == null)
{
log = Console.Out;
}
string layer = extras.Count > 0 ? extras.Get <string>(0).Trim() : string.Empty;
if (path.LayerId != layer)
{
path.WithLayer(layer);
}
log.WriteLine($"# Creating directory {path}");
(var prefix, var created) = await db.ReadWriteAsync(async tr =>
{
var folder = await db.DirectoryLayer.TryOpenAsync(tr, path);
if (folder != null)
{
return(folder.GetPrefix(), false);
}
folder = await db.DirectoryLayer.TryCreateAsync(tr, path);
return(folder.Copy().GetPrefix(), true);
}, ct);
if (!created)
{
log.WriteLine($"- Directory {path} already exists at {FdbKey.Dump(prefix)} [{prefix.ToHexaString(' ')}]");
return;
}
log.WriteLine($"- Created under {FdbKey.Dump(prefix)} [{prefix.ToHexaString(' ')}]");
// look if there is already stuff under there
var stuff = await db.ReadAsync(async tr =>
{
var folder = await db.DirectoryLayer.TryOpenAsync(tr, path);
return(await tr.GetRange(folder.ToRange()).FirstOrDefaultAsync());
}, ct);
if (stuff.Key.IsPresent)
{
Program.Error(log, $"CAUTION: There is already some data under {path} !");
Program.Error(log, $" {FdbKey.Dump(stuff.Key)} = {stuff.Value:V}");
}
}
/// <summary>Counts the number of keys inside a directory</summary>
public static async Task Count(string[] path, IFdbTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
// look if there is something under there
var folder = (await TryOpenCurrentDirectoryAsync(path, db, ct)) as FdbDirectorySubspace;
if (folder == null)
{
log.WriteLine("# Directory {0} does not exist", String.Join("/", path));
return;
}
var copy = folder.Copy();
log.WriteLine("# Counting keys under {0} ...", FdbKey.Dump(copy.Key));
var progress = new Progress <FdbTuple <long, Slice> >((state) =>
{
log.Write("\r# Found {0:N0} keys...", state.Item1);
});
long count = await Fdb.System.EstimateCountAsync(db, copy.ToRange(), progress, ct);
log.WriteLine("\r# Found {0:N0} keys in {1}", count, folder.FullName);
}
public static FutureHandle TransactionWatch(TransactionHandle transaction, Slice key)
{
if (key.IsNullOrEmpty)
{
throw new ArgumentException("Key cannot be null or empty", "key");
fixed(byte *ptrKey = key.Array)
{
var future = NativeMethods.fdb_transaction_watch(transaction, ptrKey + key.Offset, key.Count);
Contract.Assert(future != null);
#if DEBUG_NATIVE_CALLS
Debug.WriteLine("fdb_transaction_watch(0x" + transaction.Handle.ToString("x") + ", key: '" + FdbKey.Dump(key) + "') => 0x" + future.Handle.ToString("x"));
#endif
return(future);
}
}
public async Task Test_GetKey_ReadConflicts()
{
Slice key;
using (var db = MemoryDatabase.CreateNew("FOO"))
{
var location = db.Keys;
using (var tr = db.BeginTransaction(this.Cancellation))
{
tr.Set(location.Encode(42), Slice.FromString("42"));
tr.Set(location.Encode(50), Slice.FromString("50"));
tr.Set(location.Encode(60), Slice.FromString("60"));
await tr.CommitAsync();
}
db.Debug_Dump();
Func <FdbKeySelector, Slice, Task> check = async(selector, expected) =>
{
using (var tr = db.BeginTransaction(this.Cancellation))
{
key = await tr.GetKeyAsync(selector);
await tr.CommitAsync();
Assert.That(key, Is.EqualTo(expected), selector.ToString() + " => " + FdbKey.Dump(expected));
}
};
await check(
FdbKeySelector.FirstGreaterOrEqual(location.Encode(50)),
location.Encode(50)
);
await check(
FdbKeySelector.FirstGreaterThan(location.Encode(50)),
location.Encode(60)
);
await check(
FdbKeySelector.FirstGreaterOrEqual(location.Encode(49)),
location.Encode(50)
);
await check(
FdbKeySelector.FirstGreaterThan(location.Encode(49)),
location.Encode(50)
);
await check(
FdbKeySelector.FirstGreaterOrEqual(location.Encode(49)) + 1,
location.Encode(60)
);
await check(
FdbKeySelector.FirstGreaterThan(location.Encode(49)) + 1,
location.Encode(60)
);
await check(
FdbKeySelector.LastLessOrEqual(location.Encode(49)),
location.Encode(42)
);
await check(
FdbKeySelector.LastLessThan(location.Encode(49)),
location.Encode(42)
);
}
}
public static async Task Dir(FdbPath path, IVarTuple extras, DirectoryBrowseOptions options, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
if (log == null)
{
log = Console.Out;
}
Program.Comment(log, $"# Listing {path}:");
await db.ReadAsync(async tr =>
{
var parent = await TryOpenCurrentDirectoryAsync(tr, path);
if (parent == null)
{
Program.Error(log, "Directory not found.");
return;
}
if (!string.IsNullOrEmpty(parent.Layer))
{
log.WriteLine($"# Layer: {parent.Layer}");
}
var folders = await Fdb.Directory.BrowseAsync(tr, parent);
if (folders.Count > 0)
{
// to better align the names, we allow between 16 to 40 chars for the first column.
// if there is a larger name, it will stick out!
var maxLen = Math.Min(Math.Max(folders.Keys.Max(n => n.Length), 16), 40);
foreach (var kvp in folders)
{
var name = kvp.Key;
var subfolder = kvp.Value;
if (subfolder != null)
{
if ((options & DirectoryBrowseOptions.ShowCount) != 0)
{
if (!(subfolder is FdbDirectoryPartition))
{
long count = await Fdb.System.EstimateCountAsync(db, subfolder.ToRange(), ct);
Program.StdOut(log, $" {name.PadRight(maxLen)} {FdbKey.Dump(subfolder.Copy().GetPrefix()),-12} {(string.IsNullOrEmpty(subfolder.Layer) ? "-" : ("[" + subfolder.Layer + "]")),-12} {count,9:N0}", ConsoleColor.White);
}
else
{
Program.StdOut(log, $" {name.PadRight(maxLen)} {FdbKey.Dump(subfolder.Copy().GetPrefix()),-12} {(string.IsNullOrEmpty(subfolder.Layer) ? "-" : ("[" + subfolder.Layer + "]")),-12} {"-",9}", ConsoleColor.White);
}
}
else
{
Program.StdOut(log, $" {name.PadRight(maxLen)} {FdbKey.Dump(subfolder.Copy().GetPrefix()),-12} {(string.IsNullOrEmpty(subfolder.Layer) ? "-" : ("[" + subfolder.Layer + "]")),-12}", ConsoleColor.White);
}
}
else
{
Program.Error(log, $" WARNING: {name} seems to be missing!");
}
}
log.WriteLine($" {folders.Count} sub-directories.");
}
else
{
//TODO: test if it contains data?
log.WriteLine("No sub-directories.");
}
//TODO: check if there is at least one key?
}, ct);
}
public static async Task Dir(string[] path, IVarTuple extras, DirectoryBrowseOptions options, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
if (log == null)
{
log = Console.Out;
}
Program.Comment(log, $"# Listing /{string.Join("/", path)}:");
var parent = await TryOpenCurrentDirectoryAsync(path, db, ct);
if (parent == null)
{
Program.Error(log, "Directory not found.");
return;
}
if (parent.Layer.IsPresent)
{
log.WriteLine($"# Layer: {parent.Layer:P}");
}
var folders = await Fdb.Directory.BrowseAsync(db, parent, ct);
if (folders != null && folders.Count > 0)
{
foreach (var kvp in folders)
{
var name = kvp.Key;
var subfolder = kvp.Value;
if (subfolder != null)
{
if ((options & DirectoryBrowseOptions.ShowCount) != 0)
{
if (!(subfolder is FdbDirectoryPartition))
{
long count = await Fdb.System.EstimateCountAsync(db, subfolder.Keys.ToRange(), ct);
Program.StdOut(log, $" {FdbKey.Dump(subfolder.Copy().GetPrefix()),-12} {(subfolder.Layer.IsNullOrEmpty ? "-" : ("<" + subfolder.Layer.ToUnicode() + ">")),-12} {count,9:N0} {name}", ConsoleColor.White);
}
else
{
Program.StdOut(log, $" {FdbKey.Dump(subfolder.Copy().GetPrefix()),-12} {(subfolder.Layer.IsNullOrEmpty ? "-" : ("<" + subfolder.Layer.ToUnicode() + ">")),-12} {"-",9} {name}", ConsoleColor.White);
}
}
else
{
Program.StdOut(log, $" {FdbKey.Dump(subfolder.Copy().GetPrefix()),-12} {(subfolder.Layer.IsNullOrEmpty ? "-" : ("<" + subfolder.Layer.ToUnicode() + ">")),-12} {name}", ConsoleColor.White);
}
}
else
{
Program.Error(log, $" WARNING: {name} seems to be missing!");
}
}
log.WriteLine($" {folders.Count} sub-directorie(s).");
}
else
{
//TODO: test if it contains data?
log.WriteLine("No sub-directories.");
}
//TODO: check if there is at least one key?
}
public override void Debug_Dump(bool detailed)
{
Contract.Requires(m_start != null && m_current != null);
Key *current = (Key *)m_start;
Key *end = (Key *)m_current;
Trace.WriteLine(" # KeyPage: count=" + m_count.ToString("N0") + ", used=" + this.MemoryUsage.ToString("N0") + ", capacity=" + m_capacity.ToString("N0") + ", start=0x" + new IntPtr(m_start).ToString("X8") + ", end=0x" + new IntPtr(m_current).ToString("X8"));
if (detailed)
{
while (current < end)
{
Trace.WriteLine(" - [" + Entry.GetObjectType(current).ToString() + "] 0x" + new IntPtr(current).ToString("X8") + " : " + current->Header.ToString("X8") + ", size=" + current->Size + ", h=0x" + current->HashCode.ToString("X4") + " : " + FdbKey.Dump(Key.GetData(current).ToSlice()));
var value = current->Values;
while (value != null)
{
Trace.WriteLine(" -> [" + Entry.GetObjectType(value) + "] 0x" + new IntPtr(value).ToString("X8") + " @ " + value->Sequence + " : " + Value.GetData(value).ToSlice().ToAsciiOrHexaString());
value = value->Previous;
}
current = Key.WalkNext(current);
}
}
}
public static async Task Sampling(string[] path, IVarTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
double ratio = 0.1d;
bool auto = true;
if (extras.Count > 0)
{
double x = extras.Get <double>(0);
if (x > 0 && x <= 1)
{
ratio = x;
}
auto = false;
}
var folder = await TryOpenCurrentDirectoryAsync(path, db, ct);
KeyRange span;
if (folder is FdbDirectorySubspace)
{
span = KeyRange.StartsWith((folder as FdbDirectorySubspace).Copy().GetPrefix());
log.WriteLine($"Reading list of shards for /{String.Join("/", path)} under {FdbKey.Dump(span.Begin)} ...");
}
else
{
log.WriteLine("Reading list of shards for the whole cluster ...");
span = KeyRange.All;
}
// dump keyServers
var ranges = await Fdb.System.GetChunksAsync(db, span, ct);
log.WriteLine($"> Found {ranges.Count:N0} shard(s)");
// take a sample
var samples = new List <KeyRange>();
if (ranges.Count <= 32)
{ // small enough to scan it all
samples.AddRange(ranges);
log.WriteLine($"Sampling all {samples.Count:N0} shards ...");
}
else
{ // need to take a random subset
var rnd = new Random();
int sz = Math.Max((int)Math.Ceiling(ratio * ranges.Count), 1);
if (auto)
{
if (sz > 100)
{
sz = 100; //SAFETY
}
if (sz < 32)
{
sz = Math.Max(sz, Math.Min(32, ranges.Count));
}
}
var population = new List <KeyRange>(ranges);
for (int i = 0; i < sz; i++)
{
int p = rnd.Next(population.Count);
samples.Add(population[p]);
population.RemoveAt(p);
}
log.WriteLine($"Sampling {samples.Count:N0} out of {ranges.Count:N0} shards ({(100.0 * samples.Count / ranges.Count):N1}%) ...");
}
log.WriteLine();
const string FORMAT_STRING = "{0,9} ║{1,10}{6,6} {2,-29} ║{3,10}{7,7} {4,-37} ║{5,10}";
const string SCALE_KEY = "....--------========########M";
const string SCALE_VAL = "....--------========########@@@@@@@@M";
log.WriteLine(FORMAT_STRING, "Count", "Keys", SCALE_KEY, "Values", SCALE_VAL, "Total", "med.", "med.");
var rangeOptions = new FdbRangeOptions {
Mode = FdbStreamingMode.WantAll
};
samples = samples.OrderBy(x => x.Begin).ToList();
long globalSize = 0;
long globalCount = 0;
int workers = 8; // Math.Max(4, Environment.ProcessorCount);
var sw = Stopwatch.StartNew();
var tasks = new List <Task>();
int n = samples.Count;
while (samples.Count > 0)
{
while (tasks.Count < workers && samples.Count > 0)
{
var range = samples[0];
samples.RemoveAt(0);
tasks.Add(Task.Run(async() =>
{
var kk = new RobustHistogram(RobustHistogram.TimeScale.Ticks);
var vv = new RobustHistogram(RobustHistogram.TimeScale.Ticks);
#region Method 1: get_range everything...
using (var tr = db.BeginTransaction(ct))
{
long keySize = 0;
long valueSize = 0;
long count = 0;
int iter = 0;
var beginSelector = KeySelector.FirstGreaterOrEqual(range.Begin);
var endSelector = KeySelector.FirstGreaterOrEqual(range.End);
while (true)
{
FdbRangeChunk data = default(FdbRangeChunk);
FdbException error = null;
try
{
data = await tr.Snapshot.GetRangeAsync(
beginSelector,
endSelector,
rangeOptions,
iter
).ConfigureAwait(false);
}
catch (FdbException e)
{
error = e;
}
if (error != null)
{
await tr.OnErrorAsync(error.Code).ConfigureAwait(false);
continue;
}
if (data.Count == 0)
{
break;
}
count += data.Count;
foreach (var kvp in data)
{
keySize += kvp.Key.Count;
valueSize += kvp.Value.Count;
kk.Add(TimeSpan.FromTicks(kvp.Key.Count));
vv.Add(TimeSpan.FromTicks(kvp.Value.Count));
}
if (!data.HasMore)
{
break;
}
beginSelector = KeySelector.FirstGreaterThan(data.Last);
++iter;
}
long totalSize = keySize + valueSize;
Interlocked.Add(ref globalSize, totalSize);
Interlocked.Add(ref globalCount, count);
lock (log)
{
log.WriteLine(FORMAT_STRING, count.ToString("N0"), FormatSize(keySize), kk.GetDistribution(begin: 1, end: 12000, fold: 2), FormatSize(valueSize), vv.GetDistribution(begin: 1, end: 120000, fold: 2), FormatSize(totalSize), FormatSize((int)Math.Ceiling(kk.Median)), FormatSize((int)Math.Ceiling(vv.Median)));
}
}
#endregion
#region Method 2: estimate the count using key selectors...
//long counter = await Fdb.System.EstimateCountAsync(db, range, ct);
//Console.WriteLine("COUNT = " + counter.ToString("N0"));
#endregion
}, ct));
}
var done = await Task.WhenAny(tasks);
tasks.Remove(done);
}
await Task.WhenAll(tasks);
sw.Stop();
log.WriteLine();
if (n != ranges.Count)
{
log.WriteLine($"Sampled {FormatSize(globalSize)} ({globalSize:N0} bytes) and {globalCount:N0} keys in {sw.Elapsed.TotalSeconds:N1} sec");
log.WriteLine($"> Estimated total size is {FormatSize(globalSize * ranges.Count / n)}");
}
else
{
log.WriteLine($"Found {FormatSize(globalSize)} ({globalSize:N0} bytes) and {globalCount:N0} keys in {sw.Elapsed.TotalSeconds:N1} sec");
// compare to the whole cluster
ranges = await Fdb.System.GetChunksAsync(db, FdbKey.MinValue, FdbKey.MaxValue, ct);
log.WriteLine($"> This directory contains ~{(100.0 * n / ranges.Count):N2}% of all data");
}
log.WriteLine();
}
public static async Task Dir(string[] path, IFdbTuple extras, DirectoryBrowseOptions options, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
if (log == null)
{
log = Console.Out;
}
log.WriteLine("# Listing {0}:", String.Join("/", path));
var parent = await TryOpenCurrentDirectoryAsync(path, db, ct);
if (parent == null)
{
log.WriteLine(" Directory not found.");
return;
}
if (parent.Layer.IsPresent)
{
log.WriteLine("# Layer: {0}", parent.Layer.ToAsciiOrHexaString());
}
var folders = await Fdb.Directory.BrowseAsync(db, parent, ct);
if (folders != null && folders.Count > 0)
{
foreach (var kvp in folders)
{
var name = kvp.Key;
var subfolder = kvp.Value;
if (subfolder != null)
{
if ((options & DirectoryBrowseOptions.ShowCount) != 0)
{
if (!(subfolder is FdbDirectoryPartition))
{
long count = await Fdb.System.EstimateCountAsync(db, subfolder.ToRange(), ct);
log.WriteLine(" {0,-12} {1,-12} {3,9:N0} {2}", FdbKey.Dump(subfolder.Copy().Key), subfolder.Layer.IsNullOrEmpty ? "-" : ("<" + subfolder.Layer.ToUnicode() + ">"), name, count);
}
else
{
log.WriteLine(" {0,-12} {1,-12} {3,9} {2}", FdbKey.Dump(subfolder.Copy().Key), subfolder.Layer.IsNullOrEmpty ? "-" : ("<" + subfolder.Layer.ToUnicode() + ">"), name, "-");
}
}
else
{
log.WriteLine(" {0,-12} {1,-12} {2}", FdbKey.Dump(subfolder.Copy().Key), subfolder.Layer.IsNullOrEmpty ? "-" : ("<" + subfolder.Layer.ToUnicode() + ">"), name);
}
}
else
{
log.WriteLine(" WARNING: {0} seems to be missing!", name);
}
}
log.WriteLine(" {0} sub-directorie(s).", folders.Count);
}
else
{
//TODO: test if it contains data?
log.WriteLine(" No sub-directories.");
}
}