/// <summary>Creates a new directory</summary> public static async Task CreateDirectory(string[] path, IFdbTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct) { if (log == null) { log = Console.Out; } string layer = extras.Count > 0 ? extras.Get <string>(0) : null; log.WriteLine("# Creating directory {0} with layer '{1}'", String.Join("/", path), layer); var folder = await db.Directory.TryOpenAsync(path, cancellationToken : ct); if (folder != null) { log.WriteLine("- Directory {0} already exists!", string.Join("/", path)); return; } folder = await db.Directory.TryCreateAsync(path, Slice.FromString(layer), cancellationToken : ct); log.WriteLine("- Created under {0} [{1}]", FdbKey.Dump(folder.Key), folder.Key.ToHexaString(' ')); // look if there is already stuff under there var stuff = await db.ReadAsync((tr) => tr.GetRange(folder.ToRange()).FirstOrDefaultAsync(), cancellationToken : ct); if (stuff.Key.IsPresent) { log.WriteLine("CAUTION: There is already some data under {0} !"); log.WriteLine(" {0} = {1}", FdbKey.Dump(stuff.Key), stuff.Value.ToAsciiOrHexaString()); } }
/// <summary>Creates a new directory</summary> public static async Task CreateDirectory(string[] path, IVarTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct) { if (log == null) { log = Console.Out; } string layer = extras.Count > 0 ? extras.Get <string>(0) : null; log.WriteLine($"# Creating directory {String.Join("/", path)} with layer '{layer}'"); var folder = await db.Directory.TryOpenAsync(db, path, ct : ct); if (folder != null) { log.WriteLine($"- Directory {string.Join("/", path)} already exists!"); return; } folder = await db.Directory.TryCreateAsync(db, path, Slice.FromString(layer), ct : ct); log.WriteLine($"- Created under {FdbKey.Dump(folder.GetPrefix())} [{folder.GetPrefix().ToHexaString(' ')}]"); // look if there is already stuff under there var stuff = await db.ReadAsync((tr) => tr.GetRange(folder.Keys.ToRange()).FirstOrDefaultAsync(), ct : ct); if (stuff.Key.IsPresent) { Program.Error(log, $"CAUTION: There is already some data under {string.Join("/", path)} !"); Program.Error(log, $" {FdbKey.Dump(stuff.Key)} = {stuff.Value:V}"); } }
/// <summary>Counts the number of keys inside a directory</summary> public static async Task Count(string[] path, IVarTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct) { // look if there is something under there var folder = (await TryOpenCurrentDirectoryAsync(path, db, ct)) as FdbDirectorySubspace; if (folder == null) { Program.Error(log, $"# Directory {String.Join("/", path)} does not exist"); return; } var copy = folder.Copy(); log.WriteLine($"# Counting keys under {FdbKey.Dump(copy.GetPrefix())} ..."); var progress = new Progress <(long Count, Slice Current)>((state) => { log.Write($"\r# Found {state.Count:N0} keys..."); }); long count = await Fdb.System.EstimateCountAsync(db, copy.ToRange(), progress, ct); log.Write("\r"); Program.Success(log, $"Found {count:N0} keys in {folder.FullName}"); }
/// <summary>Shows the first few keys of a directory</summary> public static async Task Show(string[] path, IVarTuple extras, bool reverse, IFdbDatabase db, TextWriter log, CancellationToken ct) { int count = 20; if (extras.Count > 0) { int x = extras.Get <int>(0); if (x > 0) { count = x; } } if (path == null || path.Length == 0) { Program.Error(log, "Cannot directory list the content of Root Partition."); return; } // look if there is something under there var folder = await db.Directory.TryOpenAsync(db, path, ct : ct); if (folder != null) { if (folder.Layer == FdbDirectoryPartition.LayerId) { Program.Error(log, "Cannot list the content of a Directory Partition!"); return; } Program.Comment(log, $"# Content of {FdbKey.Dump(folder.GetPrefix())} [{folder.GetPrefix().ToHexaString(' ')}]"); var keys = await db.QueryAsync((tr) => { var query = tr.GetRange(folder.Keys.ToRange()); return(reverse ? query.Reverse().Take(count) : query.Take(count + 1)); }, ct : ct); if (keys.Count > 0) { if (reverse) { keys.Reverse(); } foreach (var key in keys.Take(count)) { log.WriteLine($"...{FdbKey.Dump(folder.ExtractKey(key.Key))} = {key.Value:V}"); } if (!reverse && keys.Count == count + 1) { log.WriteLine("... more"); } } else { log.WriteLine("Folder is empty"); } } }
public static async Task Shards(string[] path, ITuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct) { var ranges = await Fdb.System.GetChunksAsync(db, FdbKey.MinValue, FdbKey.MaxValue, ct); Console.WriteLine("Found {0} shards in the whole cluster", ranges.Count); // look if there is something under there var folder = (await TryOpenCurrentDirectoryAsync(path, db, ct)) as FdbDirectorySubspace; if (folder != null) { var r = KeyRange.StartsWith(FdbSubspace.Copy(folder).Key); Console.WriteLine("Searching for shards that intersect with /{0} ...", String.Join("/", path)); ranges = await Fdb.System.GetChunksAsync(db, r, ct); Console.WriteLine("Found {0} ranges intersecting {1}:", ranges.Count, r); var last = Slice.Empty; foreach (var range in ranges) { Console.Write("> " + FdbKey.Dump(range.Begin) + " ..."); long count = await Fdb.System.EstimateCountAsync(db, range, ct); Console.WriteLine(" {0:N0}", count); last = range.End; //TODO: we can probably get more details on this shard looking in the system keyspace (where it is, how many replicas, ...) } Console.WriteLine("> ... " + FdbKey.Dump(last)); } //Console.WriteLine("Found " + ranges.Count + " shards in the cluster"); //TODO: shards that intersect the current directory }
public static async Task Shards(string[] path, IVarTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct) { var ranges = await Fdb.System.GetChunksAsync(db, FdbKey.MinValue, FdbKey.MaxValue, ct); Console.WriteLine($"Found {ranges.Count} shards in the whole cluster"); // look if there is something under there if ((await TryOpenCurrentDirectoryAsync(path, db, ct)) is FdbDirectorySubspace folder) { var r = KeyRange.StartsWith(folder.Copy().GetPrefix()); Console.WriteLine($"Searching for shards that intersect with /{String.Join("/", path)} ..."); ranges = await Fdb.System.GetChunksAsync(db, r, ct); Console.WriteLine($"> Found {ranges.Count} ranges intersecting {r}:"); var last = Slice.Empty; foreach (var range in ranges) { Console.Write($"> {FdbKey.Dump(range.Begin)} ..."); long count = await Fdb.System.EstimateCountAsync(db, range, ct); Console.WriteLine($" {count:N0}"); last = range.End; //TODO: we can probably get more details on this shard looking in the system keyspace (where it is, how many replicas, ...) } Console.WriteLine($"> ... {FdbKey.Dump(last)}"); } //Console.WriteLine("Found " + ranges.Count + " shards in the cluster"); //TODO: shards that intersect the current directory }
protected virtual string ResolveKey(Slice key, Func <Slice, string> resolver) { if (resolver == null) { return(FdbKey.Dump(key)); } return(resolver(key)); }
public static async Task Dump(string[] path, string output, IVarTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct) { // look if there is something under there var folder = await db.Directory.TryOpenAsync(db, path, ct : ct); if (folder == null) { Program.Comment(log, $"# Directory {String.Join("/", path)} does not exist"); return; } //TODO: if file already exists, ask confirmation before overriding? output = Path.GetFullPath(output); File.Delete(output); using (var fs = new FileStream(output, FileMode.Create, FileAccess.Write, FileShare.Read, 64 * 1024)) using (var sw = new StreamWriter(fs, Encoding.UTF8, 32 * 1024)) { Program.Comment(log, $"# Dumping content of {FdbKey.Dump(folder.GetPrefix())} [{folder.GetPrefix().ToHexaString(' ')}] to {output}"); long bytes = 0; var kr = new[] { '|', '/', '-', '\\' }; int p = 0; var count = await Fdb.Bulk.ExportAsync( db, folder.ToRange(), (batch, offset, _ct) => { if (log == Console.Out) { log.Write($"\r{kr[(p++) % kr.Length]} {bytes:N0} bytes"); } foreach (var kv in batch) { bytes += kv.Key.Count + kv.Value.Count; sw.WriteLine("{0} = {1:V}", FdbKey.Dump(folder.ExtractKey(kv.Key)), kv.Value); } ct.ThrowIfCancellationRequested(); return(sw.FlushAsync()); }, ct ); if (log == Console.Out) { log.Write("\r"); } if (count > 0) { Program.Success(log, $"Found {count:N0} keys ({bytes:N0} bytes)"); } else { Program.StdOut(log, "Folder is empty."); } } }
public void Test_FdbKey_PrettyPrint() { // verify that the pretty printing of keys produce a user friendly output Assert.That(FdbKey.Dump(Slice.Nil), Is.EqualTo("<null>")); Assert.That(FdbKey.Dump(Slice.Empty), Is.EqualTo("<empty>")); Assert.That(FdbKey.Dump(Slice.FromByte(0)), Is.EqualTo("<00>")); Assert.That(FdbKey.Dump(Slice.FromByte(255)), Is.EqualTo("<FF>")); Assert.That(FdbKey.Dump(new byte[] { 0, 1, 2, 3, 4, 5, 6, 7 }.AsSlice()), Is.EqualTo("<00><01><02><03><04><05><06><07>")); Assert.That(FdbKey.Dump(new byte[] { 255, 254, 253, 252, 251, 250, 249, 248 }.AsSlice()), Is.EqualTo("<FF><FE><FD><FC><FB><FA><F9><F8>")); Assert.That(FdbKey.Dump(Slice.FromString("hello")), Is.EqualTo("hello")); Assert.That(FdbKey.Dump(Slice.FromString("héllø")), Is.EqualTo("h<C3><A9>ll<C3><B8>")); // tuples should be decoded properly Assert.That(FdbKey.Dump(TuPack.EncodeKey(123)), Is.EqualTo("(123,)"), "Singleton tuples should end with a ','"); Assert.That(FdbKey.Dump(TuPack.EncodeKey(Slice.FromByteString("hello"))), Is.EqualTo("(`hello`,)"), "ASCII strings should use single back quotes"); Assert.That(FdbKey.Dump(TuPack.EncodeKey("héllø")), Is.EqualTo("(\"héllø\",)"), "Unicode strings should use double quotes"); Assert.That(FdbKey.Dump(TuPack.EncodeKey(new byte[] { 1, 2, 3 }.AsSlice())), Is.EqualTo("(`<01><02><03>`,)")); Assert.That(FdbKey.Dump(TuPack.EncodeKey(123, 456)), Is.EqualTo("(123, 456)"), "Elements should be separated with a space, and not end up with ','"); Assert.That(FdbKey.Dump(TuPack.EncodeKey(default(object), true, false)), Is.EqualTo("(null, true, false)"), "Booleans should be displayed as numbers, and null should be in lowercase"); //note: even though it's tempting to using Python's "Nil", it's not very ".NETty" //note: the string representation of double is not identical between NetFx and .NET Core! So we cannot used a constant literal here Assert.That(FdbKey.Dump(TuPack.EncodeKey(1.0d, Math.PI, Math.E)), Is.EqualTo("(1, " + Math.PI.ToString("R", CultureInfo.InvariantCulture) + ", " + Math.E.ToString("R", CultureInfo.InvariantCulture) + ")"), "Doubles should used dot and have full precision (17 digits)"); Assert.That(FdbKey.Dump(TuPack.EncodeKey(1.0f, (float)Math.PI, (float)Math.E)), Is.EqualTo("(1, " + ((float)Math.PI).ToString("R", CultureInfo.InvariantCulture) + ", " + ((float)Math.E).ToString("R", CultureInfo.InvariantCulture) + ")"), "Singles should used dot and have full precision (10 digits)"); var guid = Guid.NewGuid(); Assert.That(FdbKey.Dump(TuPack.EncodeKey(guid)), Is.EqualTo($"({guid:B},)"), "GUIDs should be displayed as a string literal, surrounded by {{...}}, and without quotes"); var uuid128 = Uuid128.NewUuid(); Assert.That(FdbKey.Dump(TuPack.EncodeKey(uuid128)), Is.EqualTo($"({uuid128:B},)"), "Uuid128s should be displayed as a string literal, surrounded by {{...}}, and without quotes"); var uuid64 = Uuid64.NewUuid(); Assert.That(FdbKey.Dump(TuPack.EncodeKey(uuid64)), Is.EqualTo($"({uuid64:B},)"), "Uuid64s should be displayed as a string literal, surrounded by {{...}}, and without quotes"); // ranges should be decoded when possible var key = TuPack.ToRange(STuple.Create("hello")); // "<02>hello<00><00>" .. "<02>hello<00><FF>" Assert.That(FdbKey.PrettyPrint(key.Begin, FdbKey.PrettyPrintMode.Begin), Is.EqualTo("(\"hello\",).<00>")); Assert.That(FdbKey.PrettyPrint(key.End, FdbKey.PrettyPrintMode.End), Is.EqualTo("(\"hello\",).<FF>")); key = KeyRange.StartsWith(TuPack.EncodeKey("hello")); // "<02>hello<00>" .. "<02>hello<01>" Assert.That(FdbKey.PrettyPrint(key.Begin, FdbKey.PrettyPrintMode.Begin), Is.EqualTo("(\"hello\",)")); Assert.That(FdbKey.PrettyPrint(key.End, FdbKey.PrettyPrintMode.End), Is.EqualTo("(\"hello\",) + 1")); var t = TuPack.EncodeKey(123); Assert.That(FdbKey.PrettyPrint(t, FdbKey.PrettyPrintMode.Single), Is.EqualTo("(123,)")); Assert.That(FdbKey.PrettyPrint(TuPack.ToRange(t).Begin, FdbKey.PrettyPrintMode.Begin), Is.EqualTo("(123,).<00>")); Assert.That(FdbKey.PrettyPrint(TuPack.ToRange(t).End, FdbKey.PrettyPrintMode.End), Is.EqualTo("(123,).<FF>")); }
public void Test_FdbKey_PrettyPrint() { // verify that the pretty printing of keys produce a user friendly output Assert.That(FdbKey.Dump(Slice.Nil), Is.EqualTo("<null>")); Assert.That(FdbKey.Dump(Slice.Empty), Is.EqualTo("<empty>")); Assert.That(FdbKey.Dump(Slice.FromByte(0)), Is.EqualTo("<00>")); Assert.That(FdbKey.Dump(Slice.FromByte(255)), Is.EqualTo("<FF>")); Assert.That(FdbKey.Dump(Slice.Create(new byte[] { 0, 1, 2, 3, 4, 5, 6, 7 })), Is.EqualTo("<00><01><02><03><04><05><06><07>")); Assert.That(FdbKey.Dump(Slice.Create(new byte[] { 255, 254, 253, 252, 251, 250, 249, 248 })), Is.EqualTo("<FF><FE><FD><FC><FB><FA><F9><F8>")); Assert.That(FdbKey.Dump(Slice.FromString("hello")), Is.EqualTo("hello")); Assert.That(FdbKey.Dump(Slice.FromString("héllø")), Is.EqualTo("h<C3><A9>ll<C3><B8>")); // tuples should be decoded properly Assert.That(FdbKey.Dump(FdbTuple.Pack(123)), Is.EqualTo("(123,)"), "Singleton tuples should end with a ','"); Assert.That(FdbKey.Dump(FdbTuple.Pack(Slice.FromAscii("hello"))), Is.EqualTo("('hello',)"), "ASCII strings should use single quotes"); Assert.That(FdbKey.Dump(FdbTuple.Pack("héllø")), Is.EqualTo("(\"héllø\",)"), "Unicode strings should use double quotes"); Assert.That(FdbKey.Dump(FdbTuple.Pack(Slice.Create(new byte[] { 1, 2, 3 }))), Is.EqualTo("(<01 02 03>,)")); Assert.That(FdbKey.Dump(FdbTuple.Pack(123, 456)), Is.EqualTo("(123, 456)"), "Elements should be separated with a space, and not end up with ','"); Assert.That(FdbKey.Dump(FdbTuple.Pack(true, false, default(object))), Is.EqualTo("(1, 0, null)"), "Booleans should be displayed as numbers, and null should be in lowercase"); //note: even though it's tempting to using Python's "Nil", it's not very ".NETty" Assert.That(FdbKey.Dump(FdbTuple.Pack(1.0d, Math.PI, Math.E)), Is.EqualTo("(1, 3.1415926535897931, 2.7182818284590451)"), "Doubles should used dot and have full precision (17 digits)"); Assert.That(FdbKey.Dump(FdbTuple.Pack(1.0f, (float)Math.PI, (float)Math.E)), Is.EqualTo("(1, 3.14159274, 2.71828175)"), "Singles should used dot and have full precision (10 digits)"); var guid = Guid.NewGuid(); Assert.That(FdbKey.Dump(FdbTuple.Pack(guid)), Is.EqualTo(String.Format("({0},)", guid.ToString("D"))), "GUIDs should be displayed as a string literal, without quotes"); var uuid128 = Uuid128.NewUuid(); Assert.That(FdbKey.Dump(FdbTuple.Pack(uuid128)), Is.EqualTo(String.Format("({0},)", uuid128.ToString("D"))), "Uuid128s should be displayed as a string literal, without quotes"); var uuid64 = Uuid64.NewUuid(); Assert.That(FdbKey.Dump(FdbTuple.Pack(uuid64)), Is.EqualTo(String.Format("({0},)", uuid64.ToString("D"))), "Uuid64s should be displayed as a string literal, without quotes"); // ranges should be decoded when possible var key = FdbTuple.ToRange(FdbTuple.Create("hello")); // "<02>hello<00><00>" .. "<02>hello<00><FF>" Assert.That(FdbKey.PrettyPrint(key.Begin, FdbKey.PrettyPrintMode.Begin), Is.EqualTo("(\"hello\",).<00>")); Assert.That(FdbKey.PrettyPrint(key.End, FdbKey.PrettyPrintMode.End), Is.EqualTo("(\"hello\",).<FF>")); key = FdbKeyRange.StartsWith(FdbTuple.Pack("hello")); // "<02>hello<00>" .. "<02>hello<01>" Assert.That(FdbKey.PrettyPrint(key.Begin, FdbKey.PrettyPrintMode.Begin), Is.EqualTo("(\"hello\",)")); Assert.That(FdbKey.PrettyPrint(key.End, FdbKey.PrettyPrintMode.End), Is.EqualTo("(\"hello\",) + 1")); var t = FdbTuple.Pack(123); Assert.That(FdbKey.PrettyPrint(t, FdbKey.PrettyPrintMode.Single), Is.EqualTo("(123,)")); Assert.That(FdbKey.PrettyPrint(FdbTuple.ToRange(t).Begin, FdbKey.PrettyPrintMode.Begin), Is.EqualTo("(123,).<00>")); Assert.That(FdbKey.PrettyPrint(FdbTuple.ToRange(t).End, FdbKey.PrettyPrintMode.End), Is.EqualTo("(123,).<FF>")); }
/// <summary>Shows the first few keys of a directory</summary> public static async Task Show(string[] path, IFdbTuple extras, bool reverse, IFdbDatabase db, TextWriter log, CancellationToken ct) { int count = 20; if (extras.Count > 0) { int x = extras.Get <int>(0); if (x > 0) { count = x; } } // look if there is something under there var folder = await db.Directory.TryOpenAsync(path, cancellationToken : ct); if (folder != null) { log.WriteLine("# Content of {0} [{1}]", FdbKey.Dump(folder.Key), folder.Key.ToHexaString(' ')); var keys = await db.QueryAsync((tr) => { var query = tr.GetRange(folder.ToRange()); return(reverse ? query.Reverse().Take(count) : query.Take(count + 1)); }, cancellationToken : ct); if (keys.Count > 0) { if (reverse) { keys.Reverse(); } foreach (var key in keys.Take(count)) { log.WriteLine("...{0} = {1}", FdbKey.Dump(folder.Extract(key.Key)), key.Value.ToAsciiOrHexaString()); } if (!reverse && keys.Count == count + 1) { log.WriteLine("... more"); } } else { log.WriteLine(" no content found"); } } }
/// <summary>Creates a new directory</summary> public static async Task CreateDirectory(FdbPath path, IVarTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct) { if (log == null) { log = Console.Out; } string layer = extras.Count > 0 ? extras.Get <string>(0).Trim() : string.Empty; if (path.LayerId != layer) { path.WithLayer(layer); } log.WriteLine($"# Creating directory {path}"); (var prefix, var created) = await db.ReadWriteAsync(async tr => { var folder = await db.DirectoryLayer.TryOpenAsync(tr, path); if (folder != null) { return(folder.GetPrefix(), false); } folder = await db.DirectoryLayer.TryCreateAsync(tr, path); return(folder.Copy().GetPrefix(), true); }, ct); if (!created) { log.WriteLine($"- Directory {path} already exists at {FdbKey.Dump(prefix)} [{prefix.ToHexaString(' ')}]"); return; } log.WriteLine($"- Created under {FdbKey.Dump(prefix)} [{prefix.ToHexaString(' ')}]"); // look if there is already stuff under there var stuff = await db.ReadAsync(async tr => { var folder = await db.DirectoryLayer.TryOpenAsync(tr, path); return(await tr.GetRange(folder.ToRange()).FirstOrDefaultAsync()); }, ct); if (stuff.Key.IsPresent) { Program.Error(log, $"CAUTION: There is already some data under {path} !"); Program.Error(log, $" {FdbKey.Dump(stuff.Key)} = {stuff.Value:V}"); } }
/// <summary>Counts the number of keys inside a directory</summary> public static async Task Count(string[] path, IFdbTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct) { // look if there is something under there var folder = (await TryOpenCurrentDirectoryAsync(path, db, ct)) as FdbDirectorySubspace; if (folder == null) { log.WriteLine("# Directory {0} does not exist", String.Join("/", path)); return; } var copy = folder.Copy(); log.WriteLine("# Counting keys under {0} ...", FdbKey.Dump(copy.Key)); var progress = new Progress <FdbTuple <long, Slice> >((state) => { log.Write("\r# Found {0:N0} keys...", state.Item1); }); long count = await Fdb.System.EstimateCountAsync(db, copy.ToRange(), progress, ct); log.WriteLine("\r# Found {0:N0} keys in {1}", count, folder.FullName); }
public static FutureHandle TransactionWatch(TransactionHandle transaction, Slice key) { if (key.IsNullOrEmpty) { throw new ArgumentException("Key cannot be null or empty", "key"); fixed(byte *ptrKey = key.Array) { var future = NativeMethods.fdb_transaction_watch(transaction, ptrKey + key.Offset, key.Count); Contract.Assert(future != null); #if DEBUG_NATIVE_CALLS Debug.WriteLine("fdb_transaction_watch(0x" + transaction.Handle.ToString("x") + ", key: '" + FdbKey.Dump(key) + "') => 0x" + future.Handle.ToString("x")); #endif return(future); } }
public async Task Test_GetKey_ReadConflicts() { Slice key; using (var db = MemoryDatabase.CreateNew("FOO")) { var location = db.Keys; using (var tr = db.BeginTransaction(this.Cancellation)) { tr.Set(location.Encode(42), Slice.FromString("42")); tr.Set(location.Encode(50), Slice.FromString("50")); tr.Set(location.Encode(60), Slice.FromString("60")); await tr.CommitAsync(); } db.Debug_Dump(); Func <FdbKeySelector, Slice, Task> check = async(selector, expected) => { using (var tr = db.BeginTransaction(this.Cancellation)) { key = await tr.GetKeyAsync(selector); await tr.CommitAsync(); Assert.That(key, Is.EqualTo(expected), selector.ToString() + " => " + FdbKey.Dump(expected)); } }; await check( FdbKeySelector.FirstGreaterOrEqual(location.Encode(50)), location.Encode(50) ); await check( FdbKeySelector.FirstGreaterThan(location.Encode(50)), location.Encode(60) ); await check( FdbKeySelector.FirstGreaterOrEqual(location.Encode(49)), location.Encode(50) ); await check( FdbKeySelector.FirstGreaterThan(location.Encode(49)), location.Encode(50) ); await check( FdbKeySelector.FirstGreaterOrEqual(location.Encode(49)) + 1, location.Encode(60) ); await check( FdbKeySelector.FirstGreaterThan(location.Encode(49)) + 1, location.Encode(60) ); await check( FdbKeySelector.LastLessOrEqual(location.Encode(49)), location.Encode(42) ); await check( FdbKeySelector.LastLessThan(location.Encode(49)), location.Encode(42) ); } }
public static async Task Dir(FdbPath path, IVarTuple extras, DirectoryBrowseOptions options, IFdbDatabase db, TextWriter log, CancellationToken ct) { if (log == null) { log = Console.Out; } Program.Comment(log, $"# Listing {path}:"); await db.ReadAsync(async tr => { var parent = await TryOpenCurrentDirectoryAsync(tr, path); if (parent == null) { Program.Error(log, "Directory not found."); return; } if (!string.IsNullOrEmpty(parent.Layer)) { log.WriteLine($"# Layer: {parent.Layer}"); } var folders = await Fdb.Directory.BrowseAsync(tr, parent); if (folders.Count > 0) { // to better align the names, we allow between 16 to 40 chars for the first column. // if there is a larger name, it will stick out! var maxLen = Math.Min(Math.Max(folders.Keys.Max(n => n.Length), 16), 40); foreach (var kvp in folders) { var name = kvp.Key; var subfolder = kvp.Value; if (subfolder != null) { if ((options & DirectoryBrowseOptions.ShowCount) != 0) { if (!(subfolder is FdbDirectoryPartition)) { long count = await Fdb.System.EstimateCountAsync(db, subfolder.ToRange(), ct); Program.StdOut(log, $" {name.PadRight(maxLen)} {FdbKey.Dump(subfolder.Copy().GetPrefix()),-12} {(string.IsNullOrEmpty(subfolder.Layer) ? "-" : ("[" + subfolder.Layer + "]")),-12} {count,9:N0}", ConsoleColor.White); } else { Program.StdOut(log, $" {name.PadRight(maxLen)} {FdbKey.Dump(subfolder.Copy().GetPrefix()),-12} {(string.IsNullOrEmpty(subfolder.Layer) ? "-" : ("[" + subfolder.Layer + "]")),-12} {"-",9}", ConsoleColor.White); } } else { Program.StdOut(log, $" {name.PadRight(maxLen)} {FdbKey.Dump(subfolder.Copy().GetPrefix()),-12} {(string.IsNullOrEmpty(subfolder.Layer) ? "-" : ("[" + subfolder.Layer + "]")),-12}", ConsoleColor.White); } } else { Program.Error(log, $" WARNING: {name} seems to be missing!"); } } log.WriteLine($" {folders.Count} sub-directories."); } else { //TODO: test if it contains data? log.WriteLine("No sub-directories."); } //TODO: check if there is at least one key? }, ct); }
public static async Task Dir(string[] path, IVarTuple extras, DirectoryBrowseOptions options, IFdbDatabase db, TextWriter log, CancellationToken ct) { if (log == null) { log = Console.Out; } Program.Comment(log, $"# Listing /{string.Join("/", path)}:"); var parent = await TryOpenCurrentDirectoryAsync(path, db, ct); if (parent == null) { Program.Error(log, "Directory not found."); return; } if (parent.Layer.IsPresent) { log.WriteLine($"# Layer: {parent.Layer:P}"); } var folders = await Fdb.Directory.BrowseAsync(db, parent, ct); if (folders != null && folders.Count > 0) { foreach (var kvp in folders) { var name = kvp.Key; var subfolder = kvp.Value; if (subfolder != null) { if ((options & DirectoryBrowseOptions.ShowCount) != 0) { if (!(subfolder is FdbDirectoryPartition)) { long count = await Fdb.System.EstimateCountAsync(db, subfolder.Keys.ToRange(), ct); Program.StdOut(log, $" {FdbKey.Dump(subfolder.Copy().GetPrefix()),-12} {(subfolder.Layer.IsNullOrEmpty ? "-" : ("<" + subfolder.Layer.ToUnicode() + ">")),-12} {count,9:N0} {name}", ConsoleColor.White); } else { Program.StdOut(log, $" {FdbKey.Dump(subfolder.Copy().GetPrefix()),-12} {(subfolder.Layer.IsNullOrEmpty ? "-" : ("<" + subfolder.Layer.ToUnicode() + ">")),-12} {"-",9} {name}", ConsoleColor.White); } } else { Program.StdOut(log, $" {FdbKey.Dump(subfolder.Copy().GetPrefix()),-12} {(subfolder.Layer.IsNullOrEmpty ? "-" : ("<" + subfolder.Layer.ToUnicode() + ">")),-12} {name}", ConsoleColor.White); } } else { Program.Error(log, $" WARNING: {name} seems to be missing!"); } } log.WriteLine($" {folders.Count} sub-directorie(s)."); } else { //TODO: test if it contains data? log.WriteLine("No sub-directories."); } //TODO: check if there is at least one key? }
public override void Debug_Dump(bool detailed) { Contract.Requires(m_start != null && m_current != null); Key *current = (Key *)m_start; Key *end = (Key *)m_current; Trace.WriteLine(" # KeyPage: count=" + m_count.ToString("N0") + ", used=" + this.MemoryUsage.ToString("N0") + ", capacity=" + m_capacity.ToString("N0") + ", start=0x" + new IntPtr(m_start).ToString("X8") + ", end=0x" + new IntPtr(m_current).ToString("X8")); if (detailed) { while (current < end) { Trace.WriteLine(" - [" + Entry.GetObjectType(current).ToString() + "] 0x" + new IntPtr(current).ToString("X8") + " : " + current->Header.ToString("X8") + ", size=" + current->Size + ", h=0x" + current->HashCode.ToString("X4") + " : " + FdbKey.Dump(Key.GetData(current).ToSlice())); var value = current->Values; while (value != null) { Trace.WriteLine(" -> [" + Entry.GetObjectType(value) + "] 0x" + new IntPtr(value).ToString("X8") + " @ " + value->Sequence + " : " + Value.GetData(value).ToSlice().ToAsciiOrHexaString()); value = value->Previous; } current = Key.WalkNext(current); } } }
public static async Task Sampling(string[] path, IVarTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct) { double ratio = 0.1d; bool auto = true; if (extras.Count > 0) { double x = extras.Get <double>(0); if (x > 0 && x <= 1) { ratio = x; } auto = false; } var folder = await TryOpenCurrentDirectoryAsync(path, db, ct); KeyRange span; if (folder is FdbDirectorySubspace) { span = KeyRange.StartsWith((folder as FdbDirectorySubspace).Copy().GetPrefix()); log.WriteLine($"Reading list of shards for /{String.Join("/", path)} under {FdbKey.Dump(span.Begin)} ..."); } else { log.WriteLine("Reading list of shards for the whole cluster ..."); span = KeyRange.All; } // dump keyServers var ranges = await Fdb.System.GetChunksAsync(db, span, ct); log.WriteLine($"> Found {ranges.Count:N0} shard(s)"); // take a sample var samples = new List <KeyRange>(); if (ranges.Count <= 32) { // small enough to scan it all samples.AddRange(ranges); log.WriteLine($"Sampling all {samples.Count:N0} shards ..."); } else { // need to take a random subset var rnd = new Random(); int sz = Math.Max((int)Math.Ceiling(ratio * ranges.Count), 1); if (auto) { if (sz > 100) { sz = 100; //SAFETY } if (sz < 32) { sz = Math.Max(sz, Math.Min(32, ranges.Count)); } } var population = new List <KeyRange>(ranges); for (int i = 0; i < sz; i++) { int p = rnd.Next(population.Count); samples.Add(population[p]); population.RemoveAt(p); } log.WriteLine($"Sampling {samples.Count:N0} out of {ranges.Count:N0} shards ({(100.0 * samples.Count / ranges.Count):N1}%) ..."); } log.WriteLine(); const string FORMAT_STRING = "{0,9} ║{1,10}{6,6} {2,-29} ║{3,10}{7,7} {4,-37} ║{5,10}"; const string SCALE_KEY = "....--------========########M"; const string SCALE_VAL = "....--------========########@@@@@@@@M"; log.WriteLine(FORMAT_STRING, "Count", "Keys", SCALE_KEY, "Values", SCALE_VAL, "Total", "med.", "med."); var rangeOptions = new FdbRangeOptions { Mode = FdbStreamingMode.WantAll }; samples = samples.OrderBy(x => x.Begin).ToList(); long globalSize = 0; long globalCount = 0; int workers = 8; // Math.Max(4, Environment.ProcessorCount); var sw = Stopwatch.StartNew(); var tasks = new List <Task>(); int n = samples.Count; while (samples.Count > 0) { while (tasks.Count < workers && samples.Count > 0) { var range = samples[0]; samples.RemoveAt(0); tasks.Add(Task.Run(async() => { var kk = new RobustHistogram(RobustHistogram.TimeScale.Ticks); var vv = new RobustHistogram(RobustHistogram.TimeScale.Ticks); #region Method 1: get_range everything... using (var tr = db.BeginTransaction(ct)) { long keySize = 0; long valueSize = 0; long count = 0; int iter = 0; var beginSelector = KeySelector.FirstGreaterOrEqual(range.Begin); var endSelector = KeySelector.FirstGreaterOrEqual(range.End); while (true) { FdbRangeChunk data = default(FdbRangeChunk); FdbException error = null; try { data = await tr.Snapshot.GetRangeAsync( beginSelector, endSelector, rangeOptions, iter ).ConfigureAwait(false); } catch (FdbException e) { error = e; } if (error != null) { await tr.OnErrorAsync(error.Code).ConfigureAwait(false); continue; } if (data.Count == 0) { break; } count += data.Count; foreach (var kvp in data) { keySize += kvp.Key.Count; valueSize += kvp.Value.Count; kk.Add(TimeSpan.FromTicks(kvp.Key.Count)); vv.Add(TimeSpan.FromTicks(kvp.Value.Count)); } if (!data.HasMore) { break; } beginSelector = KeySelector.FirstGreaterThan(data.Last); ++iter; } long totalSize = keySize + valueSize; Interlocked.Add(ref globalSize, totalSize); Interlocked.Add(ref globalCount, count); lock (log) { log.WriteLine(FORMAT_STRING, count.ToString("N0"), FormatSize(keySize), kk.GetDistribution(begin: 1, end: 12000, fold: 2), FormatSize(valueSize), vv.GetDistribution(begin: 1, end: 120000, fold: 2), FormatSize(totalSize), FormatSize((int)Math.Ceiling(kk.Median)), FormatSize((int)Math.Ceiling(vv.Median))); } } #endregion #region Method 2: estimate the count using key selectors... //long counter = await Fdb.System.EstimateCountAsync(db, range, ct); //Console.WriteLine("COUNT = " + counter.ToString("N0")); #endregion }, ct)); } var done = await Task.WhenAny(tasks); tasks.Remove(done); } await Task.WhenAll(tasks); sw.Stop(); log.WriteLine(); if (n != ranges.Count) { log.WriteLine($"Sampled {FormatSize(globalSize)} ({globalSize:N0} bytes) and {globalCount:N0} keys in {sw.Elapsed.TotalSeconds:N1} sec"); log.WriteLine($"> Estimated total size is {FormatSize(globalSize * ranges.Count / n)}"); } else { log.WriteLine($"Found {FormatSize(globalSize)} ({globalSize:N0} bytes) and {globalCount:N0} keys in {sw.Elapsed.TotalSeconds:N1} sec"); // compare to the whole cluster ranges = await Fdb.System.GetChunksAsync(db, FdbKey.MinValue, FdbKey.MaxValue, ct); log.WriteLine($"> This directory contains ~{(100.0 * n / ranges.Count):N2}% of all data"); } log.WriteLine(); }
public static async Task Dir(string[] path, IFdbTuple extras, DirectoryBrowseOptions options, IFdbDatabase db, TextWriter log, CancellationToken ct) { if (log == null) { log = Console.Out; } log.WriteLine("# Listing {0}:", String.Join("/", path)); var parent = await TryOpenCurrentDirectoryAsync(path, db, ct); if (parent == null) { log.WriteLine(" Directory not found."); return; } if (parent.Layer.IsPresent) { log.WriteLine("# Layer: {0}", parent.Layer.ToAsciiOrHexaString()); } var folders = await Fdb.Directory.BrowseAsync(db, parent, ct); if (folders != null && folders.Count > 0) { foreach (var kvp in folders) { var name = kvp.Key; var subfolder = kvp.Value; if (subfolder != null) { if ((options & DirectoryBrowseOptions.ShowCount) != 0) { if (!(subfolder is FdbDirectoryPartition)) { long count = await Fdb.System.EstimateCountAsync(db, subfolder.ToRange(), ct); log.WriteLine(" {0,-12} {1,-12} {3,9:N0} {2}", FdbKey.Dump(subfolder.Copy().Key), subfolder.Layer.IsNullOrEmpty ? "-" : ("<" + subfolder.Layer.ToUnicode() + ">"), name, count); } else { log.WriteLine(" {0,-12} {1,-12} {3,9} {2}", FdbKey.Dump(subfolder.Copy().Key), subfolder.Layer.IsNullOrEmpty ? "-" : ("<" + subfolder.Layer.ToUnicode() + ">"), name, "-"); } } else { log.WriteLine(" {0,-12} {1,-12} {2}", FdbKey.Dump(subfolder.Copy().Key), subfolder.Layer.IsNullOrEmpty ? "-" : ("<" + subfolder.Layer.ToUnicode() + ">"), name); } } else { log.WriteLine(" WARNING: {0} seems to be missing!", name); } } log.WriteLine(" {0} sub-directorie(s).", folders.Count); } else { //TODO: test if it contains data? log.WriteLine(" No sub-directories."); } }