public static async Task Shards(string[] path, ITuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct) { var ranges = await Fdb.System.GetChunksAsync(db, FdbKey.MinValue, FdbKey.MaxValue, ct); Console.WriteLine("Found {0} shards in the whole cluster", ranges.Count); // look if there is something under there var folder = (await TryOpenCurrentDirectoryAsync(path, db, ct)) as FdbDirectorySubspace; if (folder != null) { var r = KeyRange.StartsWith(FdbSubspace.Copy(folder).Key); Console.WriteLine("Searching for shards that intersect with /{0} ...", String.Join("/", path)); ranges = await Fdb.System.GetChunksAsync(db, r, ct); Console.WriteLine("Found {0} ranges intersecting {1}:", ranges.Count, r); var last = Slice.Empty; foreach (var range in ranges) { Console.Write("> " + FdbKey.Dump(range.Begin) + " ..."); long count = await Fdb.System.EstimateCountAsync(db, range, ct); Console.WriteLine(" {0:N0}", count); last = range.End; //TODO: we can probably get more details on this shard looking in the system keyspace (where it is, how many replicas, ...) } Console.WriteLine("> ... " + FdbKey.Dump(last)); } //Console.WriteLine("Found " + ranges.Count + " shards in the cluster"); //TODO: shards that intersect the current directory }
public void Test_Empty_Subspace_Is_Empty() { var subspace = FdbSubspace.Empty; Assert.That(subspace, Is.Not.Null, "FdbSubspace.Empty should not return null"); Assert.That(FdbSubspace.Empty, Is.SameAs(subspace), "FdbSubspace.Empty is a singleton"); Assert.That(subspace.Key.Count, Is.EqualTo(0), "FdbSubspace.Empty.Key should be equal to Slice.Empty"); Assert.That(subspace.Key.HasValue, Is.True, "FdbSubspace.Empty.Key should be equal to Slice.Empty"); Assert.That(FdbSubspace.Copy(subspace), Is.Not.SameAs(subspace)); }
public void Test_Subspace_With_Binary_Prefix() { var subspace = new FdbSubspace(Slice.Create(new byte[] { 42, 255, 0, 127 })); Assert.That(subspace.Key.ToString(), Is.EqualTo("*<FF><00><7F>")); Assert.That(subspace.Copy(), Is.Not.SameAs(subspace)); Assert.That(subspace.Copy().Key, Is.EqualTo(subspace.Key)); // concat(Slice) should append the slice to the binary prefix directly Assert.That(subspace.Concat(Slice.FromInt32(0x01020304)).ToString(), Is.EqualTo("*<FF><00><7F><04><03><02><01>")); Assert.That(subspace.Concat(Slice.FromAscii("hello")).ToString(), Is.EqualTo("*<FF><00><7F>hello")); // pack(...) should use tuple serialization Assert.That(subspace.Pack(123).ToString(), Is.EqualTo("*<FF><00><7F><15>{")); Assert.That(subspace.Pack("hello").ToString(), Is.EqualTo("*<FF><00><7F><02>hello<00>")); Assert.That(subspace.Pack(Slice.FromAscii("world")).ToString(), Is.EqualTo("*<FF><00><7F><01>world<00>")); Assert.That(subspace.Pack(FdbTuple.Create("hello", 123)).ToString(), Is.EqualTo("*<FF><00><7F><02>hello<00><15>{")); // if we derive a tuple from this subspace, it should keep the binary prefix when converted to a key var t = subspace.Append("world", 123, false); Assert.That(t, Is.Not.Null); Assert.That(t.Count, Is.EqualTo(3)); Assert.That(t.Get <string>(0), Is.EqualTo("world")); Assert.That(t.Get <int>(1), Is.EqualTo(123)); Assert.That(t.Get <bool>(2), Is.False); var k = t.ToSlice(); Assert.That(k.ToString(), Is.EqualTo("*<FF><00><7F><02>world<00><15>{<14>")); // if we unpack the key with the binary prefix, we should get a valid tuple var t2 = subspace.Unpack(k); Assert.That(t2, Is.Not.Null); Assert.That(t2.Count, Is.EqualTo(3)); Assert.That(t2.Get <string>(0), Is.EqualTo("world")); Assert.That(t2.Get <int>(1), Is.EqualTo(123)); Assert.That(t2.Get <bool>(2), Is.False); }
public void Test_Subspace_With_Tuple_Prefix() { var subspace = new FdbSubspace(FdbTuple.Create("hello")); Assert.That(subspace.Key.ToString(), Is.EqualTo("<02>hello<00>")); Assert.That(subspace.Copy(), Is.Not.SameAs(subspace)); Assert.That(subspace.Copy().Key, Is.EqualTo(subspace.Key)); // concat(Slice) should append the slice to the tuple prefix directly Assert.That(subspace.Concat(Slice.FromInt32(0x01020304)).ToString(), Is.EqualTo("<02>hello<00><04><03><02><01>")); Assert.That(subspace.Concat(Slice.FromAscii("world")).ToString(), Is.EqualTo("<02>hello<00>world")); // pack(...) should use tuple serialization Assert.That(subspace.Pack(123).ToString(), Is.EqualTo("<02>hello<00><15>{")); Assert.That(subspace.Pack("world").ToString(), Is.EqualTo("<02>hello<00><02>world<00>")); // even though the subspace prefix is a tuple, appending to it will only return the new items var t = subspace.Append("world", 123, false); Assert.That(t, Is.Not.Null); Assert.That(t.Count, Is.EqualTo(3)); Assert.That(t.Get <string>(0), Is.EqualTo("world")); Assert.That(t.Get <int>(1), Is.EqualTo(123)); Assert.That(t.Get <bool>(2), Is.False); // but ToSlice() should include the prefix var k = t.ToSlice(); Assert.That(k.ToString(), Is.EqualTo("<02>hello<00><02>world<00><15>{<14>")); // if we unpack the key with the binary prefix, we should get a valid tuple var t2 = subspace.Unpack(k); Assert.That(t2, Is.Not.Null); Assert.That(t2.Count, Is.EqualTo(3)); Assert.That(t2.Get <string>(0), Is.EqualTo("world")); Assert.That(t2.Get <int>(1), Is.EqualTo(123)); Assert.That(t2.Get <bool>(2), Is.False); }
public void Test_Subspace_Copy_Does_Not_Share_Key_Buffer() { var original = FdbSubspace.Create(Slice.FromString("Hello")); var copy = FdbSubspace.Copy(original); Assert.That(copy, Is.Not.Null); Assert.That(copy, Is.Not.SameAs(original), "Copy should be a new instance"); Assert.That(copy.Key, Is.EqualTo(original.Key), "Key should be equal"); Assert.That(copy.Key.Array, Is.Not.SameAs(original.Key.Array), "Key should be a copy of the original"); Assert.That(copy, Is.EqualTo(original), "Copy and original should be considered equal"); Assert.That(copy.ToString(), Is.EqualTo(original.ToString()), "Copy and original should have the same string representation"); Assert.That(copy.GetHashCode(), Is.EqualTo(original.GetHashCode()), "Copy and original should have the same hashcode"); }
public void Test_Subspace_With_Binary_Prefix() { var subspace = new FdbSubspace(Slice.Create(new byte[] { 42, 255, 0, 127 })); Assert.That(subspace.Key.ToString(), Is.EqualTo("*<FF><00><7F>")); Assert.That(subspace.Copy(), Is.Not.SameAs(subspace)); Assert.That(subspace.Copy().Key, Is.EqualTo(subspace.Key)); // concat(Slice) should append the slice to the binary prefix directly Assert.That(subspace.Concat(Slice.FromInt32(0x01020304)).ToString(), Is.EqualTo("*<FF><00><7F><04><03><02><01>")); Assert.That(subspace.Concat(Slice.FromAscii("hello")).ToString(), Is.EqualTo("*<FF><00><7F>hello")); // pack(...) should use tuple serialization Assert.That(subspace.Pack(123).ToString(), Is.EqualTo("*<FF><00><7F><15>{")); Assert.That(subspace.Pack("hello").ToString(), Is.EqualTo("*<FF><00><7F><02>hello<00>")); Assert.That(subspace.Pack(Slice.FromAscii("world")).ToString(), Is.EqualTo("*<FF><00><7F><01>world<00>")); Assert.That(subspace.Pack(FdbTuple.Create("hello", 123)).ToString(), Is.EqualTo("*<FF><00><7F><02>hello<00><15>{")); // if we derive a tuple from this subspace, it should keep the binary prefix when converted to a key var t = subspace.Append("world", 123, false); Assert.That(t, Is.Not.Null); Assert.That(t.Count, Is.EqualTo(3)); Assert.That(t.Get<string>(0), Is.EqualTo("world")); Assert.That(t.Get<int>(1), Is.EqualTo(123)); Assert.That(t.Get<bool>(2), Is.False); var k = t.ToSlice(); Assert.That(k.ToString(), Is.EqualTo("*<FF><00><7F><02>world<00><15>{<14>")); // if we unpack the key with the binary prefix, we should get a valid tuple var t2 = subspace.Unpack(k); Assert.That(t2, Is.Not.Null); Assert.That(t2.Count, Is.EqualTo(3)); Assert.That(t2.Get<string>(0), Is.EqualTo("world")); Assert.That(t2.Get<int>(1), Is.EqualTo(123)); Assert.That(t2.Get<bool>(2), Is.False); }
/// <summary>Counts the number of keys inside a directory</summary> public static async Task Count(string[] path, ITuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct) { // look if there is something under there var folder = (await TryOpenCurrentDirectoryAsync(path, db, ct)) as FdbDirectorySubspace; if (folder == null) { log.WriteLine("# Directory {0} does not exist", String.Join("/", path)); return; } var copy = FdbSubspace.Copy(folder); log.WriteLine("# Counting keys under {0} ...", FdbKey.Dump(copy.Key)); var progress = new Progress <STuple <long, Slice> >((state) => { log.Write("\r# Found {0:N0} keys...", state.Item1); }); long count = await Fdb.System.EstimateCountAsync(db, copy.ToRange(), progress, ct); log.WriteLine("\r# Found {0:N0} keys in {1}", count, folder.FullName); }
public static async Task Sampling(string[] path, ITuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct) { double ratio = 0.1d; bool auto = true; if (extras.Count > 0) { double x = extras.Get <double>(0); if (x > 0 && x <= 1) { ratio = x; } auto = false; } var folder = await TryOpenCurrentDirectoryAsync(path, db, ct); KeyRange span; if (folder is FdbDirectorySubspace) { span = KeyRange.StartsWith(FdbSubspace.Copy(folder as FdbDirectorySubspace).Key); log.WriteLine("Reading list of shards for /{0} under {1} ...", String.Join("/", path), FdbKey.Dump(span.Begin)); } else { log.WriteLine("Reading list of shards for the whole cluster ..."); span = KeyRange.All; } // dump keyServers var ranges = await Fdb.System.GetChunksAsync(db, span, ct); log.WriteLine("> Found {0:N0} shard(s)", ranges.Count); // take a sample var samples = new List <KeyRange>(); if (ranges.Count <= 32) { // small enough to scan it all samples.AddRange(ranges); log.WriteLine("Sampling all {0:N0} shards ...", samples.Count); } else { // need to take a random subset var rnd = new Random(); int sz = Math.Max((int)Math.Ceiling(ratio * ranges.Count), 1); if (auto) { if (sz > 100) { sz = 100; //SAFETY } if (sz < 32) { sz = Math.Max(sz, Math.Min(32, ranges.Count)); } } var population = new List <KeyRange>(ranges); for (int i = 0; i < sz; i++) { int p = rnd.Next(population.Count); samples.Add(population[p]); population.RemoveAt(p); } log.WriteLine("Sampling " + samples.Count + " out of " + ranges.Count + " shards (" + (100.0 * samples.Count / ranges.Count).ToString("N1") + "%) ..."); } log.WriteLine(); const string FORMAT_STRING = "{0,9} ║{1,10}{6,6} {2,-29} ║{3,10}{7,7} {4,-37} ║{5,10}"; const string SCALE_KEY = "....--------========########M"; const string SCALE_VAL = "....--------========########@@@@@@@@M"; log.WriteLine(FORMAT_STRING, "Count", "Keys", SCALE_KEY, "Values", SCALE_VAL, "Total", "med.", "med."); var rangeOptions = new FdbRangeOptions { Mode = FdbStreamingMode.WantAll }; samples = samples.OrderBy(x => x.Begin).ToList(); long globalSize = 0; long globalCount = 0; int workers = 8; // Math.Max(4, Environment.ProcessorCount); var sw = Stopwatch.StartNew(); var tasks = new List <Task>(); int n = samples.Count; while (samples.Count > 0) { while (tasks.Count < workers && samples.Count > 0) { var range = samples[0]; samples.RemoveAt(0); tasks.Add(Task.Run(async() => { var kk = new RobustHistogram(RobustHistogram.TimeScale.Ticks); var vv = new RobustHistogram(RobustHistogram.TimeScale.Ticks); #region Method 1: get_range everything... using (var tr = db.BeginTransaction(ct)) { long keySize = 0; long valueSize = 0; long count = 0; int iter = 0; var beginSelector = KeySelector.FirstGreaterOrEqual(range.Begin); var endSelector = KeySelector.FirstGreaterOrEqual(range.End); while (true) { FdbRangeChunk data = default(FdbRangeChunk); FdbException error = null; try { data = await tr.Snapshot.GetRangeAsync( beginSelector, endSelector, rangeOptions, iter ).ConfigureAwait(false); } catch (FdbException e) { error = e; } if (error != null) { await tr.OnErrorAsync(error.Code).ConfigureAwait(false); continue; } if (data.Count == 0) { break; } count += data.Count; foreach (var kvp in data.Chunk) { keySize += kvp.Key.Count; valueSize += kvp.Value.Count; kk.Add(TimeSpan.FromTicks(kvp.Key.Count)); vv.Add(TimeSpan.FromTicks(kvp.Value.Count)); } if (!data.HasMore) { break; } beginSelector = KeySelector.FirstGreaterThan(data.Last.Key); ++iter; } long totalSize = keySize + valueSize; Interlocked.Add(ref globalSize, totalSize); Interlocked.Add(ref globalCount, count); lock (log) { log.WriteLine(FORMAT_STRING, count.ToString("N0"), FormatSize(keySize), kk.GetDistribution(begin: 1, end: 12000, fold: 2), FormatSize(valueSize), vv.GetDistribution(begin: 1, end: 120000, fold: 2), FormatSize(totalSize), FormatSize((int)Math.Ceiling(kk.Median)), FormatSize((int)Math.Ceiling(vv.Median))); } } #endregion #region Method 2: estimate the count using key selectors... //long counter = await Fdb.System.EstimateCountAsync(db, range, ct); //Console.WriteLine("COUNT = " + counter.ToString("N0")); #endregion }, ct)); } var done = await Task.WhenAny(tasks); tasks.Remove(done); } await Task.WhenAll(tasks); sw.Stop(); log.WriteLine(); if (n != ranges.Count) { log.WriteLine("Sampled " + FormatSize(globalSize) + " (" + globalSize.ToString("N0") + " bytes) and " + globalCount.ToString("N0") + " keys in " + sw.Elapsed.TotalSeconds.ToString("N1") + " sec"); log.WriteLine("> Estimated total size is " + FormatSize(globalSize * ranges.Count / n)); } else { log.WriteLine("Found " + FormatSize(globalSize) + " (" + globalSize.ToString("N0") + " bytes) and " + globalCount.ToString("N0") + " keys in " + sw.Elapsed.TotalSeconds.ToString("N1") + " sec"); // compare to the whole cluster ranges = await Fdb.System.GetChunksAsync(db, FdbKey.MinValue, FdbKey.MaxValue, ct); log.WriteLine("> This directory contains ~{0:N2}% of all data", (100.0 * n / ranges.Count)); } log.WriteLine(); }
public static async Task Dir(string[] path, ITuple extras, DirectoryBrowseOptions options, IFdbDatabase db, TextWriter log, CancellationToken ct) { if (log == null) { log = Console.Out; } log.WriteLine("# Listing {0}:", String.Join("/", path)); var parent = await TryOpenCurrentDirectoryAsync(path, db, ct); if (parent == null) { log.WriteLine(" Directory not found."); return; } if (parent.Layer.IsPresent) { log.WriteLine("# Layer: {0}", parent.Layer.ToAsciiOrHexaString()); } var folders = await Fdb.Directory.BrowseAsync(db, parent, ct); if (folders != null && folders.Count > 0) { foreach (var kvp in folders) { var name = kvp.Key; var subfolder = kvp.Value; if (subfolder != null) { if ((options & DirectoryBrowseOptions.ShowCount) != 0) { if (!(subfolder is FdbDirectoryPartition)) { long count = await Fdb.System.EstimateCountAsync(db, subfolder.Keys.ToRange(), ct); log.WriteLine(" {0,-12} {1,-12} {3,9:N0} {2}", FdbKey.Dump(FdbSubspace.Copy(subfolder).Key), subfolder.Layer.IsNullOrEmpty ? "-" : ("<" + subfolder.Layer.ToUnicode() + ">"), name, count); } else { log.WriteLine(" {0,-12} {1,-12} {3,9} {2}", FdbKey.Dump(FdbSubspace.Copy(subfolder).Key), subfolder.Layer.IsNullOrEmpty ? "-" : ("<" + subfolder.Layer.ToUnicode() + ">"), name, "-"); } } else { log.WriteLine(" {0,-12} {1,-12} {2}", FdbKey.Dump(FdbSubspace.Copy(subfolder).Key), subfolder.Layer.IsNullOrEmpty ? "-" : ("<" + subfolder.Layer.ToUnicode() + ">"), name); } } else { log.WriteLine(" WARNING: {0} seems to be missing!", name); } } log.WriteLine(" {0} sub-directorie(s).", folders.Count); } else { //TODO: test if it contains data? log.WriteLine(" No sub-directories."); } }
public void Test_Subspace_With_Tuple_Prefix() { var subspace = new FdbSubspace(FdbTuple.Create("hello")); Assert.That(subspace.Key.ToString(), Is.EqualTo("<02>hello<00>")); Assert.That(subspace.Copy(), Is.Not.SameAs(subspace)); Assert.That(subspace.Copy().Key, Is.EqualTo(subspace.Key)); // concat(Slice) should append the slice to the tuple prefix directly Assert.That(subspace.Concat(Slice.FromInt32(0x01020304)).ToString(), Is.EqualTo("<02>hello<00><04><03><02><01>")); Assert.That(subspace.Concat(Slice.FromAscii("world")).ToString(), Is.EqualTo("<02>hello<00>world")); // pack(...) should use tuple serialization Assert.That(subspace.Pack(123).ToString(), Is.EqualTo("<02>hello<00><15>{")); Assert.That(subspace.Pack("world").ToString(), Is.EqualTo("<02>hello<00><02>world<00>")); // even though the subspace prefix is a tuple, appending to it will only return the new items var t = subspace.Append("world", 123, false); Assert.That(t, Is.Not.Null); Assert.That(t.Count, Is.EqualTo(3)); Assert.That(t.Get<string>(0), Is.EqualTo("world")); Assert.That(t.Get<int>(1), Is.EqualTo(123)); Assert.That(t.Get<bool>(2), Is.False); // but ToSlice() should include the prefix var k = t.ToSlice(); Assert.That(k.ToString(), Is.EqualTo("<02>hello<00><02>world<00><15>{<14>")); // if we unpack the key with the binary prefix, we should get a valid tuple var t2 = subspace.Unpack(k); Assert.That(t2, Is.Not.Null); Assert.That(t2.Count, Is.EqualTo(3)); Assert.That(t2.Get<string>(0), Is.EqualTo("world")); Assert.That(t2.Get<int>(1), Is.EqualTo(123)); Assert.That(t2.Get<bool>(2), Is.False); }