public async Task <long?> Rank(IFdbReadOnlyTransaction trans, Slice key)
            {
                if (trans == null)
                {
                    throw new ArgumentNullException(nameof(trans));
                }
                if (key.IsNull)
                {
                    throw new ArgumentException("Empty key not allowed in set", nameof(key));
                }

                if (!(await ContainsAsync(trans, key).ConfigureAwait(false)))
                {
                    return(default(long?));
                }

                long r       = 0;
                var  rankKey = Slice.Empty;

                for (int level = MAX_LEVELS - 1; level >= 0; level--)
                {
                    var  lss       = this.Subspace.Partition.ByKey(level);
                    long lastCount = 0;
                    var  kcs       = await trans.GetRange(
                        KeySelector.FirstGreaterOrEqual(lss.Encode(rankKey)),
                        KeySelector.FirstGreaterThan(lss.Encode(key))
                        ).ToListAsync().ConfigureAwait(false);

                    foreach (var kc in kcs)
                    {
                        rankKey   = lss.Decode <Slice>(kc.Key);
                        lastCount = DecodeCount(kc.Value);
                        r        += lastCount;
                    }
                    r -= lastCount;
                    if (rankKey == key)
                    {
                        break;
                    }
                }
                return(r);
            }
        public FdbRangeQuery <TId> LookupLessThan([NotNull] IFdbReadOnlyTransaction trans, TValue value, bool orEqual, bool reverse = false)
        {
            var prefix = this.Subspace.Keys.EncodePartial(value);

            if (orEqual)
            {
                prefix = FdbKey.Increment(prefix);
            }

            var space = new KeySelectorPair(
                KeySelector.FirstGreaterOrEqual(this.Subspace.ToRange().Begin),
                KeySelector.FirstGreaterThan(prefix)
                );

            return(trans
                   .GetRange(space, new FdbRangeOptions {
                Reverse = reverse
            })
                   .Select((kvp) => this.Subspace.Keys.Decode(kvp.Key).Item2));
        }
        public static async Task Sampling(string[] path, IVarTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
        {
            double ratio = 0.1d;
            bool   auto  = true;

            if (extras.Count > 0)
            {
                double x = extras.Get <double>(0);
                if (x > 0 && x <= 1)
                {
                    ratio = x;
                }
                auto = false;
            }

            var folder = await TryOpenCurrentDirectoryAsync(path, db, ct);

            KeyRange span;

            if (folder is FdbDirectorySubspace)
            {
                span = KeyRange.StartsWith((folder as FdbDirectorySubspace).Copy().GetPrefix());
                log.WriteLine($"Reading list of shards for /{String.Join("/", path)} under {FdbKey.Dump(span.Begin)} ...");
            }
            else
            {
                log.WriteLine("Reading list of shards for the whole cluster ...");
                span = KeyRange.All;
            }

            // dump keyServers
            var ranges = await Fdb.System.GetChunksAsync(db, span, ct);

            log.WriteLine($"> Found {ranges.Count:N0} shard(s)");

            // take a sample
            var samples = new List <KeyRange>();

            if (ranges.Count <= 32)
            {             // small enough to scan it all
                samples.AddRange(ranges);
                log.WriteLine($"Sampling all {samples.Count:N0} shards ...");
            }
            else
            {             // need to take a random subset
                var rnd = new Random();
                int sz  = Math.Max((int)Math.Ceiling(ratio * ranges.Count), 1);
                if (auto)
                {
                    if (sz > 100)
                    {
                        sz = 100;                               //SAFETY
                    }
                    if (sz < 32)
                    {
                        sz = Math.Max(sz, Math.Min(32, ranges.Count));
                    }
                }

                var population = new List <KeyRange>(ranges);
                for (int i = 0; i < sz; i++)
                {
                    int p = rnd.Next(population.Count);
                    samples.Add(population[p]);
                    population.RemoveAt(p);
                }
                log.WriteLine($"Sampling {samples.Count:N0} out of {ranges.Count:N0} shards ({(100.0 * samples.Count / ranges.Count):N1}%) ...");
            }

            log.WriteLine();
            const string FORMAT_STRING = "{0,9} ║{1,10}{6,6} {2,-29} ║{3,10}{7,7} {4,-37} ║{5,10}";
            const string SCALE_KEY     = "....--------========########M";
            const string SCALE_VAL     = "....--------========########@@@@@@@@M";

            log.WriteLine(FORMAT_STRING, "Count", "Keys", SCALE_KEY, "Values", SCALE_VAL, "Total", "med.", "med.");

            var rangeOptions = new FdbRangeOptions {
                Mode = FdbStreamingMode.WantAll
            };

            samples = samples.OrderBy(x => x.Begin).ToList();

            long globalSize  = 0;
            long globalCount = 0;
            int  workers     = 8;        // Math.Max(4, Environment.ProcessorCount);

            var sw    = Stopwatch.StartNew();
            var tasks = new List <Task>();
            int n     = samples.Count;

            while (samples.Count > 0)
            {
                while (tasks.Count < workers && samples.Count > 0)
                {
                    var range = samples[0];
                    samples.RemoveAt(0);
                    tasks.Add(Task.Run(async() =>
                    {
                        var kk = new RobustHistogram(RobustHistogram.TimeScale.Ticks);
                        var vv = new RobustHistogram(RobustHistogram.TimeScale.Ticks);

                        #region Method 1: get_range everything...

                        using (var tr = db.BeginTransaction(ct))
                        {
                            long keySize   = 0;
                            long valueSize = 0;
                            long count     = 0;

                            int iter          = 0;
                            var beginSelector = KeySelector.FirstGreaterOrEqual(range.Begin);
                            var endSelector   = KeySelector.FirstGreaterOrEqual(range.End);
                            while (true)
                            {
                                FdbRangeChunk data = default(FdbRangeChunk);
                                FdbException error = null;
                                try
                                {
                                    data = await tr.Snapshot.GetRangeAsync(
                                        beginSelector,
                                        endSelector,
                                        rangeOptions,
                                        iter
                                        ).ConfigureAwait(false);
                                }
                                catch (FdbException e)
                                {
                                    error = e;
                                }

                                if (error != null)
                                {
                                    await tr.OnErrorAsync(error.Code).ConfigureAwait(false);
                                    continue;
                                }

                                if (data.Count == 0)
                                {
                                    break;
                                }

                                count += data.Count;
                                foreach (var kvp in data)
                                {
                                    keySize   += kvp.Key.Count;
                                    valueSize += kvp.Value.Count;

                                    kk.Add(TimeSpan.FromTicks(kvp.Key.Count));
                                    vv.Add(TimeSpan.FromTicks(kvp.Value.Count));
                                }

                                if (!data.HasMore)
                                {
                                    break;
                                }

                                beginSelector = KeySelector.FirstGreaterThan(data.Last);
                                ++iter;
                            }

                            long totalSize = keySize + valueSize;
                            Interlocked.Add(ref globalSize, totalSize);
                            Interlocked.Add(ref globalCount, count);

                            lock (log)
                            {
                                log.WriteLine(FORMAT_STRING, count.ToString("N0"), FormatSize(keySize), kk.GetDistribution(begin: 1, end: 12000, fold: 2), FormatSize(valueSize), vv.GetDistribution(begin: 1, end: 120000, fold: 2), FormatSize(totalSize), FormatSize((int)Math.Ceiling(kk.Median)), FormatSize((int)Math.Ceiling(vv.Median)));
                            }
                        }
                        #endregion

                        #region Method 2: estimate the count using key selectors...

                        //long counter = await Fdb.System.EstimateCountAsync(db, range, ct);
                        //Console.WriteLine("COUNT = " + counter.ToString("N0"));

                        #endregion
                    }, ct));
                }

                var done = await Task.WhenAny(tasks);

                tasks.Remove(done);
            }

            await Task.WhenAll(tasks);

            sw.Stop();

            log.WriteLine();
            if (n != ranges.Count)
            {
                log.WriteLine($"Sampled {FormatSize(globalSize)} ({globalSize:N0} bytes) and {globalCount:N0} keys in {sw.Elapsed.TotalSeconds:N1} sec");
                log.WriteLine($"> Estimated total size is {FormatSize(globalSize * ranges.Count / n)}");
            }
            else
            {
                log.WriteLine($"Found {FormatSize(globalSize)} ({globalSize:N0} bytes) and {globalCount:N0} keys in {sw.Elapsed.TotalSeconds:N1} sec");
                // compare to the whole cluster
                ranges = await Fdb.System.GetChunksAsync(db, FdbKey.MinValue, FdbKey.MaxValue, ct);

                log.WriteLine($"> This directory contains ~{(100.0 * n / ranges.Count):N2}% of all data");
            }
            log.WriteLine();
        }
        public async void Test_Case_11()
        {
            using (var zedb = await OpenTestDatabaseAsync())
            {
                var db = FoundationDB.Filters.Logging.FdbLoggingExtensions.Logged(zedb, (tr) => Log(tr.Log.GetTimingsReport(true)));
                {
                    var subspace = db.GlobalSpace;

                    // clear everything and write some values
                    await db.WriteAsync((tr) =>
                    {
                        tr.ClearRange(subspace.Keys.Encode("K0000"), subspace.Keys.Encode("K9999Z"));
                        for (int i = 0; i < 100; i++)
                        {
                            tr.Set(subspace.Keys.Encode("K" + i.ToString("D4")), Slice.FromString("V" + i.ToString("D4")));
                        }
                    }, this.Cancellation);

                    using (var tr = db.BeginTransaction(this.Cancellation))
                    {
                        tr.ClearRange(subspace.Keys.Encode("K0010"), subspace.Keys.Encode("K0020"));
                        tr.ClearRange(subspace.Keys.Encode("K0050"), subspace.Keys.Encode("K0060"));
                        tr.Set(subspace.Keys.Encode("K0021"), Slice.Empty);
                        tr.Set(subspace.Keys.Encode("K0042"), Slice.Empty);

                        await tr.GetKeyAsync(KeySelector.FirstGreaterOrEqual(subspace.Keys.Encode("K0005")));

                        await tr.GetKeyAsync(KeySelector.FirstGreaterOrEqual(subspace.Keys.Encode("K0010")));

                        await tr.GetKeyAsync(KeySelector.FirstGreaterOrEqual(subspace.Keys.Encode("K0015")));

                        await tr.GetKeyAsync(KeySelector.FirstGreaterOrEqual(subspace.Keys.Encode("K0022")));

                        await tr.GetKeyAsync(KeySelector.FirstGreaterOrEqual(subspace.Keys.Encode("K0049")));

                        await tr.GetKeyAsync(KeySelector.FirstGreaterOrEqual(subspace.Keys.Encode("K0050")));

                        await tr.GetKeyAsync(KeySelector.FirstGreaterOrEqual(subspace.Keys.Encode("K0055")));

                        await tr.GetKeyAsync(KeySelector.FirstGreaterOrEqual(subspace.Keys.Encode("K0061")));

                        //no commit
                    }

                    using (var tr = db.BeginTransaction(this.Cancellation))
                    {
                        //tr.SetOption(FdbTransactionOption.ReadYourWritesDisable);
                        await tr.GetKeyAsync(KeySelector.FirstGreaterOrEqual(subspace.Keys.Encode("K0000")));                         // equal=false, offset=1

                        await tr.GetKeyAsync(KeySelector.FirstGreaterThan(subspace.Keys.Encode("K0011")));                            // equal=true, offset=1

                        await tr.GetKeyAsync(KeySelector.LastLessOrEqual(subspace.Keys.Encode("K0022")));                             // equal=true, offset=0

                        await tr.GetKeyAsync(KeySelector.LastLessThan(subspace.Keys.Encode("K0033")));                                // equal=false, offset=0

                        await tr.GetKeyAsync(KeySelector.FirstGreaterOrEqual(subspace.Keys.Encode("K0040")) + 1000);                  // equal=false, offset=7 ?

                        await tr.GetKeyAsync(KeySelector.LastLessThan(subspace.Keys.Encode("K0050")) + 1000);                         // equal=false, offset=6 ?
                    }
                }
            }
        }
        public async Task Run(IFdbDatabase db, TextWriter log, CancellationToken ct)
        {
            // estimate the number of machines...
            Console.WriteLine("# Detecting cluster topology...");
            var servers = await db.QueryAsync(tr => tr
                                              .WithReadAccessToSystemKeys()
                                              .GetRange(KeyRange.StartsWith(Fdb.System.ServerList))
                                              .Select(kvp => new
            {
                Node       = kvp.Value.Substring(8, 16).ToHexaString(),
                Machine    = kvp.Value.Substring(24, 16).ToHexaString(),
                DataCenter = kvp.Value.Substring(40, 16).ToHexaString()
            }),
                                              ct
                                              );

            var numNodes    = servers.Select(s => s.Node).Distinct().Count();
            var numMachines = servers.Select(s => s.Machine).Distinct().Count();
            var numDCs      = servers.Select(s => s.DataCenter).Distinct().Count();

            Console.WriteLine("# > Found " + numNodes + " process(es) on " + numMachines + " machine(s) in " + numDCs + " datacenter(s)");
            Console.WriteLine("# Reading list of shards...");
            // dump keyServers
            var ranges = await Fdb.System.GetChunksAsync(db, FdbKey.MinValue, FdbKey.MaxValue, ct);

            Console.WriteLine("# > Found " + ranges.Count + " shards:");

            // take a sample
            var rnd = new Random(1234);
            int sz  = Math.Max((int)Math.Ceiling(this.Ratio * ranges.Count), 1);

            if (sz > 500)
            {
                sz = 500;                       //SAFETY
            }
            if (sz < 50)
            {
                sz = Math.Max(sz, Math.Min(50, ranges.Count));
            }

            var samples = new List <KeyRange>();

            for (int i = 0; i < sz; i++)
            {
                int p = rnd.Next(ranges.Count);
                samples.Add(ranges[p]);
                ranges.RemoveAt(p);
            }

            Console.WriteLine("# Sampling " + sz + " out of " + ranges.Count + " shards (" + (100.0 * sz / ranges.Count).ToString("N1") + "%) ...");
            Console.WriteLine("{0,9}{1,10}{2,10}{3,10} : K+V size distribution", "Count", "Keys", "Values", "Total");

            var rangeOptions = new FdbRangeOptions {
                Mode = FdbStreamingMode.WantAll
            };

            samples = samples.OrderBy(x => x.Begin).ToList();

            long total   = 0;
            int  workers = Math.Min(numMachines, 8);

            var sw    = Stopwatch.StartNew();
            var tasks = new List <Task>();

            while (samples.Count > 0)
            {
                while (tasks.Count < workers && samples.Count > 0)
                {
                    var range = samples[0];
                    samples.RemoveAt(0);
                    tasks.Add(Task.Run(async() =>
                    {
                        var hh = new RobustHistogram(RobustHistogram.TimeScale.Ticks);

                        #region Method 1: get_range everything...

                        using (var tr = await db.BeginTransactionAsync(ct))
                        {
                            long keySize   = 0;
                            long valueSize = 0;
                            long count     = 0;

                            int iter          = 0;
                            var beginSelector = KeySelector.FirstGreaterOrEqual(range.Begin);
                            var endSelector   = KeySelector.FirstGreaterOrEqual(range.End);
                            while (true)
                            {
                                FdbRangeChunk data = default(FdbRangeChunk);
                                FdbException error = null;
                                try
                                {
                                    data = await tr.Snapshot.GetRangeAsync(
                                        beginSelector,
                                        endSelector,
                                        rangeOptions,
                                        iter
                                        ).ConfigureAwait(false);
                                }
                                catch (FdbException e)
                                {
                                    error = e;
                                }

                                if (error != null)
                                {
                                    await tr.OnErrorAsync(error.Code).ConfigureAwait(false);
                                    continue;
                                }

                                if (data.Count == 0)
                                {
                                    break;
                                }

                                count += data.Count;
                                foreach (var kvp in data)
                                {
                                    keySize   += kvp.Key.Count;
                                    valueSize += kvp.Value.Count;

                                    hh.Add(TimeSpan.FromTicks(kvp.Key.Count + kvp.Value.Count));
                                }

                                if (!data.HasMore)
                                {
                                    break;
                                }

                                beginSelector = KeySelector.FirstGreaterThan(data.Last);
                                ++iter;
                            }

                            long totalSize = keySize + valueSize;
                            Interlocked.Add(ref total, totalSize);

                            Console.WriteLine("{0,9}{1,10}{2,10}{3,10} : {4}", count.ToString("N0"), FormatSize(keySize), FormatSize(valueSize), FormatSize(totalSize), hh.GetDistribution(begin: 1, end: 10000, fold: 2));
                        }
                        #endregion

                        #region Method 2: estimate the count using key selectors...

                        //long counter = await Fdb.System.EstimateCountAsync(db, range, ct);
                        //Console.WriteLine("COUNT = " + counter.ToString("N0"));

                        #endregion
                    }, ct));
                }

                var done = await Task.WhenAny(tasks);

                tasks.Remove(done);
            }

            await Task.WhenAll(tasks);

            sw.Stop();

            Console.WriteLine("> Sampled " + FormatSize(total) + " (" + total.ToString("N0") + " bytes) in " + sw.Elapsed.TotalSeconds.ToString("N1") + " sec");
            Console.WriteLine("> Estimated total size is " + FormatSize(total * ranges.Count / sz));
        }
        public async Task Test_Case_11()
        {
            using (var db = await OpenTestPartitionAsync())
            {
                db.SetDefaultLogHandler((log) => Log(log.GetTimingsReport(true)));

                var location = db.Root;

                // clear everything and write some values
                await db.WriteAsync(async tr =>
                {
                    var subspace = await location.Resolve(tr);
                    tr.ClearRange(subspace.Encode("K0000"), subspace.Encode("K9999Z"));
                    for (int i = 0; i < 100; i++)
                    {
                        tr.Set(subspace.Encode("K" + i.ToString("D4")), Value("V" + i.ToString("D4")));
                    }
                }, this.Cancellation);

                using (var tr = await db.BeginTransactionAsync(this.Cancellation))
                {
                    var subspace = await location.Resolve(tr);

                    tr.ClearRange(subspace.Encode("K0010"), subspace.Encode("K0020"));
                    tr.ClearRange(subspace.Encode("K0050"), subspace.Encode("K0060"));
                    tr.Set(subspace.Encode("K0021"), Slice.Empty);
                    tr.Set(subspace.Encode("K0042"), Slice.Empty);

                    await tr.GetKeyAsync(KeySelector.FirstGreaterOrEqual(subspace.Encode("K0005")));

                    await tr.GetKeyAsync(KeySelector.FirstGreaterOrEqual(subspace.Encode("K0010")));

                    await tr.GetKeyAsync(KeySelector.FirstGreaterOrEqual(subspace.Encode("K0015")));

                    await tr.GetKeyAsync(KeySelector.FirstGreaterOrEqual(subspace.Encode("K0022")));

                    await tr.GetKeyAsync(KeySelector.FirstGreaterOrEqual(subspace.Encode("K0049")));

                    await tr.GetKeyAsync(KeySelector.FirstGreaterOrEqual(subspace.Encode("K0050")));

                    await tr.GetKeyAsync(KeySelector.FirstGreaterOrEqual(subspace.Encode("K0055")));

                    await tr.GetKeyAsync(KeySelector.FirstGreaterOrEqual(subspace.Encode("K0061")));

                    //no commit
                }

                using (var tr = await db.BeginTransactionAsync(this.Cancellation))
                {
                    var subspace = await location.Resolve(tr);

                    //tr.SetOption(FdbTransactionOption.ReadYourWritesDisable);
                    await tr.GetKeyAsync(KeySelector.FirstGreaterOrEqual(subspace.Encode("K0000")));                     // equal=false, offset=1

                    await tr.GetKeyAsync(KeySelector.FirstGreaterThan(subspace.Encode("K0011")));                        // equal=true, offset=1

                    await tr.GetKeyAsync(KeySelector.LastLessOrEqual(subspace.Encode("K0022")));                         // equal=true, offset=0

                    await tr.GetKeyAsync(KeySelector.LastLessThan(subspace.Encode("K0033")));                            // equal=false, offset=0

                    await tr.GetKeyAsync(KeySelector.FirstGreaterOrEqual(subspace.Encode("K0040")) + 1000);              // equal=false, offset=7 ?

                    await tr.GetKeyAsync(KeySelector.LastLessThan(subspace.Encode("K0050")) + 1000);                     // equal=false, offset=6 ?
                }
            }
        }