public static void Deconstruct <T1, T2, T3>(this ITuple value, out T1 item1, out T2 item2, out T3 item3)
 {
     value.OfSize(3);
     item1 = value.Get <T1>(0);
     item2 = value.Get <T2>(1);
     item3 = value.Get <T3>(2);
 }
 /// <summary>Returns a typed version of a tuple of size 2</summary>
 /// <typeparam name="T1">Expected type of the first element</typeparam>
 /// <typeparam name="T2">Expected type of the second element</typeparam>
 /// <param name="tuple">Tuple that must be of size 2</param>
 /// <returns>Equivalent tuple, with its elements converted to the specified types</returns>
 public static STuple <T1, T2> As <T1, T2>([NotNull] this ITuple tuple)
 {
     tuple.OfSize(2);
     return(new STuple <T1, T2>(
                tuple.Get <T1>(0),
                tuple.Get <T2>(1)
                ));
 }
 public static void Deconstruct <T1, T2, T3, T4, T5>(this ITuple value, out T1 item1, out T2 item2, out T3 item3, out T4 item4, out T5 item5)
 {
     value.OfSize(5);
     item1 = value.Get <T1>(0);
     item2 = value.Get <T2>(1);
     item3 = value.Get <T3>(2);
     item4 = value.Get <T4>(3);
     item5 = value.Get <T5>(4);
 }
 /// <summary>Returns a typed version of a tuple of size 4</summary>
 /// <typeparam name="T1">Expected type of the first element</typeparam>
 /// <typeparam name="T2">Expected type of the second element</typeparam>
 /// <typeparam name="T3">Expected type of the third element</typeparam>
 /// <typeparam name="T4">Expected type of the fourth element</typeparam>
 /// <param name="tuple">Tuple that must be of size 4</param>
 /// <returns>Equivalent tuple, with its elements converted to the specified types</returns>
 public static STuple <T1, T2, T3, T4> As <T1, T2, T3, T4>([NotNull] this ITuple tuple)
 {
     tuple.OfSize(4);
     return(new STuple <T1, T2, T3, T4>(
                tuple.Get <T1>(0),
                tuple.Get <T2>(1),
                tuple.Get <T3>(2),
                tuple.Get <T4>(3)
                ));
 }
        /// <summary>Creates a new directory</summary>
        public static async Task CreateDirectory(string[] path, ITuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
        {
            if (log == null)
            {
                log = Console.Out;
            }

            string layer = extras.Count > 0 ? extras.Get <string>(0) : null;

            log.WriteLine("# Creating directory {0} with layer '{1}'", String.Join("/", path), layer);

            var folder = await db.Directory.TryOpenAsync(path, cancellationToken : ct);

            if (folder != null)
            {
                log.WriteLine("- Directory {0} already exists!", string.Join("/", path));
                return;
            }

            folder = await db.Directory.TryCreateAsync(path, Slice.FromString(layer), cancellationToken : ct);

            log.WriteLine("- Created under {0} [{1}]", FdbKey.Dump(folder.Key), folder.Key.ToHexaString(' '));

            // look if there is already stuff under there
            var stuff = await db.ReadAsync((tr) => tr.GetRange(folder.Keys.ToRange()).FirstOrDefaultAsync(), cancellationToken : ct);

            if (stuff.Key.IsPresent)
            {
                log.WriteLine("CAUTION: There is already some data under {0} !");
                log.WriteLine("  {0} = {1}", FdbKey.Dump(stuff.Key), stuff.Value.ToAsciiOrHexaString());
            }
        }
        /// <summary>Remove a directory and all its data</summary>
        public static async Task RemoveDirectory(string[] path, ITuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
        {
            if (log == null)
            {
                log = Console.Out;
            }

            string layer = extras.Count > 0 ? extras.Get <string>(0) : null;

            var folder = await db.Directory.TryOpenAsync(path, cancellationToken : ct);

            if (folder == null)
            {
                log.WriteLine("# Directory {0} does not exist", string.Join("/", path));
                return;
            }

            // are there any subdirectories ?
            var subDirs = await folder.TryListAsync(db, ct);

            if (subDirs != null && subDirs.Count > 0)
            {
                //TODO: "-r" flag ?
                log.WriteLine("# Cannot remove {0} because it still contains {1} sub-directorie(s)", string.Join("/", path), subDirs.Count);
            }

            //TODO: ask for confirmation?

            log.WriteLine("# Deleting directory {0} ...", String.Join("/", path));
            await folder.RemoveAsync(db, ct);

            log.WriteLine("# Gone!");
        }
Exemple #7
0
 /// <summary>Returns the typed value of the first item in this tuple</summary>
 /// <typeparam name="T">Expected type of the first item</typeparam>
 /// <returns>Value of the first item, adapted into type <typeparamref name="T"/>.</returns>
 public static T First <T>([NotNull] this ITuple tuple)
 {
     if (tuple == null)
     {
         throw new ArgumentNullException(nameof(tuple));
     }
     return(tuple.Get <T>(0));
 }
 public static void Deconstruct <T1, T2, T3, T4, T5, T6, T7, T8>(this ITuple value, out T1 item1, out T2 item2, out T3 item3, out T4 item4, out T5 item5, out T6 item6, out T7 item7, out T8 item8)
 {
     value.OfSize(8);
     item1 = value.Get <T1>(0);
     item2 = value.Get <T2>(1);
     item3 = value.Get <T3>(2);
     item4 = value.Get <T4>(3);
     item5 = value.Get <T5>(4);
     item6 = value.Get <T6>(5);
     item7 = value.Get <T7>(6);
     item8 = value.Get <T8>(7);
 }
Exemple #9
0
            /// <summary>Compare up to two items in both tuples</summary>
            /// <param name="x">First tuple</param>
            /// <param name="y">Second tuple</param>
            /// <returns>Returns a positive value if x is greater than y, a negative value if x is less than y and 0 if x is equal to y.</returns>
            public int Compare(ITuple x, ITuple y)
            {
                if (y == null)
                {
                    return(x == null ? 0 : +1);
                }
                if (x == null)
                {
                    return(-1);
                }

                int nx = x.Count;
                int ny = y.Count;

                if (ny == 0 || nx == 0)
                {
                    return(nx - ny);
                }

                int p = this.Offset;

                int cmp = this.Comparer1.Compare(x.Get <T1>(p), y.Get <T1>(p));

                if (cmp != 0)
                {
                    return(cmp);
                }

                if (ny == 1 || nx == 1)
                {
                    return(nx - ny);
                }
                cmp = this.Comparer2.Compare(x.Get <T2>(p + 1), y.Get <T2>(p + 1));

                return(cmp);
            }
        public static T[] ToArray <T>([NotNull] this ITuple tuple)
        {
            Contract.NotNull(tuple, nameof(tuple));

            var items = new T[tuple.Count];

            if (items.Length > 0)
            {
                for (int i = 0; i < items.Length; i++)
                {
                    items[i] = tuple.Get <T>(i);
                }
            }
            return(items);
        }
        /// <summary>Shows the first few keys of a directory</summary>
        public static async Task Show(string[] path, ITuple extras, bool reverse, IFdbDatabase db, TextWriter log, CancellationToken ct)
        {
            int count = 20;

            if (extras.Count > 0)
            {
                int x = extras.Get <int>(0);
                if (x > 0)
                {
                    count = x;
                }
            }

            // look if there is something under there
            var folder = await db.Directory.TryOpenAsync(path, cancellationToken : ct);

            if (folder != null)
            {
                log.WriteLine("# Content of {0} [{1}]", FdbKey.Dump(folder.Key), folder.Key.ToHexaString(' '));
                var keys = await db.QueryAsync((tr) =>
                {
                    var query = tr.GetRange(folder.Keys.ToRange());
                    return(reverse
                                                        ? query.Reverse().Take(count)
                                                        : query.Take(count + 1));
                }, cancellationToken : ct);

                if (keys.Count > 0)
                {
                    if (reverse)
                    {
                        keys.Reverse();
                    }
                    foreach (var key in keys.Take(count))
                    {
                        log.WriteLine("...{0} = {1}", FdbKey.Dump(folder.ExtractKey(key.Key)), key.Value.ToAsciiOrHexaString());
                    }
                    if (!reverse && keys.Count == count + 1)
                    {
                        log.WriteLine("... more");
                    }
                }
                else
                {
                    log.WriteLine("  no content found");
                }
            }
        }
Exemple #12
0
        public static T[] ToArray <T>([NotNull] this ITuple tuple)
        {
            if (tuple == null)
            {
                throw new ArgumentNullException(nameof(tuple));
            }

            var items = new T[tuple.Count];

            if (items.Length > 0)
            {
                for (int i = 0; i < items.Length; i++)
                {
                    items[i] = tuple.Get <T>(i);
                }
            }
            return(items);
        }
            /// <summary>Compare a single item in both tuples</summary>
            /// <param name="x">First tuple</param>
            /// <param name="y">Second tuple</param>
            /// <returns>Returns a positive value if x is greater than y, a negative value if x is less than y and 0 if x is equal to y.</returns>
            public int Compare(ITuple x, ITuple y)
            {
                if (y == null)
                {
                    return(x == null ? 0 : +1);
                }
                if (x == null)
                {
                    return(-1);
                }

                int nx = x.Count;
                int ny = y.Count;

                if (ny == 0 || nx == 0)
                {
                    return(nx - ny);
                }

                int p = this.Offset;

                return(this.Comparer.Compare(x.Get <T1>(p), y.Get <T1>(p)));
            }
 /// <summary>Execute a lambda Action with the content of this tuple</summary>
 /// <param name="tuple">Tuple of size 1</param>
 /// <param name="lambda">Action that will be passed the content of this tuple as parameters</param>
 /// <exception cref="InvalidOperationException">If <paramref name="tuple"/> has not the expected size</exception>
 public static void With <T1>([NotNull] this ITuple tuple, [NotNull] Action <T1> lambda)
 {
     OfSize(tuple, 1);
     lambda(tuple.Get <T1>(0));
 }
 /// <summary>Execute a lambda Function with the content of this tuple</summary>
 /// <param name="tuple">Tuple of size 8</param>
 /// <param name="lambda">Function that will be passed the content of this tuple as parameters</param>
 /// <returns>Result of calling <paramref name="lambda"/> with the items of this tuple</returns>
 /// <exception cref="InvalidOperationException">If <paramref name="tuple"/> has not the expected size</exception>
 public static TResult With <T1, T2, T3, T4, T5, T6, T7, T8, TResult>([NotNull] this ITuple tuple, [NotNull] Func <T1, T2, T3, T4, T5, T6, T7, T8, TResult> lambda)
 {
     OfSize(tuple, 8);
     return(lambda(tuple.Get <T1>(0), tuple.Get <T2>(1), tuple.Get <T3>(2), tuple.Get <T4>(3), tuple.Get <T5>(4), tuple.Get <T6>(5), tuple.Get <T7>(6), tuple.Get <T8>(7)));
 }
        public static async Task Sampling(string[] path, ITuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
        {
            double ratio = 0.1d;
            bool   auto  = true;

            if (extras.Count > 0)
            {
                double x = extras.Get <double>(0);
                if (x > 0 && x <= 1)
                {
                    ratio = x;
                }
                auto = false;
            }

            var folder = await TryOpenCurrentDirectoryAsync(path, db, ct);

            KeyRange span;

            if (folder is FdbDirectorySubspace)
            {
                span = KeyRange.StartsWith(FdbSubspace.Copy(folder as FdbDirectorySubspace).Key);
                log.WriteLine("Reading list of shards for /{0} under {1} ...", String.Join("/", path), FdbKey.Dump(span.Begin));
            }
            else
            {
                log.WriteLine("Reading list of shards for the whole cluster ...");
                span = KeyRange.All;
            }

            // dump keyServers
            var ranges = await Fdb.System.GetChunksAsync(db, span, ct);

            log.WriteLine("> Found {0:N0} shard(s)", ranges.Count);

            // take a sample
            var samples = new List <KeyRange>();

            if (ranges.Count <= 32)
            {             // small enough to scan it all
                samples.AddRange(ranges);
                log.WriteLine("Sampling all {0:N0} shards ...", samples.Count);
            }
            else
            {             // need to take a random subset
                var rnd = new Random();
                int sz  = Math.Max((int)Math.Ceiling(ratio * ranges.Count), 1);
                if (auto)
                {
                    if (sz > 100)
                    {
                        sz = 100;                               //SAFETY
                    }
                    if (sz < 32)
                    {
                        sz = Math.Max(sz, Math.Min(32, ranges.Count));
                    }
                }

                var population = new List <KeyRange>(ranges);
                for (int i = 0; i < sz; i++)
                {
                    int p = rnd.Next(population.Count);
                    samples.Add(population[p]);
                    population.RemoveAt(p);
                }
                log.WriteLine("Sampling " + samples.Count + " out of " + ranges.Count + " shards (" + (100.0 * samples.Count / ranges.Count).ToString("N1") + "%) ...");
            }

            log.WriteLine();
            const string FORMAT_STRING = "{0,9} ║{1,10}{6,6} {2,-29} ║{3,10}{7,7} {4,-37} ║{5,10}";
            const string SCALE_KEY     = "....--------========########M";
            const string SCALE_VAL     = "....--------========########@@@@@@@@M";

            log.WriteLine(FORMAT_STRING, "Count", "Keys", SCALE_KEY, "Values", SCALE_VAL, "Total", "med.", "med.");

            var rangeOptions = new FdbRangeOptions {
                Mode = FdbStreamingMode.WantAll
            };

            samples = samples.OrderBy(x => x.Begin).ToList();

            long globalSize  = 0;
            long globalCount = 0;
            int  workers     = 8;        // Math.Max(4, Environment.ProcessorCount);

            var sw    = Stopwatch.StartNew();
            var tasks = new List <Task>();
            int n     = samples.Count;

            while (samples.Count > 0)
            {
                while (tasks.Count < workers && samples.Count > 0)
                {
                    var range = samples[0];
                    samples.RemoveAt(0);
                    tasks.Add(Task.Run(async() =>
                    {
                        var kk = new RobustHistogram(RobustHistogram.TimeScale.Ticks);
                        var vv = new RobustHistogram(RobustHistogram.TimeScale.Ticks);

                        #region Method 1: get_range everything...

                        using (var tr = db.BeginTransaction(ct))
                        {
                            long keySize   = 0;
                            long valueSize = 0;
                            long count     = 0;

                            int iter          = 0;
                            var beginSelector = KeySelector.FirstGreaterOrEqual(range.Begin);
                            var endSelector   = KeySelector.FirstGreaterOrEqual(range.End);
                            while (true)
                            {
                                FdbRangeChunk data = default(FdbRangeChunk);
                                FdbException error = null;
                                try
                                {
                                    data = await tr.Snapshot.GetRangeAsync(
                                        beginSelector,
                                        endSelector,
                                        rangeOptions,
                                        iter
                                        ).ConfigureAwait(false);
                                }
                                catch (FdbException e)
                                {
                                    error = e;
                                }

                                if (error != null)
                                {
                                    await tr.OnErrorAsync(error.Code).ConfigureAwait(false);
                                    continue;
                                }

                                if (data.Count == 0)
                                {
                                    break;
                                }

                                count += data.Count;
                                foreach (var kvp in data.Chunk)
                                {
                                    keySize   += kvp.Key.Count;
                                    valueSize += kvp.Value.Count;

                                    kk.Add(TimeSpan.FromTicks(kvp.Key.Count));
                                    vv.Add(TimeSpan.FromTicks(kvp.Value.Count));
                                }

                                if (!data.HasMore)
                                {
                                    break;
                                }

                                beginSelector = KeySelector.FirstGreaterThan(data.Last.Key);
                                ++iter;
                            }

                            long totalSize = keySize + valueSize;
                            Interlocked.Add(ref globalSize, totalSize);
                            Interlocked.Add(ref globalCount, count);

                            lock (log)
                            {
                                log.WriteLine(FORMAT_STRING, count.ToString("N0"), FormatSize(keySize), kk.GetDistribution(begin: 1, end: 12000, fold: 2), FormatSize(valueSize), vv.GetDistribution(begin: 1, end: 120000, fold: 2), FormatSize(totalSize), FormatSize((int)Math.Ceiling(kk.Median)), FormatSize((int)Math.Ceiling(vv.Median)));
                            }
                        }
                        #endregion

                        #region Method 2: estimate the count using key selectors...

                        //long counter = await Fdb.System.EstimateCountAsync(db, range, ct);
                        //Console.WriteLine("COUNT = " + counter.ToString("N0"));

                        #endregion
                    }, ct));
                }

                var done = await Task.WhenAny(tasks);

                tasks.Remove(done);
            }

            await Task.WhenAll(tasks);

            sw.Stop();

            log.WriteLine();
            if (n != ranges.Count)
            {
                log.WriteLine("Sampled " + FormatSize(globalSize) + " (" + globalSize.ToString("N0") + " bytes) and " + globalCount.ToString("N0") + " keys in " + sw.Elapsed.TotalSeconds.ToString("N1") + " sec");
                log.WriteLine("> Estimated total size is " + FormatSize(globalSize * ranges.Count / n));
            }
            else
            {
                log.WriteLine("Found " + FormatSize(globalSize) + " (" + globalSize.ToString("N0") + " bytes) and " + globalCount.ToString("N0") + " keys in " + sw.Elapsed.TotalSeconds.ToString("N1") + " sec");
                // compare to the whole cluster
                ranges = await Fdb.System.GetChunksAsync(db, FdbKey.MinValue, FdbKey.MaxValue, ct);

                log.WriteLine("> This directory contains ~{0:N2}% of all data", (100.0 * n / ranges.Count));
            }
            log.WriteLine();
        }
 /// <summary>Execute a lambda Function with the content of this tuple</summary>
 /// <param name="tuple">Tuple of size 2</param>
 /// <param name="lambda">Function that will be passed the content of this tuple as parameters</param>
 /// <returns>Result of calling <paramref name="lambda"/> with the items of this tuple</returns>
 /// <exception cref="InvalidOperationException">If <paramref name="tuple"/> has not the expected size</exception>
 public static TResult With <T1, T2, TResult>([NotNull] this ITuple tuple, [NotNull] Func <T1, T2, TResult> lambda)
 {
     OfSize(tuple, 2);
     return(lambda(tuple.Get <T1>(0), tuple.Get <T2>(1)));
 }
 /// <summary>Execute a lambda Function with the content of this tuple</summary>
 /// <param name="tuple">Tuple of size 5</param>
 /// <param name="lambda">Function that will be passed the content of this tuple as parameters</param>
 /// <returns>Result of calling <paramref name="lambda"/> with the items of this tuple</returns>
 /// <exception cref="InvalidOperationException">If <paramref name="tuple"/> has not the expected size</exception>
 public static TResult With <T1, T2, T3, T4, T5, TResult>([NotNull] this ITuple tuple, [NotNull] Func <T1, T2, T3, T4, T5, TResult> lambda)
 {
     OfSize(tuple, 5);
     return(lambda(tuple.Get <T1>(0), tuple.Get <T2>(1), tuple.Get <T3>(2), tuple.Get <T4>(3), tuple.Get <T5>(4)));
 }
 /// <summary>Execute a lambda Action with the content of this tuple</summary>
 /// <param name="tuple">Tuple of size 8</param>
 /// <param name="lambda">Action that will be passed the content of this tuple as parameters</param>
 /// <exception cref="InvalidOperationException">If <paramref name="tuple"/> has not the expected size</exception>
 public static void With <T1, T2, T3, T4, T5, T6, T7, T8>([NotNull] this ITuple tuple, [NotNull] Action <T1, T2, T3, T4, T5, T6, T7, T8> lambda)
 {
     OfSize(tuple, 8);
     lambda(tuple.Get <T1>(0), tuple.Get <T2>(1), tuple.Get <T3>(2), tuple.Get <T4>(3), tuple.Get <T5>(4), tuple.Get <T6>(5), tuple.Get <T7>(6), tuple.Get <T8>(7));
 }
 /// <summary>Execute a lambda Action with the content of this tuple</summary>
 /// <param name="tuple">Tuple of size 5</param>
 /// <param name="lambda">Action that will be passed the content of this tuple as parameters</param>
 /// <exception cref="InvalidOperationException">If <paramref name="tuple"/> has not the expected size</exception>
 public static void With <T1, T2, T3, T4, T5>([NotNull] this ITuple tuple, [NotNull] Action <T1, T2, T3, T4, T5> lambda)
 {
     OfSize(tuple, 5);
     lambda(tuple.Get <T1>(0), tuple.Get <T2>(1), tuple.Get <T3>(2), tuple.Get <T4>(3), tuple.Get <T5>(4));
 }
 /// <summary>Execute a lambda Action with the content of this tuple</summary>
 /// <param name="tuple">Tuple of size 3</param>
 /// <param name="lambda">Action that will be passed the content of this tuple as parameters</param>
 /// <exception cref="InvalidOperationException">If <paramref name="tuple"/> has not the expected size</exception>
 public static void With <T1, T2, T3>([NotNull] this ITuple tuple, [NotNull] Action <T1, T2, T3> lambda)
 {
     OfSize(tuple, 3);
     lambda(tuple.Get <T1>(0), tuple.Get <T2>(1), tuple.Get <T3>(2));
 }
 public T Get <T>(int index)
 {
     return(m_items.Get <T>(index));
 }
 public static T Last <T>([NotNull] this ITuple tuple)
 {
     return(tuple.Get <T>(-1));
 }
 public static T First <T>([NotNull] this ITuple tuple)
 {
     return(tuple.Get <T>(0));
 }
Exemple #25
0
 public R Get <R>(int index)
 {
     return(m_items.Get <R>(index));
 }