public static void Deconstruct <T1, T2, T3>(this ITuple value, out T1 item1, out T2 item2, out T3 item3)
{
value.OfSize(3);
item1 = value.Get <T1>(0);
item2 = value.Get <T2>(1);
item3 = value.Get <T3>(2);
}
/// <summary>Returns a typed version of a tuple of size 2</summary>
/// <typeparam name="T1">Expected type of the first element</typeparam>
/// <typeparam name="T2">Expected type of the second element</typeparam>
/// <param name="tuple">Tuple that must be of size 2</param>
/// <returns>Equivalent tuple, with its elements converted to the specified types</returns>
public static STuple <T1, T2> As <T1, T2>([NotNull] this ITuple tuple)
{
tuple.OfSize(2);
return(new STuple <T1, T2>(
tuple.Get <T1>(0),
tuple.Get <T2>(1)
));
}
public static void Deconstruct <T1, T2, T3, T4, T5>(this ITuple value, out T1 item1, out T2 item2, out T3 item3, out T4 item4, out T5 item5)
{
value.OfSize(5);
item1 = value.Get <T1>(0);
item2 = value.Get <T2>(1);
item3 = value.Get <T3>(2);
item4 = value.Get <T4>(3);
item5 = value.Get <T5>(4);
}
/// <summary>Returns a typed version of a tuple of size 4</summary>
/// <typeparam name="T1">Expected type of the first element</typeparam>
/// <typeparam name="T2">Expected type of the second element</typeparam>
/// <typeparam name="T3">Expected type of the third element</typeparam>
/// <typeparam name="T4">Expected type of the fourth element</typeparam>
/// <param name="tuple">Tuple that must be of size 4</param>
/// <returns>Equivalent tuple, with its elements converted to the specified types</returns>
public static STuple <T1, T2, T3, T4> As <T1, T2, T3, T4>([NotNull] this ITuple tuple)
{
tuple.OfSize(4);
return(new STuple <T1, T2, T3, T4>(
tuple.Get <T1>(0),
tuple.Get <T2>(1),
tuple.Get <T3>(2),
tuple.Get <T4>(3)
));
}
/// <summary>Creates a new directory</summary>
public static async Task CreateDirectory(string[] path, ITuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
if (log == null)
{
log = Console.Out;
}
string layer = extras.Count > 0 ? extras.Get <string>(0) : null;
log.WriteLine("# Creating directory {0} with layer '{1}'", String.Join("/", path), layer);
var folder = await db.Directory.TryOpenAsync(path, cancellationToken : ct);
if (folder != null)
{
log.WriteLine("- Directory {0} already exists!", string.Join("/", path));
return;
}
folder = await db.Directory.TryCreateAsync(path, Slice.FromString(layer), cancellationToken : ct);
log.WriteLine("- Created under {0} [{1}]", FdbKey.Dump(folder.Key), folder.Key.ToHexaString(' '));
// look if there is already stuff under there
var stuff = await db.ReadAsync((tr) => tr.GetRange(folder.Keys.ToRange()).FirstOrDefaultAsync(), cancellationToken : ct);
if (stuff.Key.IsPresent)
{
log.WriteLine("CAUTION: There is already some data under {0} !");
log.WriteLine(" {0} = {1}", FdbKey.Dump(stuff.Key), stuff.Value.ToAsciiOrHexaString());
}
}
/// <summary>Remove a directory and all its data</summary>
public static async Task RemoveDirectory(string[] path, ITuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
if (log == null)
{
log = Console.Out;
}
string layer = extras.Count > 0 ? extras.Get <string>(0) : null;
var folder = await db.Directory.TryOpenAsync(path, cancellationToken : ct);
if (folder == null)
{
log.WriteLine("# Directory {0} does not exist", string.Join("/", path));
return;
}
// are there any subdirectories ?
var subDirs = await folder.TryListAsync(db, ct);
if (subDirs != null && subDirs.Count > 0)
{
//TODO: "-r" flag ?
log.WriteLine("# Cannot remove {0} because it still contains {1} sub-directorie(s)", string.Join("/", path), subDirs.Count);
}
//TODO: ask for confirmation?
log.WriteLine("# Deleting directory {0} ...", String.Join("/", path));
await folder.RemoveAsync(db, ct);
log.WriteLine("# Gone!");
}
/// <summary>Returns the typed value of the first item in this tuple</summary>
/// <typeparam name="T">Expected type of the first item</typeparam>
/// <returns>Value of the first item, adapted into type <typeparamref name="T"/>.</returns>
public static T First <T>([NotNull] this ITuple tuple)
{
if (tuple == null)
{
throw new ArgumentNullException(nameof(tuple));
}
return(tuple.Get <T>(0));
}
public static void Deconstruct <T1, T2, T3, T4, T5, T6, T7, T8>(this ITuple value, out T1 item1, out T2 item2, out T3 item3, out T4 item4, out T5 item5, out T6 item6, out T7 item7, out T8 item8)
{
value.OfSize(8);
item1 = value.Get <T1>(0);
item2 = value.Get <T2>(1);
item3 = value.Get <T3>(2);
item4 = value.Get <T4>(3);
item5 = value.Get <T5>(4);
item6 = value.Get <T6>(5);
item7 = value.Get <T7>(6);
item8 = value.Get <T8>(7);
}
/// <summary>Compare up to two items in both tuples</summary>
/// <param name="x">First tuple</param>
/// <param name="y">Second tuple</param>
/// <returns>Returns a positive value if x is greater than y, a negative value if x is less than y and 0 if x is equal to y.</returns>
public int Compare(ITuple x, ITuple y)
{
if (y == null)
{
return(x == null ? 0 : +1);
}
if (x == null)
{
return(-1);
}
int nx = x.Count;
int ny = y.Count;
if (ny == 0 || nx == 0)
{
return(nx - ny);
}
int p = this.Offset;
int cmp = this.Comparer1.Compare(x.Get <T1>(p), y.Get <T1>(p));
if (cmp != 0)
{
return(cmp);
}
if (ny == 1 || nx == 1)
{
return(nx - ny);
}
cmp = this.Comparer2.Compare(x.Get <T2>(p + 1), y.Get <T2>(p + 1));
return(cmp);
}
public static T[] ToArray <T>([NotNull] this ITuple tuple)
{
Contract.NotNull(tuple, nameof(tuple));
var items = new T[tuple.Count];
if (items.Length > 0)
{
for (int i = 0; i < items.Length; i++)
{
items[i] = tuple.Get <T>(i);
}
}
return(items);
}
/// <summary>Shows the first few keys of a directory</summary>
public static async Task Show(string[] path, ITuple extras, bool reverse, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
int count = 20;
if (extras.Count > 0)
{
int x = extras.Get <int>(0);
if (x > 0)
{
count = x;
}
}
// look if there is something under there
var folder = await db.Directory.TryOpenAsync(path, cancellationToken : ct);
if (folder != null)
{
log.WriteLine("# Content of {0} [{1}]", FdbKey.Dump(folder.Key), folder.Key.ToHexaString(' '));
var keys = await db.QueryAsync((tr) =>
{
var query = tr.GetRange(folder.Keys.ToRange());
return(reverse
? query.Reverse().Take(count)
: query.Take(count + 1));
}, cancellationToken : ct);
if (keys.Count > 0)
{
if (reverse)
{
keys.Reverse();
}
foreach (var key in keys.Take(count))
{
log.WriteLine("...{0} = {1}", FdbKey.Dump(folder.ExtractKey(key.Key)), key.Value.ToAsciiOrHexaString());
}
if (!reverse && keys.Count == count + 1)
{
log.WriteLine("... more");
}
}
else
{
log.WriteLine(" no content found");
}
}
}
public static T[] ToArray <T>([NotNull] this ITuple tuple)
{
if (tuple == null)
{
throw new ArgumentNullException(nameof(tuple));
}
var items = new T[tuple.Count];
if (items.Length > 0)
{
for (int i = 0; i < items.Length; i++)
{
items[i] = tuple.Get <T>(i);
}
}
return(items);
}
/// <summary>Compare a single item in both tuples</summary>
/// <param name="x">First tuple</param>
/// <param name="y">Second tuple</param>
/// <returns>Returns a positive value if x is greater than y, a negative value if x is less than y and 0 if x is equal to y.</returns>
public int Compare(ITuple x, ITuple y)
{
if (y == null)
{
return(x == null ? 0 : +1);
}
if (x == null)
{
return(-1);
}
int nx = x.Count;
int ny = y.Count;
if (ny == 0 || nx == 0)
{
return(nx - ny);
}
int p = this.Offset;
return(this.Comparer.Compare(x.Get <T1>(p), y.Get <T1>(p)));
}
/// <summary>Execute a lambda Action with the content of this tuple</summary>
/// <param name="tuple">Tuple of size 1</param>
/// <param name="lambda">Action that will be passed the content of this tuple as parameters</param>
/// <exception cref="InvalidOperationException">If <paramref name="tuple"/> has not the expected size</exception>
public static void With <T1>([NotNull] this ITuple tuple, [NotNull] Action <T1> lambda)
{
OfSize(tuple, 1);
lambda(tuple.Get <T1>(0));
}
/// <summary>Execute a lambda Function with the content of this tuple</summary>
/// <param name="tuple">Tuple of size 8</param>
/// <param name="lambda">Function that will be passed the content of this tuple as parameters</param>
/// <returns>Result of calling <paramref name="lambda"/> with the items of this tuple</returns>
/// <exception cref="InvalidOperationException">If <paramref name="tuple"/> has not the expected size</exception>
public static TResult With <T1, T2, T3, T4, T5, T6, T7, T8, TResult>([NotNull] this ITuple tuple, [NotNull] Func <T1, T2, T3, T4, T5, T6, T7, T8, TResult> lambda)
{
OfSize(tuple, 8);
return(lambda(tuple.Get <T1>(0), tuple.Get <T2>(1), tuple.Get <T3>(2), tuple.Get <T4>(3), tuple.Get <T5>(4), tuple.Get <T6>(5), tuple.Get <T7>(6), tuple.Get <T8>(7)));
}
public static async Task Sampling(string[] path, ITuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
double ratio = 0.1d;
bool auto = true;
if (extras.Count > 0)
{
double x = extras.Get <double>(0);
if (x > 0 && x <= 1)
{
ratio = x;
}
auto = false;
}
var folder = await TryOpenCurrentDirectoryAsync(path, db, ct);
KeyRange span;
if (folder is FdbDirectorySubspace)
{
span = KeyRange.StartsWith(FdbSubspace.Copy(folder as FdbDirectorySubspace).Key);
log.WriteLine("Reading list of shards for /{0} under {1} ...", String.Join("/", path), FdbKey.Dump(span.Begin));
}
else
{
log.WriteLine("Reading list of shards for the whole cluster ...");
span = KeyRange.All;
}
// dump keyServers
var ranges = await Fdb.System.GetChunksAsync(db, span, ct);
log.WriteLine("> Found {0:N0} shard(s)", ranges.Count);
// take a sample
var samples = new List <KeyRange>();
if (ranges.Count <= 32)
{ // small enough to scan it all
samples.AddRange(ranges);
log.WriteLine("Sampling all {0:N0} shards ...", samples.Count);
}
else
{ // need to take a random subset
var rnd = new Random();
int sz = Math.Max((int)Math.Ceiling(ratio * ranges.Count), 1);
if (auto)
{
if (sz > 100)
{
sz = 100; //SAFETY
}
if (sz < 32)
{
sz = Math.Max(sz, Math.Min(32, ranges.Count));
}
}
var population = new List <KeyRange>(ranges);
for (int i = 0; i < sz; i++)
{
int p = rnd.Next(population.Count);
samples.Add(population[p]);
population.RemoveAt(p);
}
log.WriteLine("Sampling " + samples.Count + " out of " + ranges.Count + " shards (" + (100.0 * samples.Count / ranges.Count).ToString("N1") + "%) ...");
}
log.WriteLine();
const string FORMAT_STRING = "{0,9} ║{1,10}{6,6} {2,-29} ║{3,10}{7,7} {4,-37} ║{5,10}";
const string SCALE_KEY = "....--------========########M";
const string SCALE_VAL = "....--------========########@@@@@@@@M";
log.WriteLine(FORMAT_STRING, "Count", "Keys", SCALE_KEY, "Values", SCALE_VAL, "Total", "med.", "med.");
var rangeOptions = new FdbRangeOptions {
Mode = FdbStreamingMode.WantAll
};
samples = samples.OrderBy(x => x.Begin).ToList();
long globalSize = 0;
long globalCount = 0;
int workers = 8; // Math.Max(4, Environment.ProcessorCount);
var sw = Stopwatch.StartNew();
var tasks = new List <Task>();
int n = samples.Count;
while (samples.Count > 0)
{
while (tasks.Count < workers && samples.Count > 0)
{
var range = samples[0];
samples.RemoveAt(0);
tasks.Add(Task.Run(async() =>
{
var kk = new RobustHistogram(RobustHistogram.TimeScale.Ticks);
var vv = new RobustHistogram(RobustHistogram.TimeScale.Ticks);
#region Method 1: get_range everything...
using (var tr = db.BeginTransaction(ct))
{
long keySize = 0;
long valueSize = 0;
long count = 0;
int iter = 0;
var beginSelector = KeySelector.FirstGreaterOrEqual(range.Begin);
var endSelector = KeySelector.FirstGreaterOrEqual(range.End);
while (true)
{
FdbRangeChunk data = default(FdbRangeChunk);
FdbException error = null;
try
{
data = await tr.Snapshot.GetRangeAsync(
beginSelector,
endSelector,
rangeOptions,
iter
).ConfigureAwait(false);
}
catch (FdbException e)
{
error = e;
}
if (error != null)
{
await tr.OnErrorAsync(error.Code).ConfigureAwait(false);
continue;
}
if (data.Count == 0)
{
break;
}
count += data.Count;
foreach (var kvp in data.Chunk)
{
keySize += kvp.Key.Count;
valueSize += kvp.Value.Count;
kk.Add(TimeSpan.FromTicks(kvp.Key.Count));
vv.Add(TimeSpan.FromTicks(kvp.Value.Count));
}
if (!data.HasMore)
{
break;
}
beginSelector = KeySelector.FirstGreaterThan(data.Last.Key);
++iter;
}
long totalSize = keySize + valueSize;
Interlocked.Add(ref globalSize, totalSize);
Interlocked.Add(ref globalCount, count);
lock (log)
{
log.WriteLine(FORMAT_STRING, count.ToString("N0"), FormatSize(keySize), kk.GetDistribution(begin: 1, end: 12000, fold: 2), FormatSize(valueSize), vv.GetDistribution(begin: 1, end: 120000, fold: 2), FormatSize(totalSize), FormatSize((int)Math.Ceiling(kk.Median)), FormatSize((int)Math.Ceiling(vv.Median)));
}
}
#endregion
#region Method 2: estimate the count using key selectors...
//long counter = await Fdb.System.EstimateCountAsync(db, range, ct);
//Console.WriteLine("COUNT = " + counter.ToString("N0"));
#endregion
}, ct));
}
var done = await Task.WhenAny(tasks);
tasks.Remove(done);
}
await Task.WhenAll(tasks);
sw.Stop();
log.WriteLine();
if (n != ranges.Count)
{
log.WriteLine("Sampled " + FormatSize(globalSize) + " (" + globalSize.ToString("N0") + " bytes) and " + globalCount.ToString("N0") + " keys in " + sw.Elapsed.TotalSeconds.ToString("N1") + " sec");
log.WriteLine("> Estimated total size is " + FormatSize(globalSize * ranges.Count / n));
}
else
{
log.WriteLine("Found " + FormatSize(globalSize) + " (" + globalSize.ToString("N0") + " bytes) and " + globalCount.ToString("N0") + " keys in " + sw.Elapsed.TotalSeconds.ToString("N1") + " sec");
// compare to the whole cluster
ranges = await Fdb.System.GetChunksAsync(db, FdbKey.MinValue, FdbKey.MaxValue, ct);
log.WriteLine("> This directory contains ~{0:N2}% of all data", (100.0 * n / ranges.Count));
}
log.WriteLine();
}
/// <summary>Execute a lambda Function with the content of this tuple</summary>
/// <param name="tuple">Tuple of size 2</param>
/// <param name="lambda">Function that will be passed the content of this tuple as parameters</param>
/// <returns>Result of calling <paramref name="lambda"/> with the items of this tuple</returns>
/// <exception cref="InvalidOperationException">If <paramref name="tuple"/> has not the expected size</exception>
public static TResult With <T1, T2, TResult>([NotNull] this ITuple tuple, [NotNull] Func <T1, T2, TResult> lambda)
{
OfSize(tuple, 2);
return(lambda(tuple.Get <T1>(0), tuple.Get <T2>(1)));
}
/// <summary>Execute a lambda Function with the content of this tuple</summary>
/// <param name="tuple">Tuple of size 5</param>
/// <param name="lambda">Function that will be passed the content of this tuple as parameters</param>
/// <returns>Result of calling <paramref name="lambda"/> with the items of this tuple</returns>
/// <exception cref="InvalidOperationException">If <paramref name="tuple"/> has not the expected size</exception>
public static TResult With <T1, T2, T3, T4, T5, TResult>([NotNull] this ITuple tuple, [NotNull] Func <T1, T2, T3, T4, T5, TResult> lambda)
{
OfSize(tuple, 5);
return(lambda(tuple.Get <T1>(0), tuple.Get <T2>(1), tuple.Get <T3>(2), tuple.Get <T4>(3), tuple.Get <T5>(4)));
}
/// <summary>Execute a lambda Action with the content of this tuple</summary>
/// <param name="tuple">Tuple of size 8</param>
/// <param name="lambda">Action that will be passed the content of this tuple as parameters</param>
/// <exception cref="InvalidOperationException">If <paramref name="tuple"/> has not the expected size</exception>
public static void With <T1, T2, T3, T4, T5, T6, T7, T8>([NotNull] this ITuple tuple, [NotNull] Action <T1, T2, T3, T4, T5, T6, T7, T8> lambda)
{
OfSize(tuple, 8);
lambda(tuple.Get <T1>(0), tuple.Get <T2>(1), tuple.Get <T3>(2), tuple.Get <T4>(3), tuple.Get <T5>(4), tuple.Get <T6>(5), tuple.Get <T7>(6), tuple.Get <T8>(7));
}
/// <summary>Execute a lambda Action with the content of this tuple</summary>
/// <param name="tuple">Tuple of size 5</param>
/// <param name="lambda">Action that will be passed the content of this tuple as parameters</param>
/// <exception cref="InvalidOperationException">If <paramref name="tuple"/> has not the expected size</exception>
public static void With <T1, T2, T3, T4, T5>([NotNull] this ITuple tuple, [NotNull] Action <T1, T2, T3, T4, T5> lambda)
{
OfSize(tuple, 5);
lambda(tuple.Get <T1>(0), tuple.Get <T2>(1), tuple.Get <T3>(2), tuple.Get <T4>(3), tuple.Get <T5>(4));
}
/// <summary>Execute a lambda Action with the content of this tuple</summary>
/// <param name="tuple">Tuple of size 3</param>
/// <param name="lambda">Action that will be passed the content of this tuple as parameters</param>
/// <exception cref="InvalidOperationException">If <paramref name="tuple"/> has not the expected size</exception>
public static void With <T1, T2, T3>([NotNull] this ITuple tuple, [NotNull] Action <T1, T2, T3> lambda)
{
OfSize(tuple, 3);
lambda(tuple.Get <T1>(0), tuple.Get <T2>(1), tuple.Get <T3>(2));
}
public T Get <T>(int index)
{
return(m_items.Get <T>(index));
}
public static T Last <T>([NotNull] this ITuple tuple)
{
return(tuple.Get <T>(-1));
}
public static T First <T>([NotNull] this ITuple tuple)
{
return(tuple.Get <T>(0));
}
public R Get <R>(int index)
{
return(m_items.Get <R>(index));
}