public static async Task Shards(string[] path, ITuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
var ranges = await Fdb.System.GetChunksAsync(db, FdbKey.MinValue, FdbKey.MaxValue, ct);
Console.WriteLine("Found {0} shards in the whole cluster", ranges.Count);
// look if there is something under there
var folder = (await TryOpenCurrentDirectoryAsync(path, db, ct)) as FdbDirectorySubspace;
if (folder != null)
{
var r = KeyRange.StartsWith(FdbSubspace.Copy(folder).Key);
Console.WriteLine("Searching for shards that intersect with /{0} ...", String.Join("/", path));
ranges = await Fdb.System.GetChunksAsync(db, r, ct);
Console.WriteLine("Found {0} ranges intersecting {1}:", ranges.Count, r);
var last = Slice.Empty;
foreach (var range in ranges)
{
Console.Write("> " + FdbKey.Dump(range.Begin) + " ...");
long count = await Fdb.System.EstimateCountAsync(db, range, ct);
Console.WriteLine(" {0:N0}", count);
last = range.End;
//TODO: we can probably get more details on this shard looking in the system keyspace (where it is, how many replicas, ...)
}
Console.WriteLine("> ... " + FdbKey.Dump(last));
}
//Console.WriteLine("Found " + ranges.Count + " shards in the cluster");
//TODO: shards that intersect the current directory
}
public void Test_Empty_Subspace_Is_Empty()
{
var subspace = FdbSubspace.Empty;
Assert.That(subspace, Is.Not.Null, "FdbSubspace.Empty should not return null");
Assert.That(FdbSubspace.Empty, Is.SameAs(subspace), "FdbSubspace.Empty is a singleton");
Assert.That(subspace.Key.Count, Is.EqualTo(0), "FdbSubspace.Empty.Key should be equal to Slice.Empty");
Assert.That(subspace.Key.HasValue, Is.True, "FdbSubspace.Empty.Key should be equal to Slice.Empty");
Assert.That(FdbSubspace.Copy(subspace), Is.Not.SameAs(subspace));
}
public void Test_Subspace_With_Binary_Prefix()
{
var subspace = new FdbSubspace(Slice.Create(new byte[] { 42, 255, 0, 127 }));
Assert.That(subspace.Key.ToString(), Is.EqualTo("*<FF><00><7F>"));
Assert.That(subspace.Copy(), Is.Not.SameAs(subspace));
Assert.That(subspace.Copy().Key, Is.EqualTo(subspace.Key));
// concat(Slice) should append the slice to the binary prefix directly
Assert.That(subspace.Concat(Slice.FromInt32(0x01020304)).ToString(), Is.EqualTo("*<FF><00><7F><04><03><02><01>"));
Assert.That(subspace.Concat(Slice.FromAscii("hello")).ToString(), Is.EqualTo("*<FF><00><7F>hello"));
// pack(...) should use tuple serialization
Assert.That(subspace.Pack(123).ToString(), Is.EqualTo("*<FF><00><7F><15>{"));
Assert.That(subspace.Pack("hello").ToString(), Is.EqualTo("*<FF><00><7F><02>hello<00>"));
Assert.That(subspace.Pack(Slice.FromAscii("world")).ToString(), Is.EqualTo("*<FF><00><7F><01>world<00>"));
Assert.That(subspace.Pack(FdbTuple.Create("hello", 123)).ToString(), Is.EqualTo("*<FF><00><7F><02>hello<00><15>{"));
// if we derive a tuple from this subspace, it should keep the binary prefix when converted to a key
var t = subspace.Append("world", 123, false);
Assert.That(t, Is.Not.Null);
Assert.That(t.Count, Is.EqualTo(3));
Assert.That(t.Get <string>(0), Is.EqualTo("world"));
Assert.That(t.Get <int>(1), Is.EqualTo(123));
Assert.That(t.Get <bool>(2), Is.False);
var k = t.ToSlice();
Assert.That(k.ToString(), Is.EqualTo("*<FF><00><7F><02>world<00><15>{<14>"));
// if we unpack the key with the binary prefix, we should get a valid tuple
var t2 = subspace.Unpack(k);
Assert.That(t2, Is.Not.Null);
Assert.That(t2.Count, Is.EqualTo(3));
Assert.That(t2.Get <string>(0), Is.EqualTo("world"));
Assert.That(t2.Get <int>(1), Is.EqualTo(123));
Assert.That(t2.Get <bool>(2), Is.False);
}
public void Test_Subspace_With_Tuple_Prefix()
{
var subspace = new FdbSubspace(FdbTuple.Create("hello"));
Assert.That(subspace.Key.ToString(), Is.EqualTo("<02>hello<00>"));
Assert.That(subspace.Copy(), Is.Not.SameAs(subspace));
Assert.That(subspace.Copy().Key, Is.EqualTo(subspace.Key));
// concat(Slice) should append the slice to the tuple prefix directly
Assert.That(subspace.Concat(Slice.FromInt32(0x01020304)).ToString(), Is.EqualTo("<02>hello<00><04><03><02><01>"));
Assert.That(subspace.Concat(Slice.FromAscii("world")).ToString(), Is.EqualTo("<02>hello<00>world"));
// pack(...) should use tuple serialization
Assert.That(subspace.Pack(123).ToString(), Is.EqualTo("<02>hello<00><15>{"));
Assert.That(subspace.Pack("world").ToString(), Is.EqualTo("<02>hello<00><02>world<00>"));
// even though the subspace prefix is a tuple, appending to it will only return the new items
var t = subspace.Append("world", 123, false);
Assert.That(t, Is.Not.Null);
Assert.That(t.Count, Is.EqualTo(3));
Assert.That(t.Get <string>(0), Is.EqualTo("world"));
Assert.That(t.Get <int>(1), Is.EqualTo(123));
Assert.That(t.Get <bool>(2), Is.False);
// but ToSlice() should include the prefix
var k = t.ToSlice();
Assert.That(k.ToString(), Is.EqualTo("<02>hello<00><02>world<00><15>{<14>"));
// if we unpack the key with the binary prefix, we should get a valid tuple
var t2 = subspace.Unpack(k);
Assert.That(t2, Is.Not.Null);
Assert.That(t2.Count, Is.EqualTo(3));
Assert.That(t2.Get <string>(0), Is.EqualTo("world"));
Assert.That(t2.Get <int>(1), Is.EqualTo(123));
Assert.That(t2.Get <bool>(2), Is.False);
}
public void Test_Subspace_Copy_Does_Not_Share_Key_Buffer()
{
var original = FdbSubspace.Create(Slice.FromString("Hello"));
var copy = FdbSubspace.Copy(original);
Assert.That(copy, Is.Not.Null);
Assert.That(copy, Is.Not.SameAs(original), "Copy should be a new instance");
Assert.That(copy.Key, Is.EqualTo(original.Key), "Key should be equal");
Assert.That(copy.Key.Array, Is.Not.SameAs(original.Key.Array), "Key should be a copy of the original");
Assert.That(copy, Is.EqualTo(original), "Copy and original should be considered equal");
Assert.That(copy.ToString(), Is.EqualTo(original.ToString()), "Copy and original should have the same string representation");
Assert.That(copy.GetHashCode(), Is.EqualTo(original.GetHashCode()), "Copy and original should have the same hashcode");
}
public void Test_Subspace_With_Binary_Prefix()
{
var subspace = new FdbSubspace(Slice.Create(new byte[] { 42, 255, 0, 127 }));
Assert.That(subspace.Key.ToString(), Is.EqualTo("*<FF><00><7F>"));
Assert.That(subspace.Copy(), Is.Not.SameAs(subspace));
Assert.That(subspace.Copy().Key, Is.EqualTo(subspace.Key));
// concat(Slice) should append the slice to the binary prefix directly
Assert.That(subspace.Concat(Slice.FromInt32(0x01020304)).ToString(), Is.EqualTo("*<FF><00><7F><04><03><02><01>"));
Assert.That(subspace.Concat(Slice.FromAscii("hello")).ToString(), Is.EqualTo("*<FF><00><7F>hello"));
// pack(...) should use tuple serialization
Assert.That(subspace.Pack(123).ToString(), Is.EqualTo("*<FF><00><7F><15>{"));
Assert.That(subspace.Pack("hello").ToString(), Is.EqualTo("*<FF><00><7F><02>hello<00>"));
Assert.That(subspace.Pack(Slice.FromAscii("world")).ToString(), Is.EqualTo("*<FF><00><7F><01>world<00>"));
Assert.That(subspace.Pack(FdbTuple.Create("hello", 123)).ToString(), Is.EqualTo("*<FF><00><7F><02>hello<00><15>{"));
// if we derive a tuple from this subspace, it should keep the binary prefix when converted to a key
var t = subspace.Append("world", 123, false);
Assert.That(t, Is.Not.Null);
Assert.That(t.Count, Is.EqualTo(3));
Assert.That(t.Get<string>(0), Is.EqualTo("world"));
Assert.That(t.Get<int>(1), Is.EqualTo(123));
Assert.That(t.Get<bool>(2), Is.False);
var k = t.ToSlice();
Assert.That(k.ToString(), Is.EqualTo("*<FF><00><7F><02>world<00><15>{<14>"));
// if we unpack the key with the binary prefix, we should get a valid tuple
var t2 = subspace.Unpack(k);
Assert.That(t2, Is.Not.Null);
Assert.That(t2.Count, Is.EqualTo(3));
Assert.That(t2.Get<string>(0), Is.EqualTo("world"));
Assert.That(t2.Get<int>(1), Is.EqualTo(123));
Assert.That(t2.Get<bool>(2), Is.False);
}
/// <summary>Counts the number of keys inside a directory</summary>
public static async Task Count(string[] path, ITuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
// look if there is something under there
var folder = (await TryOpenCurrentDirectoryAsync(path, db, ct)) as FdbDirectorySubspace;
if (folder == null)
{
log.WriteLine("# Directory {0} does not exist", String.Join("/", path));
return;
}
var copy = FdbSubspace.Copy(folder);
log.WriteLine("# Counting keys under {0} ...", FdbKey.Dump(copy.Key));
var progress = new Progress <STuple <long, Slice> >((state) =>
{
log.Write("\r# Found {0:N0} keys...", state.Item1);
});
long count = await Fdb.System.EstimateCountAsync(db, copy.ToRange(), progress, ct);
log.WriteLine("\r# Found {0:N0} keys in {1}", count, folder.FullName);
}
public static async Task Sampling(string[] path, ITuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
double ratio = 0.1d;
bool auto = true;
if (extras.Count > 0)
{
double x = extras.Get <double>(0);
if (x > 0 && x <= 1)
{
ratio = x;
}
auto = false;
}
var folder = await TryOpenCurrentDirectoryAsync(path, db, ct);
KeyRange span;
if (folder is FdbDirectorySubspace)
{
span = KeyRange.StartsWith(FdbSubspace.Copy(folder as FdbDirectorySubspace).Key);
log.WriteLine("Reading list of shards for /{0} under {1} ...", String.Join("/", path), FdbKey.Dump(span.Begin));
}
else
{
log.WriteLine("Reading list of shards for the whole cluster ...");
span = KeyRange.All;
}
// dump keyServers
var ranges = await Fdb.System.GetChunksAsync(db, span, ct);
log.WriteLine("> Found {0:N0} shard(s)", ranges.Count);
// take a sample
var samples = new List <KeyRange>();
if (ranges.Count <= 32)
{ // small enough to scan it all
samples.AddRange(ranges);
log.WriteLine("Sampling all {0:N0} shards ...", samples.Count);
}
else
{ // need to take a random subset
var rnd = new Random();
int sz = Math.Max((int)Math.Ceiling(ratio * ranges.Count), 1);
if (auto)
{
if (sz > 100)
{
sz = 100; //SAFETY
}
if (sz < 32)
{
sz = Math.Max(sz, Math.Min(32, ranges.Count));
}
}
var population = new List <KeyRange>(ranges);
for (int i = 0; i < sz; i++)
{
int p = rnd.Next(population.Count);
samples.Add(population[p]);
population.RemoveAt(p);
}
log.WriteLine("Sampling " + samples.Count + " out of " + ranges.Count + " shards (" + (100.0 * samples.Count / ranges.Count).ToString("N1") + "%) ...");
}
log.WriteLine();
const string FORMAT_STRING = "{0,9} ║{1,10}{6,6} {2,-29} ║{3,10}{7,7} {4,-37} ║{5,10}";
const string SCALE_KEY = "....--------========########M";
const string SCALE_VAL = "....--------========########@@@@@@@@M";
log.WriteLine(FORMAT_STRING, "Count", "Keys", SCALE_KEY, "Values", SCALE_VAL, "Total", "med.", "med.");
var rangeOptions = new FdbRangeOptions {
Mode = FdbStreamingMode.WantAll
};
samples = samples.OrderBy(x => x.Begin).ToList();
long globalSize = 0;
long globalCount = 0;
int workers = 8; // Math.Max(4, Environment.ProcessorCount);
var sw = Stopwatch.StartNew();
var tasks = new List <Task>();
int n = samples.Count;
while (samples.Count > 0)
{
while (tasks.Count < workers && samples.Count > 0)
{
var range = samples[0];
samples.RemoveAt(0);
tasks.Add(Task.Run(async() =>
{
var kk = new RobustHistogram(RobustHistogram.TimeScale.Ticks);
var vv = new RobustHistogram(RobustHistogram.TimeScale.Ticks);
#region Method 1: get_range everything...
using (var tr = db.BeginTransaction(ct))
{
long keySize = 0;
long valueSize = 0;
long count = 0;
int iter = 0;
var beginSelector = KeySelector.FirstGreaterOrEqual(range.Begin);
var endSelector = KeySelector.FirstGreaterOrEqual(range.End);
while (true)
{
FdbRangeChunk data = default(FdbRangeChunk);
FdbException error = null;
try
{
data = await tr.Snapshot.GetRangeAsync(
beginSelector,
endSelector,
rangeOptions,
iter
).ConfigureAwait(false);
}
catch (FdbException e)
{
error = e;
}
if (error != null)
{
await tr.OnErrorAsync(error.Code).ConfigureAwait(false);
continue;
}
if (data.Count == 0)
{
break;
}
count += data.Count;
foreach (var kvp in data.Chunk)
{
keySize += kvp.Key.Count;
valueSize += kvp.Value.Count;
kk.Add(TimeSpan.FromTicks(kvp.Key.Count));
vv.Add(TimeSpan.FromTicks(kvp.Value.Count));
}
if (!data.HasMore)
{
break;
}
beginSelector = KeySelector.FirstGreaterThan(data.Last.Key);
++iter;
}
long totalSize = keySize + valueSize;
Interlocked.Add(ref globalSize, totalSize);
Interlocked.Add(ref globalCount, count);
lock (log)
{
log.WriteLine(FORMAT_STRING, count.ToString("N0"), FormatSize(keySize), kk.GetDistribution(begin: 1, end: 12000, fold: 2), FormatSize(valueSize), vv.GetDistribution(begin: 1, end: 120000, fold: 2), FormatSize(totalSize), FormatSize((int)Math.Ceiling(kk.Median)), FormatSize((int)Math.Ceiling(vv.Median)));
}
}
#endregion
#region Method 2: estimate the count using key selectors...
//long counter = await Fdb.System.EstimateCountAsync(db, range, ct);
//Console.WriteLine("COUNT = " + counter.ToString("N0"));
#endregion
}, ct));
}
var done = await Task.WhenAny(tasks);
tasks.Remove(done);
}
await Task.WhenAll(tasks);
sw.Stop();
log.WriteLine();
if (n != ranges.Count)
{
log.WriteLine("Sampled " + FormatSize(globalSize) + " (" + globalSize.ToString("N0") + " bytes) and " + globalCount.ToString("N0") + " keys in " + sw.Elapsed.TotalSeconds.ToString("N1") + " sec");
log.WriteLine("> Estimated total size is " + FormatSize(globalSize * ranges.Count / n));
}
else
{
log.WriteLine("Found " + FormatSize(globalSize) + " (" + globalSize.ToString("N0") + " bytes) and " + globalCount.ToString("N0") + " keys in " + sw.Elapsed.TotalSeconds.ToString("N1") + " sec");
// compare to the whole cluster
ranges = await Fdb.System.GetChunksAsync(db, FdbKey.MinValue, FdbKey.MaxValue, ct);
log.WriteLine("> This directory contains ~{0:N2}% of all data", (100.0 * n / ranges.Count));
}
log.WriteLine();
}
public static async Task Dir(string[] path, ITuple extras, DirectoryBrowseOptions options, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
if (log == null)
{
log = Console.Out;
}
log.WriteLine("# Listing {0}:", String.Join("/", path));
var parent = await TryOpenCurrentDirectoryAsync(path, db, ct);
if (parent == null)
{
log.WriteLine(" Directory not found.");
return;
}
if (parent.Layer.IsPresent)
{
log.WriteLine("# Layer: {0}", parent.Layer.ToAsciiOrHexaString());
}
var folders = await Fdb.Directory.BrowseAsync(db, parent, ct);
if (folders != null && folders.Count > 0)
{
foreach (var kvp in folders)
{
var name = kvp.Key;
var subfolder = kvp.Value;
if (subfolder != null)
{
if ((options & DirectoryBrowseOptions.ShowCount) != 0)
{
if (!(subfolder is FdbDirectoryPartition))
{
long count = await Fdb.System.EstimateCountAsync(db, subfolder.Keys.ToRange(), ct);
log.WriteLine(" {0,-12} {1,-12} {3,9:N0} {2}", FdbKey.Dump(FdbSubspace.Copy(subfolder).Key), subfolder.Layer.IsNullOrEmpty ? "-" : ("<" + subfolder.Layer.ToUnicode() + ">"), name, count);
}
else
{
log.WriteLine(" {0,-12} {1,-12} {3,9} {2}", FdbKey.Dump(FdbSubspace.Copy(subfolder).Key), subfolder.Layer.IsNullOrEmpty ? "-" : ("<" + subfolder.Layer.ToUnicode() + ">"), name, "-");
}
}
else
{
log.WriteLine(" {0,-12} {1,-12} {2}", FdbKey.Dump(FdbSubspace.Copy(subfolder).Key), subfolder.Layer.IsNullOrEmpty ? "-" : ("<" + subfolder.Layer.ToUnicode() + ">"), name);
}
}
else
{
log.WriteLine(" WARNING: {0} seems to be missing!", name);
}
}
log.WriteLine(" {0} sub-directorie(s).", folders.Count);
}
else
{
//TODO: test if it contains data?
log.WriteLine(" No sub-directories.");
}
}
public void Test_Subspace_With_Tuple_Prefix()
{
var subspace = new FdbSubspace(FdbTuple.Create("hello"));
Assert.That(subspace.Key.ToString(), Is.EqualTo("<02>hello<00>"));
Assert.That(subspace.Copy(), Is.Not.SameAs(subspace));
Assert.That(subspace.Copy().Key, Is.EqualTo(subspace.Key));
// concat(Slice) should append the slice to the tuple prefix directly
Assert.That(subspace.Concat(Slice.FromInt32(0x01020304)).ToString(), Is.EqualTo("<02>hello<00><04><03><02><01>"));
Assert.That(subspace.Concat(Slice.FromAscii("world")).ToString(), Is.EqualTo("<02>hello<00>world"));
// pack(...) should use tuple serialization
Assert.That(subspace.Pack(123).ToString(), Is.EqualTo("<02>hello<00><15>{"));
Assert.That(subspace.Pack("world").ToString(), Is.EqualTo("<02>hello<00><02>world<00>"));
// even though the subspace prefix is a tuple, appending to it will only return the new items
var t = subspace.Append("world", 123, false);
Assert.That(t, Is.Not.Null);
Assert.That(t.Count, Is.EqualTo(3));
Assert.That(t.Get<string>(0), Is.EqualTo("world"));
Assert.That(t.Get<int>(1), Is.EqualTo(123));
Assert.That(t.Get<bool>(2), Is.False);
// but ToSlice() should include the prefix
var k = t.ToSlice();
Assert.That(k.ToString(), Is.EqualTo("<02>hello<00><02>world<00><15>{<14>"));
// if we unpack the key with the binary prefix, we should get a valid tuple
var t2 = subspace.Unpack(k);
Assert.That(t2, Is.Not.Null);
Assert.That(t2.Count, Is.EqualTo(3));
Assert.That(t2.Get<string>(0), Is.EqualTo("world"));
Assert.That(t2.Get<int>(1), Is.EqualTo(123));
Assert.That(t2.Get<bool>(2), Is.False);
}