public async Task Test_GetKey()
{
Slice key;
Slice value;
using (var db = MemoryDatabase.CreateNew("DB"))
{
using (var tr = db.BeginTransaction(this.Cancellation))
{
tr.Set(db.Pack(0), Slice.FromString("first"));
tr.Set(db.Pack(10), Slice.FromString("ten"));
tr.Set(db.Pack(20), Slice.FromString("ten ten"));
tr.Set(db.Pack(42), Slice.FromString("narf!"));
tr.Set(db.Pack(100), Slice.FromString("a hundred missipis"));
await tr.CommitAsync();
}
db.Debug_Dump();
using (var tr = db.BeginTransaction(this.Cancellation))
{
value = await tr.GetAsync(db.Pack(42));
Console.WriteLine(value);
Assert.That(value.ToString(), Is.EqualTo("narf!"));
key = await tr.GetKeyAsync(FdbKeySelector.FirstGreaterOrEqual(db.Pack(42)));
Assert.That(key, Is.EqualTo(db.Pack(42)));
key = await tr.GetKeyAsync(FdbKeySelector.FirstGreaterThan(db.Pack(42)));
Assert.That(key, Is.EqualTo(db.Pack(100)));
key = await tr.GetKeyAsync(FdbKeySelector.LastLessOrEqual(db.Pack(42)));
Assert.That(key, Is.EqualTo(db.Pack(42)));
key = await tr.GetKeyAsync(FdbKeySelector.LastLessThan(db.Pack(42)));
Assert.That(key, Is.EqualTo(db.Pack(20)));
var keys = await tr.GetKeysAsync(new[]
{
FdbKeySelector.FirstGreaterOrEqual(db.Pack(42)),
FdbKeySelector.FirstGreaterThan(db.Pack(42)),
FdbKeySelector.LastLessOrEqual(db.Pack(42)),
FdbKeySelector.LastLessThan(db.Pack(42))
});
Assert.That(keys.Length, Is.EqualTo(4));
Assert.That(keys[0], Is.EqualTo(db.Pack(42)));
Assert.That(keys[1], Is.EqualTo(db.Pack(100)));
Assert.That(keys[2], Is.EqualTo(db.Pack(42)));
Assert.That(keys[3], Is.EqualTo(db.Pack(20)));
await tr.CommitAsync();
}
}
}
public async Task <long?> Rank([NotNull] IFdbReadOnlyTransaction trans, Slice key)
{
if (trans == null)
{
throw new ArgumentNullException("trans");
}
if (key.IsNull)
{
throw new ArgumentException("Empty key not allowed in set", "key");
}
if (!(await ContainsAsync(trans, key).ConfigureAwait(false)))
{
return(default(long?));
}
long r = 0;
var rankKey = Slice.Empty;
for (int level = MAX_LEVELS - 1; level >= 0; level--)
{
var lss = this.Subspace.Partition(level);
long lastCount = 0;
var kcs = await trans.GetRange(
FdbKeySelector.FirstGreaterOrEqual(lss.Pack(rankKey)),
FdbKeySelector.FirstGreaterThan(lss.Pack(key))
).ToListAsync().ConfigureAwait(false);
foreach (var kc in kcs)
{
rankKey = lss.UnpackSingle <Slice>(kc.Key);
lastCount = DecodeCount(kc.Value);
r += lastCount;
}
r -= lastCount;
if (rankKey == key)
{
break;
}
}
return(r);
}
public FdbRangeQuery <TId> LookupLessThan([NotNull] IFdbReadOnlyTransaction trans, TValue value, bool orEqual, bool reverse = false)
{
var prefix = this.Location.Partial.Keys.Encode(value);
if (orEqual)
{
prefix = FdbKey.Increment(prefix);
}
var space = new FdbKeySelectorPair(
FdbKeySelector.FirstGreaterOrEqual(this.Location.ToRange().Begin),
FdbKeySelector.FirstGreaterThan(prefix)
);
return(trans
.GetRange(space, new FdbRangeOptions {
Reverse = reverse
})
.Select((kvp) => this.Location.Keys.Decode(kvp.Key).Item2));
}
public static async Task Sampling(string[] path, IFdbTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
double ratio = 0.1d;
bool auto = true;
if (extras.Count > 0)
{
double x = extras.Get <double>(0);
if (x > 0 && x <= 1)
{
ratio = x;
}
auto = false;
}
var folder = await TryOpenCurrentDirectoryAsync(path, db, ct);
FdbKeyRange span;
if (folder is FdbDirectorySubspace)
{
span = FdbKeyRange.StartsWith((folder as FdbDirectorySubspace).Copy());
log.WriteLine("Reading list of shards for /{0} under {1} ...", String.Join("/", path), FdbKey.Dump(span.Begin));
}
else
{
log.WriteLine("Reading list of shards for the whole cluster ...");
span = FdbKeyRange.All;
}
// dump keyServers
var ranges = await Fdb.System.GetChunksAsync(db, span, ct);
log.WriteLine("> Found {0:N0} shard(s)", ranges.Count);
// take a sample
var samples = new List <FdbKeyRange>();
if (ranges.Count <= 32)
{ // small enough to scan it all
samples.AddRange(ranges);
log.WriteLine("Sampling all {0:N0} shards ...", samples.Count);
}
else
{ // need to take a random subset
var rnd = new Random();
int sz = Math.Max((int)Math.Ceiling(ratio * ranges.Count), 1);
if (auto)
{
if (sz > 100)
{
sz = 100; //SAFETY
}
if (sz < 32)
{
sz = Math.Max(sz, Math.Min(32, ranges.Count));
}
}
var population = new List <FdbKeyRange>(ranges);
for (int i = 0; i < sz; i++)
{
int p = rnd.Next(population.Count);
samples.Add(population[p]);
population.RemoveAt(p);
}
log.WriteLine("Sampling " + samples.Count + " out of " + ranges.Count + " shards (" + (100.0 * samples.Count / ranges.Count).ToString("N1") + "%) ...");
}
log.WriteLine();
const string FORMAT_STRING = "{0,9} ║{1,10}{6,6} {2,-29} ║{3,10}{7,7} {4,-37} ║{5,10}";
const string SCALE_KEY = "....--------========########M";
const string SCALE_VAL = "....--------========########@@@@@@@@M";
log.WriteLine(FORMAT_STRING, "Count", "Keys", SCALE_KEY, "Values", SCALE_VAL, "Total", "med.", "med.");
var rangeOptions = new FdbRangeOptions {
Mode = FdbStreamingMode.WantAll
};
samples = samples.OrderBy(x => x.Begin).ToList();
long globalSize = 0;
long globalCount = 0;
int workers = 8; // Math.Max(4, Environment.ProcessorCount);
var sw = Stopwatch.StartNew();
var tasks = new List <Task>();
int n = samples.Count;
while (samples.Count > 0)
{
while (tasks.Count < workers && samples.Count > 0)
{
var range = samples[0];
samples.RemoveAt(0);
tasks.Add(Task.Run(async() =>
{
var kk = new RobustHistogram(RobustHistogram.TimeScale.Ticks);
var vv = new RobustHistogram(RobustHistogram.TimeScale.Ticks);
#region Method 1: get_range everything...
using (var tr = db.BeginTransaction(ct))
{
long keySize = 0;
long valueSize = 0;
long count = 0;
int iter = 0;
var beginSelector = FdbKeySelector.FirstGreaterOrEqual(range.Begin);
var endSelector = FdbKeySelector.FirstGreaterOrEqual(range.End);
while (true)
{
FdbRangeChunk data = default(FdbRangeChunk);
FdbException error = null;
try
{
data = await tr.Snapshot.GetRangeAsync(
beginSelector,
endSelector,
rangeOptions,
iter
).ConfigureAwait(false);
}
catch (FdbException e)
{
error = e;
}
if (error != null)
{
await tr.OnErrorAsync(error.Code).ConfigureAwait(false);
continue;
}
if (data.Count == 0)
{
break;
}
count += data.Count;
foreach (var kvp in data.Chunk)
{
keySize += kvp.Key.Count;
valueSize += kvp.Value.Count;
kk.Add(TimeSpan.FromTicks(kvp.Key.Count));
vv.Add(TimeSpan.FromTicks(kvp.Value.Count));
}
if (!data.HasMore)
{
break;
}
beginSelector = FdbKeySelector.FirstGreaterThan(data.Last.Key);
++iter;
}
long totalSize = keySize + valueSize;
Interlocked.Add(ref globalSize, totalSize);
Interlocked.Add(ref globalCount, count);
lock (log)
{
log.WriteLine(FORMAT_STRING, count.ToString("N0"), FormatSize(keySize), kk.GetDistribution(begin: 1, end: 12000, fold: 2), FormatSize(valueSize), vv.GetDistribution(begin: 1, end: 120000, fold: 2), FormatSize(totalSize), FormatSize((int)Math.Ceiling(kk.Median)), FormatSize((int)Math.Ceiling(vv.Median)));
}
}
#endregion
#region Method 2: estimate the count using key selectors...
//long counter = await Fdb.System.EstimateCountAsync(db, range, ct);
//Console.WriteLine("COUNT = " + counter.ToString("N0"));
#endregion
}, ct));
}
var done = await Task.WhenAny(tasks);
tasks.Remove(done);
}
await Task.WhenAll(tasks);
sw.Stop();
log.WriteLine();
if (n != ranges.Count)
{
log.WriteLine("Sampled " + FormatSize(globalSize) + " (" + globalSize.ToString("N0") + " bytes) and " + globalCount.ToString("N0") + " keys in " + sw.Elapsed.TotalSeconds.ToString("N1") + " sec");
log.WriteLine("> Estimated total size is " + FormatSize(globalSize * ranges.Count / n));
}
else
{
log.WriteLine("Found " + FormatSize(globalSize) + " (" + globalSize.ToString("N0") + " bytes) and " + globalCount.ToString("N0") + " keys in " + sw.Elapsed.TotalSeconds.ToString("N1") + " sec");
// compare to the whole cluster
ranges = await Fdb.System.GetChunksAsync(db, FdbKey.MinValue, FdbKey.MaxValue, ct);
log.WriteLine("> This directory contains ~{0:N2}% of all data", (100.0 * n / ranges.Count));
}
log.WriteLine();
}
public async Task Run(IFdbDatabase db, TextWriter log, CancellationToken ct)
{
// estimate the number of machines...
Console.WriteLine("# Detecting cluster topology...");
var servers = await db.QueryAsync(tr => tr
.WithReadAccessToSystemKeys()
.GetRange(FdbKeyRange.StartsWith(Fdb.System.ServerList))
.Select(kvp => new
{
Node = kvp.Value.Substring(8, 16).ToHexaString(),
Machine = kvp.Value.Substring(24, 16).ToHexaString(),
DataCenter = kvp.Value.Substring(40, 16).ToHexaString()
}),
ct
);
var numNodes = servers.Select(s => s.Node).Distinct().Count();
var numMachines = servers.Select(s => s.Machine).Distinct().Count();
var numDCs = servers.Select(s => s.DataCenter).Distinct().Count();
Console.WriteLine("# > Found " + numNodes + " process(es) on " + numMachines + " machine(s) in " + numDCs + " datacenter(s)");
Console.WriteLine("# Reading list of shards...");
// dump keyServers
var ranges = await Fdb.System.GetChunksAsync(db, FdbKey.MinValue, FdbKey.MaxValue, ct);
Console.WriteLine("# > Found " + ranges.Count + " shards:");
// take a sample
var rnd = new Random(1234);
int sz = Math.Max((int)Math.Ceiling(this.Ratio * ranges.Count), 1);
if (sz > 500)
{
sz = 500; //SAFETY
}
if (sz < 50)
{
sz = Math.Max(sz, Math.Min(50, ranges.Count));
}
var samples = new List <FdbKeyRange>();
for (int i = 0; i < sz; i++)
{
int p = rnd.Next(ranges.Count);
samples.Add(ranges[p]);
ranges.RemoveAt(p);
}
Console.WriteLine("# Sampling " + sz + " out of " + ranges.Count + " shards (" + (100.0 * sz / ranges.Count).ToString("N1") + "%) ...");
Console.WriteLine("{0,9}{1,10}{2,10}{3,10} : K+V size distribution", "Count", "Keys", "Values", "Total");
var rangeOptions = new FdbRangeOptions {
Mode = FdbStreamingMode.WantAll
};
samples = samples.OrderBy(x => x.Begin).ToList();
long total = 0;
int workers = Math.Min(numMachines, 8);
var sw = Stopwatch.StartNew();
var tasks = new List <Task>();
while (samples.Count > 0)
{
while (tasks.Count < workers && samples.Count > 0)
{
var range = samples[0];
samples.RemoveAt(0);
tasks.Add(Task.Run(async() =>
{
var hh = new RobustHistogram(RobustHistogram.TimeScale.Ticks);
#region Method 1: get_range everything...
using (var tr = db.BeginTransaction(ct))
{
long keySize = 0;
long valueSize = 0;
long count = 0;
int iter = 0;
var beginSelector = FdbKeySelector.FirstGreaterOrEqual(range.Begin);
var endSelector = FdbKeySelector.FirstGreaterOrEqual(range.End);
while (true)
{
FdbRangeChunk data = default(FdbRangeChunk);
FdbException error = null;
try
{
data = await tr.Snapshot.GetRangeAsync(
beginSelector,
endSelector,
rangeOptions,
iter
).ConfigureAwait(false);
}
catch (FdbException e)
{
error = e;
}
if (error != null)
{
await tr.OnErrorAsync(error.Code).ConfigureAwait(false);
continue;
}
if (data.Count == 0)
{
break;
}
count += data.Count;
foreach (var kvp in data.Chunk)
{
keySize += kvp.Key.Count;
valueSize += kvp.Value.Count;
hh.Add(TimeSpan.FromTicks(kvp.Key.Count + kvp.Value.Count));
}
if (!data.HasMore)
{
break;
}
beginSelector = FdbKeySelector.FirstGreaterThan(data.Last.Key);
++iter;
}
long totalSize = keySize + valueSize;
Interlocked.Add(ref total, totalSize);
Console.WriteLine("{0,9}{1,10}{2,10}{3,10} : {4}", count.ToString("N0"), FormatSize(keySize), FormatSize(valueSize), FormatSize(totalSize), hh.GetDistribution(begin: 1, end: 10000, fold: 2));
}
#endregion
#region Method 2: estimate the count using key selectors...
//long counter = await Fdb.System.EstimateCountAsync(db, range, ct);
//Console.WriteLine("COUNT = " + counter.ToString("N0"));
#endregion
}, ct));
}
var done = await Task.WhenAny(tasks);
tasks.Remove(done);
}
await Task.WhenAll(tasks);
sw.Stop();
Console.WriteLine("> Sampled " + FormatSize(total) + " (" + total.ToString("N0") + " bytes) in " + sw.Elapsed.TotalSeconds.ToString("N1") + " sec");
Console.WriteLine("> Estimated total size is " + FormatSize(total * ranges.Count / sz));
}
public async Task Test_Can_Save_And_Reload_Snapshot()
{
const string FILE_PATH = ".\\test.pndb";
const int N = 1 * 1000 * 1000;
if (File.Exists(FILE_PATH))
{
File.Delete(FILE_PATH);
}
// insert N sequential items and bulk load with "ordered = true" to skip the sorting of levels
Console.WriteLine("Generating " + N.ToString("N0") + " keys...");
var data = new KeyValuePair <Slice, Slice> [N];
var rnd = new Random();
for (int i = 0; i < N; i++)
{
data[i] = new KeyValuePair <Slice, Slice>(
Slice.FromAscii(i.ToString("D16")),
Slice.Random(rnd, 50)
);
}
var sw = new Stopwatch();
using (var db = MemoryDatabase.CreateNew())
{
Console.Write("Inserting ...");
sw.Restart();
await db.BulkLoadAsync(data, ordered : true);
sw.Stop();
Console.WriteLine(" done in " + sw.Elapsed.TotalSeconds.ToString("N1") + " secs");
db.Debug_Dump();
Console.Write("Saving...");
sw.Restart();
await db.SaveSnapshotAsync(FILE_PATH, null, this.Cancellation);
sw.Stop();
Console.WriteLine(" done in " + sw.Elapsed.TotalSeconds.ToString("N1") + " secs");
}
var fi = new FileInfo(FILE_PATH);
Assert.That(fi.Exists, Is.True, "Snapshot file not found");
Console.WriteLine("File size is " + fi.Length.ToString("N0") + " bytes (" + (fi.Length * 1.0d / N).ToString("N2") + " bytes/item, " + (fi.Length / (1048576.0 * sw.Elapsed.TotalSeconds)).ToString("N3") + " MB/sec)");
Console.Write("Loading...");
sw.Restart();
using (var db = await MemoryDatabase.LoadFromAsync(FILE_PATH, this.Cancellation))
{
sw.Stop();
Console.WriteLine(" done in " + sw.Elapsed.TotalSeconds.ToString("N1") + " secs (" + (fi.Length / (1048576.0 * sw.Elapsed.TotalSeconds)).ToString("N0") + " MB/sec)");
db.Debug_Dump();
Console.WriteLine("Checking data integrity...");
sw.Restart();
long n = 0;
foreach (var batch in data.Buffered(50 * 1000))
{
using (var tx = db.BeginReadOnlyTransaction(this.Cancellation))
{
var res = await tx
.Snapshot
.GetRange(
FdbKeySelector.FirstGreaterOrEqual(batch[0].Key),
FdbKeySelector.FirstGreaterThan(batch[batch.Count - 1].Key))
.ToListAsync()
.ConfigureAwait(false);
Assert.That(res.Count, Is.EqualTo(batch.Count), "Some keys are missing from {0} to {1} :(", batch[0], batch[batch.Count - 1]);
for (int i = 0; i < res.Count; i++)
{
// note: Is.EqualTo(...) is slow on Slices so we speed things a bit
if (res[i].Key != batch[i].Key)
{
Assert.That(res[i].Key, Is.EqualTo(batch[i].Key), "Key is different :(");
}
if (res[i].Value != batch[i].Value)
{
Assert.That(res[i].Value, Is.EqualTo(batch[i].Value), "Value is different for key {0} :(", batch[i].Key);
}
}
}
n += batch.Count;
Console.Write("\r" + n.ToString("N0"));
}
sw.Stop();
Console.WriteLine(" done in " + sw.Elapsed.TotalSeconds.ToString("N1") + " secs");
}
Console.WriteLine("Content of database are identical ^_^");
}
public async Task Test_Can_Resolve_Key_Selector_Outside_Boundaries()
{
// test various corner cases:
// - k < first_key or k <= <00> resolves to:
// - '' always
// - k > last_key or k >= <FF> resolve to:
// - '<FF>' when access to system keys is off
// - '<FF>/backupRange' (usually) when access to system keys is ON
// - k >= <FF><00> resolves to:
// - key_outside_legal_range when access to system keys is off
// - '<FF>/backupRange' (usually) when access to system keys is ON
// - k >= <FF><FF> resolved to:
// - key_outside_legal_range when access to system keys is off
// - '<FF><FF>' when access to system keys is ON
Slice key;
using (var db = MemoryDatabase.CreateNew("FOO"))
{
using (var tr = db.BeginTransaction(this.Cancellation))
{
tr.Set(Slice.FromString("A"), Slice.FromString("min"));
tr.Set(Slice.FromString("Z"), Slice.FromString("max"));
await tr.CommitAsync();
}
using (var tr = db.BeginReadOnlyTransaction(this.Cancellation))
{
// before <00>
key = await tr.GetKeyAsync(FdbKeySelector.LastLessThan(FdbKey.MinValue));
Assert.That(key, Is.EqualTo(Slice.Empty), "lLT(<00>) => ''");
// before the first key in the db
var minKey = await tr.GetKeyAsync(FdbKeySelector.FirstGreaterOrEqual(FdbKey.MinValue));
Assert.That(minKey, Is.Not.Null);
Console.WriteLine("minKey = " + minKey);
key = await tr.GetKeyAsync(FdbKeySelector.LastLessThan(minKey));
Assert.That(key, Is.EqualTo(Slice.Empty), "lLT(min_key) => ''");
// after the last key in the db
var maxKey = await tr.GetKeyAsync(FdbKeySelector.LastLessThan(FdbKey.MaxValue));
Assert.That(maxKey, Is.Not.Null);
Console.WriteLine("maxKey = " + maxKey);
key = await tr.GetKeyAsync(FdbKeySelector.FirstGreaterThan(maxKey));
Assert.That(key, Is.EqualTo(FdbKey.MaxValue), "fGT(maxKey) => <FF>");
// after <FF>
key = await tr.GetKeyAsync(FdbKeySelector.FirstGreaterThan(FdbKey.MaxValue));
Assert.That(key, Is.EqualTo(FdbKey.MaxValue), "fGT(<FF>) => <FF>");
Assert.That(async() => await tr.GetKeyAsync(FdbKeySelector.FirstGreaterThan(Slice.FromAscii("\xFF\xFF"))), Throws.InstanceOf <FdbException>().With.Property("Code").EqualTo(FdbError.KeyOutsideLegalRange));
Assert.That(async() => await tr.GetKeyAsync(FdbKeySelector.LastLessThan(Slice.FromAscii("\xFF\x00"))), Throws.InstanceOf <FdbException>().With.Property("Code").EqualTo(FdbError.KeyOutsideLegalRange));
tr.WithReadAccessToSystemKeys();
var firstSystemKey = await tr.GetKeyAsync(FdbKeySelector.FirstGreaterThan(FdbKey.MaxValue));
// usually the first key in the system space is <FF>/backupDataFormat, but that may change in the future version.
Assert.That(firstSystemKey, Is.Not.Null);
Assert.That(firstSystemKey, Is.GreaterThan(FdbKey.MaxValue), "key should be between <FF> and <FF><FF>");
Assert.That(firstSystemKey, Is.LessThan(Slice.FromAscii("\xFF\xFF")), "key should be between <FF> and <FF><FF>");
// with access to system keys, the maximum possible key becomes <FF><FF>
key = await tr.GetKeyAsync(FdbKeySelector.FirstGreaterOrEqual(Slice.FromAscii("\xFF\xFF")));
Assert.That(key, Is.EqualTo(Slice.FromAscii("\xFF\xFF")), "fGE(<FF><FF>) => <FF><FF> (with access to system keys)");
key = await tr.GetKeyAsync(FdbKeySelector.FirstGreaterThan(Slice.FromAscii("\xFF\xFF")));
Assert.That(key, Is.EqualTo(Slice.FromAscii("\xFF\xFF")), "fGT(<FF><FF>) => <FF><FF> (with access to system keys)");
key = await tr.GetKeyAsync(FdbKeySelector.LastLessThan(Slice.FromAscii("\xFF\x00")));
Assert.That(key, Is.EqualTo(maxKey), "lLT(<FF><00>) => max_key (with access to system keys)");
key = await tr.GetKeyAsync(FdbKeySelector.FirstGreaterThan(maxKey));
Assert.That(key, Is.EqualTo(firstSystemKey), "fGT(max_key) => first_system_key (with access to system keys)");
}
}
}
public async Task Test_GetKey_ReadConflicts()
{
Slice key;
using (var db = MemoryDatabase.CreateNew("FOO"))
{
var location = db.Keys;
using (var tr = db.BeginTransaction(this.Cancellation))
{
tr.Set(location.Encode(42), Slice.FromString("42"));
tr.Set(location.Encode(50), Slice.FromString("50"));
tr.Set(location.Encode(60), Slice.FromString("60"));
await tr.CommitAsync();
}
db.Debug_Dump();
Func <FdbKeySelector, Slice, Task> check = async(selector, expected) =>
{
using (var tr = db.BeginTransaction(this.Cancellation))
{
key = await tr.GetKeyAsync(selector);
await tr.CommitAsync();
Assert.That(key, Is.EqualTo(expected), selector.ToString() + " => " + FdbKey.Dump(expected));
}
};
await check(
FdbKeySelector.FirstGreaterOrEqual(location.Encode(50)),
location.Encode(50)
);
await check(
FdbKeySelector.FirstGreaterThan(location.Encode(50)),
location.Encode(60)
);
await check(
FdbKeySelector.FirstGreaterOrEqual(location.Encode(49)),
location.Encode(50)
);
await check(
FdbKeySelector.FirstGreaterThan(location.Encode(49)),
location.Encode(50)
);
await check(
FdbKeySelector.FirstGreaterOrEqual(location.Encode(49)) + 1,
location.Encode(60)
);
await check(
FdbKeySelector.FirstGreaterThan(location.Encode(49)) + 1,
location.Encode(60)
);
await check(
FdbKeySelector.LastLessOrEqual(location.Encode(49)),
location.Encode(42)
);
await check(
FdbKeySelector.LastLessThan(location.Encode(49)),
location.Encode(42)
);
}
}