public override FdbRangeQuery <KeyValuePair <Slice, Slice> > GetRange(FdbKeySelector beginInclusive, FdbKeySelector endExclusive, FdbRangeOptions options = null)
{
m_parent.ThrowIfDisposed();
var query = m_transaction.GetRange(beginInclusive, endExclusive, options);
return(query.UseTransaction(this));
}
public async Task Test_GetKey()
{
Slice key;
Slice value;
using (var db = MemoryDatabase.CreateNew("DB"))
{
using (var tr = db.BeginTransaction(this.Cancellation))
{
tr.Set(db.Pack(0), Slice.FromString("first"));
tr.Set(db.Pack(10), Slice.FromString("ten"));
tr.Set(db.Pack(20), Slice.FromString("ten ten"));
tr.Set(db.Pack(42), Slice.FromString("narf!"));
tr.Set(db.Pack(100), Slice.FromString("a hundred missipis"));
await tr.CommitAsync();
}
db.Debug_Dump();
using (var tr = db.BeginTransaction(this.Cancellation))
{
value = await tr.GetAsync(db.Pack(42));
Console.WriteLine(value);
Assert.That(value.ToString(), Is.EqualTo("narf!"));
key = await tr.GetKeyAsync(FdbKeySelector.FirstGreaterOrEqual(db.Pack(42)));
Assert.That(key, Is.EqualTo(db.Pack(42)));
key = await tr.GetKeyAsync(FdbKeySelector.FirstGreaterThan(db.Pack(42)));
Assert.That(key, Is.EqualTo(db.Pack(100)));
key = await tr.GetKeyAsync(FdbKeySelector.LastLessOrEqual(db.Pack(42)));
Assert.That(key, Is.EqualTo(db.Pack(42)));
key = await tr.GetKeyAsync(FdbKeySelector.LastLessThan(db.Pack(42)));
Assert.That(key, Is.EqualTo(db.Pack(20)));
var keys = await tr.GetKeysAsync(new[]
{
FdbKeySelector.FirstGreaterOrEqual(db.Pack(42)),
FdbKeySelector.FirstGreaterThan(db.Pack(42)),
FdbKeySelector.LastLessOrEqual(db.Pack(42)),
FdbKeySelector.LastLessThan(db.Pack(42))
});
Assert.That(keys.Length, Is.EqualTo(4));
Assert.That(keys[0], Is.EqualTo(db.Pack(42)));
Assert.That(keys[1], Is.EqualTo(db.Pack(100)));
Assert.That(keys[2], Is.EqualTo(db.Pack(42)));
Assert.That(keys[3], Is.EqualTo(db.Pack(20)));
await tr.CommitAsync();
}
}
}
private async Task <Slice> GetPreviousNodeAsync(IFdbTransaction trans, int level, Slice key)
{
// GetPreviousNodeAsync looks for the previous node on a level, but "doesn't care"
// about the contents of that node. It therefore uses a non-isolated (snaphot)
// read and explicitly adds a conflict range that is exclusive of the actual,
// found previous node. This allows an increment of that node not to trigger
// a transaction conflict. We also add a conflict key on the found previous
// key in level 0. This allows detection of erasures.
var k = this.Subspace.Pack(level, key);
//Console.WriteLine(k);
//Console.WriteLine("GetPreviousNode(" + level + ", " + key + ")");
//Console.WriteLine(FdbKeySelector.LastLessThan(k) + " <= x < " + FdbKeySelector.FirstGreaterOrEqual(k));
var kv = await trans
.Snapshot
.GetRange(
FdbKeySelector.LastLessThan(k),
FdbKeySelector.FirstGreaterOrEqual(k)
)
.FirstAsync()
.ConfigureAwait(false);
//Console.WriteLine("Found " + FdbKey.Dump(kv.Key));
var prevKey = this.Subspace.UnpackLast <Slice>(kv.Key);
trans.AddReadConflictRange(kv.Key + FdbKey.MinValue, k);
trans.AddReadConflictKey(this.Subspace.Pack(0, prevKey));
return(prevKey);
}
public async Task <FdbRangeChunk> GetRangeAsync(FdbKeySelector beginInclusive, FdbKeySelector endExclusive, FdbRangeOptions options, int iteration, bool snapshot, CancellationToken cancellationToken)
{
Contract.Requires(beginInclusive.Key.HasValue && endExclusive.Key.HasValue && options != null);
cancellationToken.ThrowIfCancellationRequested();
//TODO: check system keys
//Trace.WriteLine("## GetRange " + beginInclusive + " <= k < " + endExclusive + ", limit=" + options.Limit + ", reverse=" + options.Reverse + ", snapshot=" + snapshot);
lock (m_buffer)
{
beginInclusive = m_buffer.InternSelector(beginInclusive);
endExclusive = m_buffer.InternSelector(endExclusive);
}
// we need the read version
EnsureHasReadVersion();
options = FdbRangeOptions.EnsureDefaults(options, null, null, FdbStreamingMode.Iterator, false);
options.EnsureLegalValues();
var result = await m_db.GetRangeAtVersion(beginInclusive, endExclusive, options.Limit ?? 0, options.TargetBytes ?? 0, options.Mode.Value, iteration, options.Reverse.Value, m_readVersion.Value).ConfigureAwait(false);
if (!snapshot)
{
lock (m_lock)
{
//TODO: use the result to create the conflict range (between the resolver key and the returned key)
//AddReadConflict_NeedsLocking(range);
}
}
return(result);
}
public GetRangeCommand(FdbKeySelector begin, FdbKeySelector end, FdbRangeOptions options, int iteration)
{
this.Begin = begin;
this.End = end;
this.Options = options;
this.Iteration = iteration;
}
public override Task <FdbRangeChunk> GetRangeAsync(FdbKeySelector beginInclusive, FdbKeySelector endExclusive, FdbRangeOptions options = null, int iteration = 0)
{
return(ExecuteAsync(
new FdbTransactionLog.GetRangeCommand(m_parent.Grab(beginInclusive), m_parent.Grab(endExclusive), options, iteration),
(tr, cmd) => tr.GetRangeAsync(cmd.Begin, cmd.End, cmd.Options, cmd.Iteration)
));
}
private async Task <Chunk> GetChunkAtAsync([NotNull] IFdbTransaction trans, long offset)
{
Contract.Requires(trans != null && offset >= 0);
var chunkKey = await trans.GetKeyAsync(FdbKeySelector.LastLessOrEqual(DataKey(offset))).ConfigureAwait(false);
if (chunkKey.IsNull)
{ // nothing before (sparse)
return(default(Chunk));
}
if (chunkKey < DataKey(0))
{ // off beginning
return(default(Chunk));
}
long chunkOffset = DataKeyOffset(chunkKey);
Slice chunkData = await trans.GetAsync(chunkKey).ConfigureAwait(false);
if (chunkOffset + chunkData.Count <= offset)
{ // in sparse region after chunk
return(default(Chunk));
}
return(new Chunk(chunkKey, chunkData, chunkOffset));
}
public override Task <Slice> GetKeyAsync(FdbKeySelector selector)
{
return(ExecuteAsync(
new FdbTransactionLog.GetKeyCommand(m_parent.Grab(selector)),
(tr, cmd) => tr.GetKeyAsync(cmd.Selector)
));
}
private FdbKeySelector Encode(FdbKeySelector selector)
{
return new FdbKeySelector(
m_prefix.Concat(selector.Key),
selector.OrEqual,
selector.Offset
);
}
/// <summary>Copy a key selector into the buffer, and return a new identical selector</summary>
/// <param name="selector">Key selector to copy to the buffer</param>
/// <returns>Equivalent key selector that is backed by the buffer.</returns>
public FdbKeySelector InternSelector(FdbKeySelector selector)
{
return new FdbKeySelector(
Intern(selector.Key, aligned: true),
selector.OrEqual,
selector.Offset
);
}
private FdbKeySelector Encode(FdbKeySelector selector)
{
return(new FdbKeySelector(
m_prefix.ConcatKey(selector.Key),
selector.OrEqual,
selector.Offset
));
}
public Task <Slice> GetKeyAsync(FdbKeySelector selector, bool snapshot, CancellationToken cancellationToken)
{
var future = FdbNative.TransactionGetKey(m_handle, selector, snapshot);
return(FdbFuture.CreateTaskFromHandle(
future,
(h) => GetKeyResult(h),
cancellationToken
));
}
public override FdbRangeQuery <KeyValuePair <Slice, Slice> > GetRange(FdbKeySelector beginInclusive, FdbKeySelector endExclusive, FdbRangeOptions options = null)
{
ThrowIfDisposed();
var query = m_transaction.GetRange(beginInclusive, endExclusive, options);
// this method does not execute immediately, so we don't need to record any operation here, only when GetRangeAsync() is called (by ToListAsync() or any other LINQ operator)
// we must override the transaction used by the query, so that we are notified when this happens
return(query.UseTransaction(this));
}
private async Task <long> GetNextIndexAsync([NotNull] IFdbReadOnlyTransaction tr, IFdbDynamicSubspace subspace)
{
var range = subspace.Keys.ToRange();
var lastKey = await tr.GetKeyAsync(FdbKeySelector.LastLessThan(range.End)).ConfigureAwait(false);
if (lastKey < range.Begin)
{
return(0);
}
return(subspace.Keys.DecodeFirst <long>(lastKey) + 1);
}
private async Task Scenario5(IFdbTransaction tr)
{
var location = FdbSubspace.Create(Slice.FromAscii("TEST"));
//tr.Set(location.Pack(42), Slice.FromString("42"));
//tr.Set(location.Pack(50), Slice.FromString("50"));
//tr.Set(location.Pack(60), Slice.FromString("60"));
var x = await tr.GetKeyAsync(FdbKeySelector.LastLessThan(location.Pack(49)));
Console.WriteLine(x);
tr.Set(location.Pack("FOO"), Slice.FromString("BAR"));
}
private FdbKeySelector[] Grab(FdbKeySelector[] selectors)
{
if (selectors == null || selectors.Length == 0)
{
return(null);
}
var res = new FdbKeySelector[selectors.Length];
for (int i = 0; i < selectors.Length; i++)
{
res[i] = Grab(selectors[i]);
}
return(res);
}
private async Task PushQueueAsync(IFdbTransaction tr, FdbSubspace queue, Slice taskId)
{
//TODO: use a high contention algo ?
// - must support Push and Pop
// - an empty queue must correspond to an empty subspace
// get the current size of the queue
var range = queue.ToRange();
var lastKey = await tr.Snapshot.GetKeyAsync(FdbKeySelector.LastLessThan(range.End)).ConfigureAwait(false);
int count = lastKey < range.Begin ? 0 : queue.Unpack(lastKey).Get <int>(0) + 1;
// set the value
tr.Set(queue.Pack(count, GetRandomId()), taskId);
}
private async Task <long> ComputeSizeAsync(IFdbReadOnlyTransaction tr)
{
Contract.Requires(tr != null);
var keyRange = this.Subspace.Keys.ToRange();
var lastKey = await tr.GetKeyAsync(FdbKeySelector.LastLessOrEqual(keyRange.End)).ConfigureAwait(false);
if (lastKey < keyRange.Begin)
{
return(0);
}
return(this.Subspace.Keys.DecodeFirst <long>(lastKey) + 1);
}
public async Task <Slice[]> GetKeysAsync(FdbKeySelector[] selectors, bool snapshot, CancellationToken cancellationToken)
{
Contract.Requires(selectors != null);
cancellationToken.ThrowIfCancellationRequested();
// order and check the keys
var ordered = new FdbKeySelector[selectors.Length];
for (int i = 0; i < selectors.Length; i++)
{
if (selectors[i].Key.IsNullOrEmpty)
{
throw new ArgumentException("Key cannot be null or empty");
}
//CheckAccessToSystemKeys(key);
ordered[i] = selectors[i];
}
if (ordered.Length > 1)
{ // the db expects the keys to be sorted
Array.Sort(ordered, SelectorKeyComparer.Default);
}
// we need the read version
EnsureHasReadVersion();
lock (m_buffer)
{
for (int i = 0; i < ordered.Length; i++)
{
ordered[i] = m_buffer.InternSelector(ordered[i]);
}
}
var results = await m_db.GetKeysAtVersion(ordered, m_readVersion.Value).ConfigureAwait(false);
if (!snapshot)
{
lock (m_lock)
{
#if !DEBUGz
throw new NotImplementedException("TODO: track read ranges in GetKeysAsync() !");
#endif
}
}
return(results);
}
public async Task <long?> Rank([NotNull] IFdbReadOnlyTransaction trans, Slice key)
{
if (trans == null)
{
throw new ArgumentNullException("trans");
}
if (key.IsNull)
{
throw new ArgumentException("Empty key not allowed in set", "key");
}
if (!(await ContainsAsync(trans, key).ConfigureAwait(false)))
{
return(default(long?));
}
long r = 0;
var rankKey = Slice.Empty;
for (int level = MAX_LEVELS - 1; level >= 0; level--)
{
var lss = this.Subspace.Partition(level);
long lastCount = 0;
var kcs = await trans.GetRange(
FdbKeySelector.FirstGreaterOrEqual(lss.Pack(rankKey)),
FdbKeySelector.FirstGreaterThan(lss.Pack(key))
).ToListAsync().ConfigureAwait(false);
foreach (var kc in kcs)
{
rankKey = lss.UnpackSingle <Slice>(kc.Key);
lastCount = DecodeCount(kc.Value);
r += lastCount;
}
r -= lastCount;
if (rankKey == key)
{
break;
}
}
return(r);
}
public FdbRangeQuery <TId> LookupLessThan([NotNull] IFdbReadOnlyTransaction trans, TValue value, bool orEqual, bool reverse = false)
{
var prefix = this.Location.Partial.Keys.Encode(value);
if (orEqual)
{
prefix = FdbKey.Increment(prefix);
}
var space = new FdbKeySelectorPair(
FdbKeySelector.FirstGreaterOrEqual(this.Location.ToRange().Begin),
FdbKeySelector.FirstGreaterThan(prefix)
);
return(trans
.GetRange(space, new FdbRangeOptions {
Reverse = reverse
})
.Select((kvp) => this.Location.Keys.Decode(kvp.Key).Item2));
}
private FdbKeySelector[] Encode(FdbKeySelector[] selectors)
{
var keys = new Slice[selectors.Length];
for (int i = 0; i < selectors.Length;i++)
{
keys[i] = selectors[i].Key;
}
keys = m_prefix.ConcatRange(keys);
var res = new FdbKeySelector[selectors.Length];
for (int i = 0; i < selectors.Length; i++)
{
res[i] = new FdbKeySelector(
keys[i],
selectors[i].OrEqual,
selectors[i].Offset
);
}
return res;
}
/// <summary>Asynchronously fetch a new page of results</summary>
/// <returns>True if Chunk contains a new page of results. False if all results have been read.</returns>
public Task <FdbRangeChunk> GetRangeAsync(FdbKeySelector begin, FdbKeySelector end, FdbRangeOptions options, int iteration, bool snapshot, CancellationToken cancellationToken)
{
Contract.Requires(options != null);
bool reversed = options.Reverse ?? false;
var future = FdbNative.TransactionGetRange(m_handle, begin, end, options.Limit ?? 0, options.TargetBytes ?? 0, options.Mode ?? FdbStreamingMode.Iterator, iteration, snapshot, reversed);
return(FdbFuture.CreateTaskFromHandle(
future,
(h) =>
{
// TODO: quietly return if disposed
bool hasMore;
var chunk = GetKeyValueArrayResult(h, out hasMore);
return new FdbRangeChunk(hasMore, chunk, iteration, reversed);
},
cancellationToken
));
}
private FdbKeySelector[] Encode(FdbKeySelector[] selectors)
{
var keys = new Slice[selectors.Length];
for (int i = 0; i < selectors.Length; i++)
{
keys[i] = selectors[i].Key;
}
keys = m_prefix.ConcatKeys(keys);
var res = new FdbKeySelector[selectors.Length];
for (int i = 0; i < selectors.Length; i++)
{
res[i] = new FdbKeySelector(
keys[i],
selectors[i].OrEqual,
selectors[i].Offset
);
}
return(res);
}
public async Task MiniBench()
{
const int M = 1 * 1000 * 1000;
const int B = 100;
const int ENTROPY = 10 * 1000;
const int T = M / B;
const int KEYSIZE = 10;
const int VALUESIZE = 100;
const bool RANDOM = false;
var rnd = new Random();
//WARMUP
using (var db = MemoryDatabase.CreateNew("FOO"))
{
await db.WriteAsync((tr) => tr.Set(db.Keys.Encode("hello"), Slice.FromString("world")), this.Cancellation);
Slice.Random(rnd, KEYSIZE);
Slice.Random(rnd, VALUESIZE);
}
Log("Inserting {0}-bytes {1} keys / {2}-bytes values, in {3:N0} transactions", KEYSIZE, RANDOM ? "random" : "ordered", VALUESIZE, T);
bool random = RANDOM;
string fmt = "D" + KEYSIZE;
using (var db = MemoryDatabase.CreateNew("FOO"))
{
DumpMemory(collect: true);
long total = 0;
var payload = new byte[ENTROPY + VALUESIZE];
rnd.NextBytes(payload);
// help with compression by doubling every byte
for (int i = 0; i < payload.Length; i += 2)
{
payload[i + 1] = payload[i];
}
var sw = Stopwatch.StartNew();
sw.Stop();
sw.Restart();
for (int i = 0; i < T; i++)
{
using (var tr = db.BeginTransaction(this.Cancellation))
{
for (int j = 0; j < B; j++)
{
Slice key;
if (random)
{
do
{
key = Slice.Random(rnd, KEYSIZE);
}while (key[0] == 255);
}
else
{
int x = i * B + j;
//x = x % 1000;
key = Slice.FromString(x.ToString(fmt));
}
tr.Set(key, Slice.Create(payload, rnd.Next(ENTROPY), VALUESIZE));
Interlocked.Increment(ref total);
}
await tr.CommitAsync().ConfigureAwait(false);
}
if (i % 1000 == 0)
{
Console.Write("."); // + (i * B).ToString("D10"));
}
}
sw.Stop();
Log("done");
Log("* Inserted: {0:N0} keys", total);
Log("* Elapsed : {0:N3} sec", sw.Elapsed.TotalSeconds);
Log("* TPS: {0:N0} transactions/sec", T / sw.Elapsed.TotalSeconds);
Log("* KPS: {0:N0} keys/sec", total / sw.Elapsed.TotalSeconds);
Log("* BPS: {0:N0} bytes/sec", (total * (KEYSIZE + VALUESIZE)) / sw.Elapsed.TotalSeconds);
DumpMemory(collect: true);
db.Debug_Dump(false);
DumpResult("WriteSeq" + B, total, total / B, sw.Elapsed);
string path = @".\\minibench.pndb";
Log("Saving {0} ...", path);
sw.Restart();
await db.SaveSnapshotAsync(path);
sw.Stop();
Log("* Saved {0:N0} bytes in {1:N3} sec", new System.IO.FileInfo(path).Length, sw.Elapsed.TotalSeconds);
Log("Warming up reads...");
var data = await db.GetValuesAsync(Enumerable.Range(0, 100).Select(i => Slice.FromString(i.ToString(fmt))), this.Cancellation);
Log("Starting read tests...");
#region sequential reads
sw.Restart();
for (int i = 0; i < total; i += 10)
{
using (var tr = db.BeginReadOnlyTransaction(this.Cancellation))
{
await tr.GetValuesAsync(Enumerable.Range(i, 10).Select(x => Slice.FromString(x.ToString(fmt)))).ConfigureAwait(false);
}
}
sw.Stop();
DumpResult("SeqRead10", total, total / 10, sw.Elapsed);
sw.Restart();
for (int i = 0; i < total; i += 10)
{
using (var tr = db.BeginReadOnlyTransaction(this.Cancellation))
{
await tr.Snapshot.GetValuesAsync(Enumerable.Range(i, 10).Select(x => Slice.FromString(x.ToString(fmt)))).ConfigureAwait(false);
}
}
sw.Stop();
DumpResult("SeqRead10S", total, total / 10, sw.Elapsed);
sw.Restart();
for (int i = 0; i < total; i += 10)
{
using (var tr = db.BeginReadOnlyTransaction(this.Cancellation))
{
int x = i;
int y = i + 10;
await tr.GetRangeAsync(
FdbKeySelector.FirstGreaterOrEqual(Slice.FromString(x.ToString(fmt))),
FdbKeySelector.FirstGreaterOrEqual(Slice.FromString(y.ToString(fmt)))
).ConfigureAwait(false);
}
}
sw.Stop();
DumpResult("SeqRead10R", total, total / 10, sw.Elapsed);
sw.Restart();
for (int i = 0; i < total; i += 100)
{
using (var tr = db.BeginReadOnlyTransaction(this.Cancellation))
{
await tr.GetValuesAsync(Enumerable.Range(i, 100).Select(x => Slice.FromString(x.ToString(fmt)))).ConfigureAwait(false);
}
}
sw.Stop();
DumpResult("SeqRead100", total, total / 100, sw.Elapsed);
sw.Restart();
for (int i = 0; i < total; i += 100)
{
using (var tr = db.BeginReadOnlyTransaction(this.Cancellation))
{
await tr.Snapshot.GetValuesAsync(Enumerable.Range(i, 100).Select(x => Slice.FromString(x.ToString(fmt)))).ConfigureAwait(false);
}
}
sw.Stop();
DumpResult("SeqRead100S", total, total / 100, sw.Elapsed);
sw.Restart();
for (int i = 0; i < total; i += 100)
{
using (var tr = db.BeginReadOnlyTransaction(this.Cancellation))
{
int x = i;
int y = i + 100;
await tr.GetRangeAsync(
FdbKeySelector.FirstGreaterOrEqual(Slice.FromString(x.ToString(fmt))),
FdbKeySelector.FirstGreaterOrEqual(Slice.FromString(y.ToString(fmt)))
).ConfigureAwait(false);
}
}
sw.Stop();
DumpResult("SeqRead100R", total, total / 100, sw.Elapsed);
sw.Restart();
for (int i = 0; i < total; i += 100)
{
using (var tr = db.BeginReadOnlyTransaction(this.Cancellation))
{
int x = i;
int y = i + 100;
await tr.Snapshot.GetRangeAsync(
FdbKeySelector.FirstGreaterOrEqual(Slice.FromString(x.ToString(fmt))),
FdbKeySelector.FirstGreaterOrEqual(Slice.FromString(y.ToString(fmt)))
).ConfigureAwait(false);
}
}
sw.Stop();
DumpResult("SeqRead100RS", total, total / 100, sw.Elapsed);
sw.Restart();
for (int i = 0; i < total; i += 1000)
{
using (var tr = db.BeginReadOnlyTransaction(this.Cancellation))
{
await tr.GetValuesAsync(Enumerable.Range(i, 1000).Select(x => Slice.FromString(x.ToString(fmt)))).ConfigureAwait(false);
}
}
sw.Stop();
DumpResult("SeqRead1k", total, total / 1000, sw.Elapsed);
#endregion
DumpMemory();
#region random reads
//sw.Restart();
//for (int i = 0; i < total; i++)
//{
// using (var tr = db.BeginReadOnlyTransaction())
// {
// int x = rnd.Next((int)total);
// await tr.GetAsync(Slice.FromString(x.ToString(fmt)));
// }
//}
//sw.Stop();
//Log("RndRead1 : " + total.ToString("N0") + " keys in " + sw.Elapsed.TotalSeconds.ToString("N3") + " sec => " + (total / sw.Elapsed.TotalSeconds).ToString("N0") + " kps");
sw.Restart();
for (int i = 0; i < total; i += 10)
{
using (var tr = db.BeginReadOnlyTransaction(this.Cancellation))
{
await tr.GetValuesAsync(Enumerable.Range(i, 10).Select(x => Slice.FromString(rnd.Next((int)total).ToString(fmt)))).ConfigureAwait(false);
}
}
sw.Stop();
//Log("RndRead10 : " + total.ToString("N0") + " keys in " + sw.Elapsed.TotalSeconds.ToString("N3") + " sec => " + (total / sw.Elapsed.TotalSeconds).ToString("N0") + " kps, " + (total / (10 * sw.Elapsed.TotalSeconds)).ToString("N0") + " tps");
DumpResult("RndRead10", total, total / 10, sw.Elapsed);
sw.Restart();
for (int i = 0; i < total; i += 10)
{
using (var tr = db.BeginReadOnlyTransaction(this.Cancellation))
{
await tr.Snapshot.GetValuesAsync(Enumerable.Range(i, 10).Select(x => Slice.FromString(rnd.Next((int)total).ToString(fmt)))).ConfigureAwait(false);
}
}
sw.Stop();
//Log("RndRead10S : " + total.ToString("N0") + " keys in " + sw.Elapsed.TotalSeconds.ToString("N3") + " sec => " + (total / sw.Elapsed.TotalSeconds).ToString("N0") + " kps, " + (total / (10 * sw.Elapsed.TotalSeconds)).ToString("N0") + " tps");
DumpResult("RndRead10S", total, total / 10, sw.Elapsed);
sw.Restart();
for (int i = 0; i < total; i += 10)
{
using (var tr = db.BeginReadOnlyTransaction(this.Cancellation))
{
int x = rnd.Next((int)total - 10);
int y = x + 10;
await tr.GetRangeAsync(
FdbKeySelector.FirstGreaterOrEqual(Slice.FromString(x.ToString(fmt))),
FdbKeySelector.FirstGreaterOrEqual(Slice.FromString(y.ToString(fmt)))
).ConfigureAwait(false);
}
}
sw.Stop();
//Log("RndRead10R : " + total.ToString("N0") + " keys in " + sw.Elapsed.TotalSeconds.ToString("N3") + " sec => " + (total / sw.Elapsed.TotalSeconds).ToString("N0") + " kps, " + (total / (10 * sw.Elapsed.TotalSeconds)).ToString("N0") + " tps");
DumpResult("RndRead10R", total, total / 10, sw.Elapsed);
sw.Restart();
for (int i = 0; i < total; i += 100)
{
using (var tr = db.BeginReadOnlyTransaction(this.Cancellation))
{
await tr.GetValuesAsync(Enumerable.Range(i, 100).Select(x => Slice.FromString(rnd.Next((int)total).ToString(fmt)))).ConfigureAwait(false);
}
}
sw.Stop();
//Log("RndRead100 : " + total.ToString("N0") + " keys in " + sw.Elapsed.TotalSeconds.ToString("N3") + " sec => " + (total / sw.Elapsed.TotalSeconds).ToString("N0") + " kps, " + (total / (100 * sw.Elapsed.TotalSeconds)).ToString("N0") + " tps");
DumpResult("RndRead100", total, total / 100, sw.Elapsed);
sw.Restart();
for (int i = 0; i < total; i += 1000)
{
using (var tr = db.BeginReadOnlyTransaction(this.Cancellation))
{
await tr.GetValuesAsync(Enumerable.Range(i, 1000).Select(x => Slice.FromString(rnd.Next((int)total).ToString(fmt)))).ConfigureAwait(false);
}
}
sw.Stop();
//Log("RndRead1k : " + total.ToString("N0") + " keys in " + sw.Elapsed.TotalSeconds.ToString("N3") + " sec => " + (total / sw.Elapsed.TotalSeconds).ToString("N0") + " kps, " + (total / (1000 * sw.Elapsed.TotalSeconds)).ToString("N0") + " tps");
DumpResult("RndRead1k", total, total / 1000, sw.Elapsed);
#endregion
DumpMemory();
#region Parallel Reads...
int CPUS = Environment.ProcessorCount;
long read = 0;
var mre = new ManualResetEvent(false);
var tasks = Enumerable
.Range(0, CPUS)
.Select(k => Task.Run(async() =>
{
var rndz = new Random(k);
mre.WaitOne();
int keys = 0;
for (int j = 0; j < 20; j++)
{
for (int i = 0; i < total / CPUS; i += 100)
{
int pp = i; // rndz.Next((int)total - 10);
using (var tr = db.BeginReadOnlyTransaction(this.Cancellation))
{
var res = await tr.GetValuesAsync(Enumerable.Range(i, 100).Select(x => Slice.FromString((pp + x).ToString(fmt)))).ConfigureAwait(false);
keys += res.Length;
}
}
}
Interlocked.Add(ref read, keys);
return(keys);
})).ToArray();
sw.Restart();
mre.Set();
await Task.WhenAll(tasks);
sw.Stop();
mre.Dispose();
//Log("ParaSeqRead: " + read.ToString("N0") + " keys in " + sw.Elapsed.TotalSeconds.ToString("N3") + " sec => " + (read / sw.Elapsed.TotalSeconds).ToString("N0") + " kps");
DumpResult("ParaSeqRead", read, read / 100, sw.Elapsed);
read = 0;
mre = new ManualResetEvent(false);
tasks = Enumerable
.Range(0, CPUS)
.Select(k => Task.Run(async() =>
{
var rndz = new Random(k);
mre.WaitOne();
int keys = 0;
for (int j = 0; j < 20; j++)
{
for (int i = 0; i < total / CPUS; i += 100)
{
int pp = i; // rndz.Next((int)total - 100);
using (var tr = db.BeginReadOnlyTransaction(this.Cancellation))
{
var res = await tr.GetRangeAsync(
FdbKeySelector.FirstGreaterOrEqual(Slice.FromString(pp.ToString(fmt))),
FdbKeySelector.FirstGreaterOrEqual(Slice.FromString((pp + 100).ToString(fmt)))
).ConfigureAwait(false);
keys += res.Count;
}
}
}
Interlocked.Add(ref read, keys);
return(keys);
})).ToArray();
sw.Restart();
mre.Set();
await Task.WhenAll(tasks);
sw.Stop();
mre.Dispose();
DumpResult("ParaSeqRange", read, read / 100, sw.Elapsed);
#endregion
DumpMemory();
}
}
public GetKeyCommand(FdbKeySelector selector)
{
this.Selector = selector;
}
public virtual Task<Slice> GetKeyAsync(FdbKeySelector selector)
{
ThrowIfDisposed();
return m_transaction.GetKeyAsync(selector);
}
public async Task Run(IFdbDatabase db, TextWriter log, CancellationToken ct)
{
// estimate the number of machines...
Console.WriteLine("# Detecting cluster topology...");
var servers = await db.QueryAsync(tr => tr
.WithReadAccessToSystemKeys()
.GetRange(FdbKeyRange.StartsWith(Fdb.System.ServerList))
.Select(kvp => new
{
Node = kvp.Value.Substring(8, 16).ToHexaString(),
Machine = kvp.Value.Substring(24, 16).ToHexaString(),
DataCenter = kvp.Value.Substring(40, 16).ToHexaString()
}),
ct
);
var numNodes = servers.Select(s => s.Node).Distinct().Count();
var numMachines = servers.Select(s => s.Machine).Distinct().Count();
var numDCs = servers.Select(s => s.DataCenter).Distinct().Count();
Console.WriteLine("# > Found " + numNodes + " process(es) on " + numMachines + " machine(s) in " + numDCs + " datacenter(s)");
Console.WriteLine("# Reading list of shards...");
// dump keyServers
var ranges = await Fdb.System.GetChunksAsync(db, FdbKey.MinValue, FdbKey.MaxValue, ct);
Console.WriteLine("# > Found " + ranges.Count + " shards:");
// take a sample
var rnd = new Random(1234);
int sz = Math.Max((int)Math.Ceiling(this.Ratio * ranges.Count), 1);
if (sz > 500)
{
sz = 500; //SAFETY
}
if (sz < 50)
{
sz = Math.Max(sz, Math.Min(50, ranges.Count));
}
var samples = new List <FdbKeyRange>();
for (int i = 0; i < sz; i++)
{
int p = rnd.Next(ranges.Count);
samples.Add(ranges[p]);
ranges.RemoveAt(p);
}
Console.WriteLine("# Sampling " + sz + " out of " + ranges.Count + " shards (" + (100.0 * sz / ranges.Count).ToString("N1") + "%) ...");
Console.WriteLine("{0,9}{1,10}{2,10}{3,10} : K+V size distribution", "Count", "Keys", "Values", "Total");
var rangeOptions = new FdbRangeOptions {
Mode = FdbStreamingMode.WantAll
};
samples = samples.OrderBy(x => x.Begin).ToList();
long total = 0;
int workers = Math.Min(numMachines, 8);
var sw = Stopwatch.StartNew();
var tasks = new List <Task>();
while (samples.Count > 0)
{
while (tasks.Count < workers && samples.Count > 0)
{
var range = samples[0];
samples.RemoveAt(0);
tasks.Add(Task.Run(async() =>
{
var hh = new RobustHistogram(RobustHistogram.TimeScale.Ticks);
#region Method 1: get_range everything...
using (var tr = db.BeginTransaction(ct))
{
long keySize = 0;
long valueSize = 0;
long count = 0;
int iter = 0;
var beginSelector = FdbKeySelector.FirstGreaterOrEqual(range.Begin);
var endSelector = FdbKeySelector.FirstGreaterOrEqual(range.End);
while (true)
{
FdbRangeChunk data = default(FdbRangeChunk);
FdbException error = null;
try
{
data = await tr.Snapshot.GetRangeAsync(
beginSelector,
endSelector,
rangeOptions,
iter
).ConfigureAwait(false);
}
catch (FdbException e)
{
error = e;
}
if (error != null)
{
await tr.OnErrorAsync(error.Code).ConfigureAwait(false);
continue;
}
if (data.Count == 0)
{
break;
}
count += data.Count;
foreach (var kvp in data.Chunk)
{
keySize += kvp.Key.Count;
valueSize += kvp.Value.Count;
hh.Add(TimeSpan.FromTicks(kvp.Key.Count + kvp.Value.Count));
}
if (!data.HasMore)
{
break;
}
beginSelector = FdbKeySelector.FirstGreaterThan(data.Last.Key);
++iter;
}
long totalSize = keySize + valueSize;
Interlocked.Add(ref total, totalSize);
Console.WriteLine("{0,9}{1,10}{2,10}{3,10} : {4}", count.ToString("N0"), FormatSize(keySize), FormatSize(valueSize), FormatSize(totalSize), hh.GetDistribution(begin: 1, end: 10000, fold: 2));
}
#endregion
#region Method 2: estimate the count using key selectors...
//long counter = await Fdb.System.EstimateCountAsync(db, range, ct);
//Console.WriteLine("COUNT = " + counter.ToString("N0"));
#endregion
}, ct));
}
var done = await Task.WhenAny(tasks);
tasks.Remove(done);
}
await Task.WhenAll(tasks);
sw.Stop();
Console.WriteLine("> Sampled " + FormatSize(total) + " (" + total.ToString("N0") + " bytes) in " + sw.Elapsed.TotalSeconds.ToString("N1") + " sec");
Console.WriteLine("> Estimated total size is " + FormatSize(total * ranges.Count / sz));
}
public override FdbRangeQuery<System.Collections.Generic.KeyValuePair<Slice, Slice>> GetRange(FdbKeySelector beginInclusive, FdbKeySelector endExclusive, FdbRangeOptions options = null)
{
throw new NotImplementedException();
}
public virtual Task<Slice> GetKeyAsync(FdbKeySelector selector)
{
return m_transaction.GetKeyAsync(selector);
}
/// <summary>
/// Create a new range query that will read all key-value pairs in the database snapshot represented by the transaction
/// </summary>
/// <param name="beginInclusive">key selector defining the beginning of the range</param>
/// <param name="endExclusive">key selector defining the end of the range</param>
/// <param name="options">Optionnal query options (Limit, TargetBytes, Mode, Reverse, ...)</param>
/// <returns>Range query that, once executed, will return all the key-value pairs matching the providing selector pair</returns>
public FdbRangeQuery<KeyValuePair<Slice, Slice>> GetRange(FdbKeySelector beginInclusive, FdbKeySelector endExclusive, FdbRangeOptions options = null)
{
EnsureCanRead();
return GetRangeCore(beginInclusive, endExclusive, options, snapshot: false);
}
public static FdbQueryRangeExpression Range(FdbKeySelector start, FdbKeySelector stop, FdbRangeOptions options = null)
{
return(Range(new FdbKeySelectorPair(start, stop), options));
}
/// <summary>Resolves a key selector against the keys in the database snapshot represented by transaction.</summary>
/// <param name="selector">Key selector to resolve</param>
/// <returns>Task that will return the key matching the selector, or an exception</returns>
public async Task<Slice> GetKeyAsync(FdbKeySelector selector)
{
EnsureCanRead();
m_database.EnsureKeyIsValid(selector.Key);
#if DEBUG
if (Logging.On && Logging.IsVerbose) Logging.Verbose(this, "GetKeyAsync", String.Format("Getting key '{0}'", selector.ToString()));
#endif
var key = await m_handler.GetKeyAsync(selector, snapshot: false, cancellationToken: m_cancellation).ConfigureAwait(false);
// don't forget to truncate keys that would fall outside of the database's globalspace !
return m_database.BoundCheck(key);
}
public async Task <Slice> GetKeyAsync(FdbKeySelector selector, bool snapshot, CancellationToken cancellationToken)
{
Contract.Requires(selector.Key.HasValue);
cancellationToken.ThrowIfCancellationRequested();
CheckAccessToSystemKeys(selector.Key, end: true);
//Trace.WriteLine("## GetKey " + selector + ", snapshot=" + snapshot);
FdbKeyRange keyRange;
lock (m_buffer)
{
keyRange = m_buffer.InternRangeFromKey(selector.Key);
selector = new FdbKeySelector(keyRange.Begin, selector.OrEqual, selector.Offset);
}
// we need the read version
EnsureHasReadVersion();
var results = await m_db.GetKeysAtVersion(new [] { selector }, m_readVersion.Value).ConfigureAwait(false);
Contract.Assert(results != null && results.Length == 1);
var result = results[0];
FdbKeyRange resultRange;
int c = result.CompareTo(selector.Key);
if (c == 0)
{ // the result is identical to the key
resultRange = keyRange;
result = keyRange.Begin;
}
else
{ // intern the result
lock (m_buffer)
{
resultRange = m_buffer.InternRangeFromKey(result);
result = resultRange.Begin;
}
}
//TODO: how to merge the results with the local state mutations ?
// => add values that were inserted
// => remove values that were cleared
// => change the value of keys that were mutated locally
if (!snapshot)
{
lock (m_lock)
{
//TODO: use the result to create the conflict range (between the resolver key and the returned key)
if (c == 0)
{ // the key itself was selected, so it can only conflict if it gets deleted by another transaction
// [ result, result+\0 )
AddReadConflict_NeedsLocking(resultRange);
}
else if (c < 0)
{ // the result is before the selected key, so any change between them (including deletion of the result) will conflict
// orEqual == true => [ result, key + \0 )
// orEqual == false => [ result, key )
AddReadConflict_NeedsLocking(FdbKeyRange.Create(resultRange.Begin, selector.OrEqual ? keyRange.End : keyRange.Begin));
}
else
{ // the result is after the selected key, so any change between it and the result will conflict
// orEqual == true => [ key + \0, result + \0 )
// orEqual == false => [ key , result + \0 )
AddReadConflict_NeedsLocking(FdbKeyRange.Create(selector.OrEqual ? keyRange.End : keyRange.Begin, resultRange.End));
}
}
}
return(result);
}
public Task<Slice[]> GetKeysAsync(FdbKeySelector[] selectors, bool snapshot, CancellationToken cancellationToken)
{
Contract.Requires(selectors != null);
var futures = new FutureHandle[selectors.Length];
try
{
for (int i = 0; i < selectors.Length; i++)
{
futures[i] = FdbNative.TransactionGetKey(m_handle, selectors[i], snapshot);
}
}
catch
{
for (int i = 0; i < selectors.Length; i++)
{
if (futures[i] == null) break;
futures[i].Dispose();
}
throw;
}
return FdbFuture.CreateTaskFromHandleArray(futures, (h) => GetKeyResult(h), cancellationToken);
}
public Task<Slice> GetKeyAsync(FdbKeySelector selector, bool snapshot, CancellationToken cancellationToken)
{
var future = FdbNative.TransactionGetKey(m_handle, selector, snapshot);
return FdbFuture.CreateTaskFromHandle(
future,
(h) => GetKeyResult(h),
cancellationToken
);
}
/// <summary>
/// Resolves several key selectors against the keys in the database snapshot represented by the current transaction.
/// </summary>
/// <param name="selectors">Key selectors to resolve</param>
/// <returns>Task that will return an array of keys matching the selectors, or an exception</returns>
public Task<Slice[]> GetKeysAsync(FdbKeySelector[] selectors)
{
EnsureCanRead();
foreach (var selector in selectors)
{
m_database.EnsureKeyIsValid(selector.Key);
}
#if DEBUG
if (Logging.On && Logging.IsVerbose) Logging.Verbose(this, "GetKeysAsync", String.Format("Getting batch of {0} keys ...", selectors.Length));
#endif
return m_handler.GetKeysAsync(selectors, snapshot: false, cancellationToken: m_cancellation);
}
public virtual Task<FdbRangeChunk> GetRangeAsync(FdbKeySelector beginInclusive, FdbKeySelector endExclusive, FdbRangeOptions options = null, int iteration = 0)
{
return m_transaction.GetRangeAsync(beginInclusive, endExclusive, options, iteration);
}
internal FdbRangeQuery<KeyValuePair<Slice, Slice>> GetRangeCore(FdbKeySelector begin, FdbKeySelector end, FdbRangeOptions options, bool snapshot)
{
this.Database.EnsureKeyIsValid(begin.Key);
this.Database.EnsureKeyIsValid(end.Key, endExclusive: true);
options = FdbRangeOptions.EnsureDefaults(options, null, null, FdbStreamingMode.Iterator, false);
options.EnsureLegalValues();
#if DEBUG
if (Logging.On && Logging.IsVerbose) Logging.Verbose(this, "GetRangeCore", String.Format("Getting range '{0} <= x < {1}'", begin.ToString(), end.ToString()));
#endif
return new FdbRangeQuery<KeyValuePair<Slice, Slice>>(this, begin, end, TaskHelpers.Cache<KeyValuePair<Slice, Slice>>.Identity, snapshot, options);
}
public virtual Task<Slice[]> GetKeysAsync(FdbKeySelector[] selectors)
{
return m_transaction.GetKeysAsync(selectors);
}
/// <summary>
/// Reads all key-value pairs in the database snapshot represented by transaction (potentially limited by limit, target_bytes, or mode)
/// which have a key lexicographically greater than or equal to the key resolved by the begin key selector
/// and lexicographically less than the key resolved by the end key selector.
/// </summary>
/// <param name="beginInclusive">key selector defining the beginning of the range</param>
/// <param name="endExclusive">key selector defining the end of the range</param>
/// <param name="options">Optionnal query options (Limit, TargetBytes, StreamingMode, Reverse, ...)</param>
/// <param name="iteration">If streaming mode is FdbStreamingMode.Iterator, this parameter should start at 1 and be incremented by 1 for each successive call while reading this range. In all other cases it is ignored.</param>
/// <returns></returns>
public Task<FdbRangeChunk> GetRangeAsync(FdbKeySelector beginInclusive, FdbKeySelector endExclusive, FdbRangeOptions options = null, int iteration = 0)
{
EnsureCanRead();
m_database.EnsureKeyIsValid(beginInclusive.Key);
m_database.EnsureKeyIsValid(endExclusive.Key, endExclusive: true);
options = FdbRangeOptions.EnsureDefaults(options, null, null, FdbStreamingMode.Iterator, false);
options.EnsureLegalValues();
// The iteration value is only needed when in iterator mode, but then it should start from 1
if (iteration == 0) iteration = 1;
return m_handler.GetRangeAsync(beginInclusive, endExclusive, options, iteration, snapshot: false, cancellationToken: m_cancellation);
}
public override async Task<Slice[]> GetKeysAsync(FdbKeySelector[] selectors)
{
return Decode(await base.GetKeysAsync(Encode(selectors)).ConfigureAwait(false));
}
public static FutureHandle TransactionGetKey(TransactionHandle transaction, FdbKeySelector selector, bool snapshot)
{
if (selector.Key.IsNull) throw new ArgumentException("Key cannot be null", "selector");
fixed (byte* ptrKey = selector.Key.Array)
{
var future = NativeMethods.fdb_transaction_get_key(transaction, ptrKey + selector.Key.Offset, selector.Key.Count, selector.OrEqual, selector.Offset, snapshot);
Contract.Assert(future != null);
#if DEBUG_NATIVE_CALLS
Debug.WriteLine("fdb_transaction_get_key(0x" + transaction.Handle.ToString("x") + ", " + selector.ToString() + ", " + snapshot + ") => 0x" + future.Handle.ToString("x"));
#endif
return future;
}
}
public override Task<FdbRangeChunk> GetRangeAsync(FdbKeySelector beginInclusive, FdbKeySelector endExclusive, FdbRangeOptions options = null, int iteration = 0)
{
throw new NotImplementedException();
}
public static FutureHandle TransactionGetRange(TransactionHandle transaction, FdbKeySelector begin, FdbKeySelector end, int limit, int targetBytes, FdbStreamingMode mode, int iteration, bool snapshot, bool reverse)
{
fixed (byte* ptrBegin = begin.Key.Array)
fixed (byte* ptrEnd = end.Key.Array)
{
var future = NativeMethods.fdb_transaction_get_range(
transaction,
ptrBegin + begin.Key.Offset, begin.Key.Count, begin.OrEqual, begin.Offset,
ptrEnd + end.Key.Offset, end.Key.Count, end.OrEqual, end.Offset,
limit, targetBytes, mode, iteration, snapshot, reverse);
Contract.Assert(future != null);
#if DEBUG_NATIVE_CALLS
Debug.WriteLine("fdb_transaction_get_range(0x" + transaction.Handle.ToString("x") + ", begin: " + begin.PrettyPrint(FdbKey.PrettyPrintMode.Begin) + ", end: " + end.PrettyPrint(FdbKey.PrettyPrintMode.End) + ", " + snapshot + ") => 0x" + future.Handle.ToString("x"));
#endif
return future;
}
}
public virtual Task<Slice[]> GetKeysAsync(FdbKeySelector[] selectors)
{
ThrowIfDisposed();
return m_transaction.GetKeysAsync(selectors);
}
public virtual FdbRangeQuery<KeyValuePair<Slice, Slice>> GetRange(FdbKeySelector beginInclusive, FdbKeySelector endInclusive, FdbRangeOptions options = null)
{
return m_transaction.GetRange(beginInclusive, endInclusive, options);
}
/// <summary>
/// Read from the blob, starting at <paramref name="offset"/>, retrieving up to <paramref name="n"/> bytes (fewer then n bytes are returned when the end of the blob is reached).
/// </summary>
public async Task <Slice> ReadAsync([NotNull] IFdbReadOnlyTransaction trans, long offset, int n)
{
if (trans == null)
{
throw new ArgumentNullException("trans");
}
if (offset < 0)
{
throw new ArgumentNullException("offset", "Offset cannot be less than zero");
}
long?size = await GetSizeAsync(trans).ConfigureAwait(false);
if (size == null)
{
return(Slice.Nil); // not found
}
if (offset >= size.Value)
{
return(Slice.Empty);
}
// read all chunks matching the segment we need, and copy them in our buffer
var buffer = new byte[Math.Min(n, size.Value - offset)];
await trans
.GetRange(
FdbKeySelector.LastLessOrEqual(DataKey(offset)),
FdbKeySelector.FirstGreaterOrEqual(DataKey(offset + n))
)
.ForEachAsync((chunk) =>
{
// get offset of this chunk
long chunkOffset = DataKeyOffset(chunk.Key);
Slice chunkData = chunk.Value;
checked
{
// intersect chunk bounds with output
int delta = (int)(chunkOffset - offset);
int start = delta;
int end = delta + chunkData.Count;
if (start < 0)
{
start = 0;
}
if (end > n)
{
end = n;
}
// compute the relative offsets in the chunk
int rStart = start - delta;
int rEnd = end - delta;
var intersect = chunkData[rStart, rEnd];
if (intersect.IsPresent)
{ // copy the data that fits
intersect.CopyTo(buffer, start);
}
}
})
.ConfigureAwait(false);
return(new Slice(buffer, 0, buffer.Length));
}
private FdbKeySelector Grab(FdbKeySelector selector)
{
return(new FdbKeySelector(Grab(selector.Key), selector.OrEqual, selector.Offset));
}
public static async Task Sampling(string[] path, IFdbTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
double ratio = 0.1d;
bool auto = true;
if (extras.Count > 0)
{
double x = extras.Get <double>(0);
if (x > 0 && x <= 1)
{
ratio = x;
}
auto = false;
}
var folder = await TryOpenCurrentDirectoryAsync(path, db, ct);
FdbKeyRange span;
if (folder is FdbDirectorySubspace)
{
span = FdbKeyRange.StartsWith((folder as FdbDirectorySubspace).Copy());
log.WriteLine("Reading list of shards for /{0} under {1} ...", String.Join("/", path), FdbKey.Dump(span.Begin));
}
else
{
log.WriteLine("Reading list of shards for the whole cluster ...");
span = FdbKeyRange.All;
}
// dump keyServers
var ranges = await Fdb.System.GetChunksAsync(db, span, ct);
log.WriteLine("> Found {0:N0} shard(s)", ranges.Count);
// take a sample
var samples = new List <FdbKeyRange>();
if (ranges.Count <= 32)
{ // small enough to scan it all
samples.AddRange(ranges);
log.WriteLine("Sampling all {0:N0} shards ...", samples.Count);
}
else
{ // need to take a random subset
var rnd = new Random();
int sz = Math.Max((int)Math.Ceiling(ratio * ranges.Count), 1);
if (auto)
{
if (sz > 100)
{
sz = 100; //SAFETY
}
if (sz < 32)
{
sz = Math.Max(sz, Math.Min(32, ranges.Count));
}
}
var population = new List <FdbKeyRange>(ranges);
for (int i = 0; i < sz; i++)
{
int p = rnd.Next(population.Count);
samples.Add(population[p]);
population.RemoveAt(p);
}
log.WriteLine("Sampling " + samples.Count + " out of " + ranges.Count + " shards (" + (100.0 * samples.Count / ranges.Count).ToString("N1") + "%) ...");
}
log.WriteLine();
const string FORMAT_STRING = "{0,9} ║{1,10}{6,6} {2,-29} ║{3,10}{7,7} {4,-37} ║{5,10}";
const string SCALE_KEY = "....--------========########M";
const string SCALE_VAL = "....--------========########@@@@@@@@M";
log.WriteLine(FORMAT_STRING, "Count", "Keys", SCALE_KEY, "Values", SCALE_VAL, "Total", "med.", "med.");
var rangeOptions = new FdbRangeOptions {
Mode = FdbStreamingMode.WantAll
};
samples = samples.OrderBy(x => x.Begin).ToList();
long globalSize = 0;
long globalCount = 0;
int workers = 8; // Math.Max(4, Environment.ProcessorCount);
var sw = Stopwatch.StartNew();
var tasks = new List <Task>();
int n = samples.Count;
while (samples.Count > 0)
{
while (tasks.Count < workers && samples.Count > 0)
{
var range = samples[0];
samples.RemoveAt(0);
tasks.Add(Task.Run(async() =>
{
var kk = new RobustHistogram(RobustHistogram.TimeScale.Ticks);
var vv = new RobustHistogram(RobustHistogram.TimeScale.Ticks);
#region Method 1: get_range everything...
using (var tr = db.BeginTransaction(ct))
{
long keySize = 0;
long valueSize = 0;
long count = 0;
int iter = 0;
var beginSelector = FdbKeySelector.FirstGreaterOrEqual(range.Begin);
var endSelector = FdbKeySelector.FirstGreaterOrEqual(range.End);
while (true)
{
FdbRangeChunk data = default(FdbRangeChunk);
FdbException error = null;
try
{
data = await tr.Snapshot.GetRangeAsync(
beginSelector,
endSelector,
rangeOptions,
iter
).ConfigureAwait(false);
}
catch (FdbException e)
{
error = e;
}
if (error != null)
{
await tr.OnErrorAsync(error.Code).ConfigureAwait(false);
continue;
}
if (data.Count == 0)
{
break;
}
count += data.Count;
foreach (var kvp in data.Chunk)
{
keySize += kvp.Key.Count;
valueSize += kvp.Value.Count;
kk.Add(TimeSpan.FromTicks(kvp.Key.Count));
vv.Add(TimeSpan.FromTicks(kvp.Value.Count));
}
if (!data.HasMore)
{
break;
}
beginSelector = FdbKeySelector.FirstGreaterThan(data.Last.Key);
++iter;
}
long totalSize = keySize + valueSize;
Interlocked.Add(ref globalSize, totalSize);
Interlocked.Add(ref globalCount, count);
lock (log)
{
log.WriteLine(FORMAT_STRING, count.ToString("N0"), FormatSize(keySize), kk.GetDistribution(begin: 1, end: 12000, fold: 2), FormatSize(valueSize), vv.GetDistribution(begin: 1, end: 120000, fold: 2), FormatSize(totalSize), FormatSize((int)Math.Ceiling(kk.Median)), FormatSize((int)Math.Ceiling(vv.Median)));
}
}
#endregion
#region Method 2: estimate the count using key selectors...
//long counter = await Fdb.System.EstimateCountAsync(db, range, ct);
//Console.WriteLine("COUNT = " + counter.ToString("N0"));
#endregion
}, ct));
}
var done = await Task.WhenAny(tasks);
tasks.Remove(done);
}
await Task.WhenAll(tasks);
sw.Stop();
log.WriteLine();
if (n != ranges.Count)
{
log.WriteLine("Sampled " + FormatSize(globalSize) + " (" + globalSize.ToString("N0") + " bytes) and " + globalCount.ToString("N0") + " keys in " + sw.Elapsed.TotalSeconds.ToString("N1") + " sec");
log.WriteLine("> Estimated total size is " + FormatSize(globalSize * ranges.Count / n));
}
else
{
log.WriteLine("Found " + FormatSize(globalSize) + " (" + globalSize.ToString("N0") + " bytes) and " + globalCount.ToString("N0") + " keys in " + sw.Elapsed.TotalSeconds.ToString("N1") + " sec");
// compare to the whole cluster
ranges = await Fdb.System.GetChunksAsync(db, FdbKey.MinValue, FdbKey.MaxValue, ct);
log.WriteLine("> This directory contains ~{0:N2}% of all data", (100.0 * n / ranges.Count));
}
log.WriteLine();
}
/// <summary>Asynchronously fetch a new page of results</summary>
/// <returns>True if Chunk contains a new page of results. False if all results have been read.</returns>
public Task<FdbRangeChunk> GetRangeAsync(FdbKeySelector begin, FdbKeySelector end, FdbRangeOptions options, int iteration, bool snapshot, CancellationToken cancellationToken)
{
Contract.Requires(options != null);
bool reversed = options.Reverse ?? false;
var future = FdbNative.TransactionGetRange(m_handle, begin, end, options.Limit ?? 0, options.TargetBytes ?? 0, options.Mode ?? FdbStreamingMode.Iterator, iteration, snapshot, reversed);
return FdbFuture.CreateTaskFromHandle(
future,
(h) =>
{
// TODO: quietly return if disposed
bool hasMore;
var chunk = GetKeyValueArrayResult(h, out hasMore);
return new FdbRangeChunk(hasMore, chunk, iteration, reversed);
},
cancellationToken
);
}
