/// <summary>Asynchronously fetch a new page of results</summary>
/// <returns>True if Chunk contains a new page of results. False if all results have been read.</returns>
public Task <FdbRangeChunk> GetRangeAsync(KeySelector begin, KeySelector end, FdbRangeOptions options, int iteration, bool snapshot, CancellationToken ct)
{
Contract.Requires(options != null);
bool reversed = options.Reverse ?? false;
var future = FdbNative.TransactionGetRange(m_handle, begin, end, options.Limit ?? 0, options.TargetBytes ?? 0, options.Mode ?? FdbStreamingMode.Iterator, iteration, snapshot, reversed);
return(FdbFuture.CreateTaskFromHandle(
future,
(h) =>
{
var mode = options.Read ?? FdbReadMode.Both;
KeyValuePair <Slice, Slice>[] items;
bool hasMore;
Slice first, last;
switch (mode)
{
case FdbReadMode.Both:
{
items = GetKeyValueArrayResult(h, out hasMore, out first, out last);
break;
}
case FdbReadMode.Keys:
{
items = GetKeyValueArrayResultKeysOnly(h, out hasMore, out first, out last);
break;
}
case FdbReadMode.Values:
{
items = GetKeyValueArrayResultValuesOnly(h, out hasMore, out first, out last);
break;
}
default:
{
throw new InvalidOperationException();
}
}
return new FdbRangeChunk(items, hasMore, iteration, reversed, mode, first, last);
},
ct
));
}
/// <inheritdoc />
public virtual Task <FdbRangeChunk> GetRangeAsync(KeySelector beginInclusive, KeySelector endExclusive, FdbRangeOptions options = null, int iteration = 0)
{
return(m_transaction.GetRangeAsync(beginInclusive, endExclusive, options, iteration));
}
public virtual FdbRangeQuery <KeyValuePair <Slice, Slice> > GetRange(FdbKeySelector beginInclusive, FdbKeySelector endExclusive, FdbRangeOptions options = null)
{
ThrowIfDisposed();
return(m_transaction.GetRange(beginInclusive, endExclusive, options));
}
/// <summary>Asynchronously fetch a new page of results</summary>
/// <returns>True if Chunk contains a new page of results. False if all results have been read.</returns>
public Task<FdbRangeChunk> GetRangeAsync(FdbKeySelector begin, FdbKeySelector end, FdbRangeOptions options, int iteration, bool snapshot, CancellationToken cancellationToken)
{
Contract.Requires(options != null);
bool reversed = options.Reverse ?? false;
var future = FdbNative.TransactionGetRange(m_handle, begin, end, options.Limit ?? 0, options.TargetBytes ?? 0, options.Mode ?? FdbStreamingMode.Iterator, iteration, snapshot, reversed);
return FdbFuture.CreateTaskFromHandle(
future,
(h) =>
{
// TODO: quietly return if disposed
bool hasMore;
var chunk = GetKeyValueArrayResult(h, out hasMore);
return new FdbRangeChunk(hasMore, chunk, iteration, reversed);
},
cancellationToken
);
}
public IFdbAsyncEnumerable <KeyValuePair <TKey, TValue> > All([NotNull] IFdbReadOnlyTransaction trans, FdbRangeOptions options = null)
{
if (trans == null)
{
throw new ArgumentNullException("trans");
}
return(trans
.GetRange(this.Location.ToRange(), options)
.Select(this.DecodeItem));
}
public override FdbRangeQuery <System.Collections.Generic.KeyValuePair <Slice, Slice> > GetRange(KeySelector beginInclusive, KeySelector endExclusive, FdbRangeOptions options = null)
{
throw new NotImplementedException();
}
/// <summary>Asynchronously fetch a new page of results</summary>
/// <returns>True if Chunk contains a new page of results. False if all results have been read.</returns>
public Task <FdbRangeChunk> GetRangeAsync(KeySelector begin, KeySelector end, FdbRangeOptions options, int iteration, bool snapshot, CancellationToken ct)
{
Contract.Requires(options != null);
bool reversed = options.Reverse ?? false;
var future = FdbNative.TransactionGetRange(m_handle, begin, end, options.Limit ?? 0, options.TargetBytes ?? 0, options.Mode ?? FdbStreamingMode.Iterator, iteration, snapshot, reversed);
return(FdbFuture.CreateTaskFromHandle(
future,
(h) =>
{
// TODO: quietly return if disposed
bool hasMore;
var chunk = GetKeyValueArrayResult(h, out hasMore);
return new FdbRangeChunk(hasMore, chunk, iteration, reversed);
},
ct
));
}
public static async Task Sampling(string[] path, IVarTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
double ratio = 0.1d;
bool auto = true;
if (extras.Count > 0)
{
double x = extras.Get <double>(0);
if (x > 0 && x <= 1)
{
ratio = x;
}
auto = false;
}
var folder = await TryOpenCurrentDirectoryAsync(path, db, ct);
KeyRange span;
if (folder is FdbDirectorySubspace)
{
span = KeyRange.StartsWith((folder as FdbDirectorySubspace).Copy().GetPrefix());
log.WriteLine($"Reading list of shards for /{String.Join("/", path)} under {FdbKey.Dump(span.Begin)} ...");
}
else
{
log.WriteLine("Reading list of shards for the whole cluster ...");
span = KeyRange.All;
}
// dump keyServers
var ranges = await Fdb.System.GetChunksAsync(db, span, ct);
log.WriteLine($"> Found {ranges.Count:N0} shard(s)");
// take a sample
var samples = new List <KeyRange>();
if (ranges.Count <= 32)
{ // small enough to scan it all
samples.AddRange(ranges);
log.WriteLine($"Sampling all {samples.Count:N0} shards ...");
}
else
{ // need to take a random subset
var rnd = new Random();
int sz = Math.Max((int)Math.Ceiling(ratio * ranges.Count), 1);
if (auto)
{
if (sz > 100)
{
sz = 100; //SAFETY
}
if (sz < 32)
{
sz = Math.Max(sz, Math.Min(32, ranges.Count));
}
}
var population = new List <KeyRange>(ranges);
for (int i = 0; i < sz; i++)
{
int p = rnd.Next(population.Count);
samples.Add(population[p]);
population.RemoveAt(p);
}
log.WriteLine($"Sampling {samples.Count:N0} out of {ranges.Count:N0} shards ({(100.0 * samples.Count / ranges.Count):N1}%) ...");
}
log.WriteLine();
const string FORMAT_STRING = "{0,9} ║{1,10}{6,6} {2,-29} ║{3,10}{7,7} {4,-37} ║{5,10}";
const string SCALE_KEY = "....--------========########M";
const string SCALE_VAL = "....--------========########@@@@@@@@M";
log.WriteLine(FORMAT_STRING, "Count", "Keys", SCALE_KEY, "Values", SCALE_VAL, "Total", "med.", "med.");
var rangeOptions = new FdbRangeOptions {
Mode = FdbStreamingMode.WantAll
};
samples = samples.OrderBy(x => x.Begin).ToList();
long globalSize = 0;
long globalCount = 0;
int workers = 8; // Math.Max(4, Environment.ProcessorCount);
var sw = Stopwatch.StartNew();
var tasks = new List <Task>();
int n = samples.Count;
while (samples.Count > 0)
{
while (tasks.Count < workers && samples.Count > 0)
{
var range = samples[0];
samples.RemoveAt(0);
tasks.Add(Task.Run(async() =>
{
var kk = new RobustHistogram(RobustHistogram.TimeScale.Ticks);
var vv = new RobustHistogram(RobustHistogram.TimeScale.Ticks);
#region Method 1: get_range everything...
using (var tr = db.BeginTransaction(ct))
{
long keySize = 0;
long valueSize = 0;
long count = 0;
int iter = 0;
var beginSelector = KeySelector.FirstGreaterOrEqual(range.Begin);
var endSelector = KeySelector.FirstGreaterOrEqual(range.End);
while (true)
{
FdbRangeChunk data = default(FdbRangeChunk);
FdbException error = null;
try
{
data = await tr.Snapshot.GetRangeAsync(
beginSelector,
endSelector,
rangeOptions,
iter
).ConfigureAwait(false);
}
catch (FdbException e)
{
error = e;
}
if (error != null)
{
await tr.OnErrorAsync(error.Code).ConfigureAwait(false);
continue;
}
if (data.Count == 0)
{
break;
}
count += data.Count;
foreach (var kvp in data)
{
keySize += kvp.Key.Count;
valueSize += kvp.Value.Count;
kk.Add(TimeSpan.FromTicks(kvp.Key.Count));
vv.Add(TimeSpan.FromTicks(kvp.Value.Count));
}
if (!data.HasMore)
{
break;
}
beginSelector = KeySelector.FirstGreaterThan(data.Last);
++iter;
}
long totalSize = keySize + valueSize;
Interlocked.Add(ref globalSize, totalSize);
Interlocked.Add(ref globalCount, count);
lock (log)
{
log.WriteLine(FORMAT_STRING, count.ToString("N0"), FormatSize(keySize), kk.GetDistribution(begin: 1, end: 12000, fold: 2), FormatSize(valueSize), vv.GetDistribution(begin: 1, end: 120000, fold: 2), FormatSize(totalSize), FormatSize((int)Math.Ceiling(kk.Median)), FormatSize((int)Math.Ceiling(vv.Median)));
}
}
#endregion
#region Method 2: estimate the count using key selectors...
//long counter = await Fdb.System.EstimateCountAsync(db, range, ct);
//Console.WriteLine("COUNT = " + counter.ToString("N0"));
#endregion
}, ct));
}
var done = await Task.WhenAny(tasks);
tasks.Remove(done);
}
await Task.WhenAll(tasks);
sw.Stop();
log.WriteLine();
if (n != ranges.Count)
{
log.WriteLine($"Sampled {FormatSize(globalSize)} ({globalSize:N0} bytes) and {globalCount:N0} keys in {sw.Elapsed.TotalSeconds:N1} sec");
log.WriteLine($"> Estimated total size is {FormatSize(globalSize * ranges.Count / n)}");
}
else
{
log.WriteLine($"Found {FormatSize(globalSize)} ({globalSize:N0} bytes) and {globalCount:N0} keys in {sw.Elapsed.TotalSeconds:N1} sec");
// compare to the whole cluster
ranges = await Fdb.System.GetChunksAsync(db, FdbKey.MinValue, FdbKey.MaxValue, ct);
log.WriteLine($"> This directory contains ~{(100.0 * n / ranges.Count):N2}% of all data");
}
log.WriteLine();
}
public static FdbQueryRangeExpression RangeStartsWith(Slice prefix, FdbRangeOptions options = null)
{
// starts_with('A') means ['A', B')
return(Range(KeySelectorPair.StartsWith(prefix), options));
}
public async Task Run(IFdbDatabase db, TextWriter log, CancellationToken ct)
{
// estimate the number of machines...
Console.WriteLine("# Detecting cluster topology...");
var servers = await db.QueryAsync(tr => tr
.WithReadAccessToSystemKeys()
.GetRange(KeyRange.StartsWith(Fdb.System.ServerList))
.Select(kvp => new
{
Node = kvp.Value.Substring(8, 16).ToHexaString(),
Machine = kvp.Value.Substring(24, 16).ToHexaString(),
DataCenter = kvp.Value.Substring(40, 16).ToHexaString()
}),
ct
);
var numNodes = servers.Select(s => s.Node).Distinct().Count();
var numMachines = servers.Select(s => s.Machine).Distinct().Count();
var numDCs = servers.Select(s => s.DataCenter).Distinct().Count();
Console.WriteLine("# > Found " + numNodes + " process(es) on " + numMachines + " machine(s) in " + numDCs + " datacenter(s)");
Console.WriteLine("# Reading list of shards...");
// dump keyServers
var ranges = await Fdb.System.GetChunksAsync(db, FdbKey.MinValue, FdbKey.MaxValue, ct);
Console.WriteLine("# > Found " + ranges.Count + " shards:");
// take a sample
var rnd = new Random(1234);
int sz = Math.Max((int)Math.Ceiling(this.Ratio * ranges.Count), 1);
if (sz > 500)
{
sz = 500; //SAFETY
}
if (sz < 50)
{
sz = Math.Max(sz, Math.Min(50, ranges.Count));
}
var samples = new List <KeyRange>();
for (int i = 0; i < sz; i++)
{
int p = rnd.Next(ranges.Count);
samples.Add(ranges[p]);
ranges.RemoveAt(p);
}
Console.WriteLine("# Sampling " + sz + " out of " + ranges.Count + " shards (" + (100.0 * sz / ranges.Count).ToString("N1") + "%) ...");
Console.WriteLine("{0,9}{1,10}{2,10}{3,10} : K+V size distribution", "Count", "Keys", "Values", "Total");
var rangeOptions = new FdbRangeOptions {
Mode = FdbStreamingMode.WantAll
};
samples = samples.OrderBy(x => x.Begin).ToList();
long total = 0;
int workers = Math.Min(numMachines, 8);
var sw = Stopwatch.StartNew();
var tasks = new List <Task>();
while (samples.Count > 0)
{
while (tasks.Count < workers && samples.Count > 0)
{
var range = samples[0];
samples.RemoveAt(0);
tasks.Add(Task.Run(async() =>
{
var hh = new RobustHistogram(RobustHistogram.TimeScale.Ticks);
#region Method 1: get_range everything...
using (var tr = db.BeginTransaction(ct))
{
long keySize = 0;
long valueSize = 0;
long count = 0;
int iter = 0;
var beginSelector = KeySelector.FirstGreaterOrEqual(range.Begin);
var endSelector = KeySelector.FirstGreaterOrEqual(range.End);
while (true)
{
FdbRangeChunk data = default(FdbRangeChunk);
FdbException error = null;
try
{
data = await tr.Snapshot.GetRangeAsync(
beginSelector,
endSelector,
rangeOptions,
iter
).ConfigureAwait(false);
}
catch (FdbException e)
{
error = e;
}
if (error != null)
{
await tr.OnErrorAsync(error.Code).ConfigureAwait(false);
continue;
}
if (data.Count == 0)
{
break;
}
count += data.Count;
foreach (var kvp in data.Chunk)
{
keySize += kvp.Key.Count;
valueSize += kvp.Value.Count;
hh.Add(TimeSpan.FromTicks(kvp.Key.Count + kvp.Value.Count));
}
if (!data.HasMore)
{
break;
}
beginSelector = KeySelector.FirstGreaterThan(data.Last.Key);
++iter;
}
long totalSize = keySize + valueSize;
Interlocked.Add(ref total, totalSize);
Console.WriteLine("{0,9}{1,10}{2,10}{3,10} : {4}", count.ToString("N0"), FormatSize(keySize), FormatSize(valueSize), FormatSize(totalSize), hh.GetDistribution(begin: 1, end: 10000, fold: 2));
}
#endregion
#region Method 2: estimate the count using key selectors...
//long counter = await Fdb.System.EstimateCountAsync(db, range, ct);
//Console.WriteLine("COUNT = " + counter.ToString("N0"));
#endregion
}, ct));
}
var done = await Task.WhenAny(tasks);
tasks.Remove(done);
}
await Task.WhenAll(tasks);
sw.Stop();
Console.WriteLine("> Sampled " + FormatSize(total) + " (" + total.ToString("N0") + " bytes) in " + sw.Elapsed.TotalSeconds.ToString("N1") + " sec");
Console.WriteLine("> Estimated total size is " + FormatSize(total * ranges.Count / sz));
}
public static FdbQueryRangeExpression Range(KeySelector start, KeySelector stop, FdbRangeOptions options = null)
{
return(Range(new KeySelectorPair(start, stop), options));
}
public static FdbQueryRangeExpression Range(KeySelectorPair range, FdbRangeOptions options = null)
{
return(new FdbQueryRangeExpression(range, options));
}
public static FdbQueryRangeExpression RangeStartsWith(IVarTuple tuple, FdbRangeOptions options = null)
{
return(RangeStartsWith(TuPack.Pack(tuple), options));
}
/// <inheritdoc />
public FdbRangeQuery <KeyValuePair <Slice, Slice> > GetRange(KeySelector beginInclusive, KeySelector endInclusive, FdbRangeOptions options = null)
{
return(GetRange(beginInclusive, endInclusive, kv => kv, options));
}
public virtual FdbRangeQuery <KeyValuePair <Slice, Slice> > GetRange(KeySelector beginInclusive, KeySelector endInclusive, FdbRangeOptions options = null)
{
return(m_transaction.GetRange(beginInclusive, endInclusive, options));
}
/// <inheritdoc />
public virtual FdbRangeQuery <TResult> GetRange <TResult>(KeySelector beginInclusive, KeySelector endInclusive, Func <KeyValuePair <Slice, Slice>, TResult> selector, FdbRangeOptions options = null)
{
return(m_transaction.GetRange(beginInclusive, endInclusive, selector, options));
}
public override FdbRangeQuery <KeyValuePair <Slice, Slice> > GetRange(FdbKeySelector beginInclusive, FdbKeySelector endExclusive, FdbRangeOptions options = null)
{
ThrowIfDisposed();
var query = m_transaction.GetRange(beginInclusive, endExclusive, options);
// this method does not execute immediately, so we don't need to record any operation here, only when GetRangeAsync() is called (by ToListAsync() or any other LINQ operator)
// we must override the transaction used by the query, so that we are notified when this happens
return(query.UseTransaction(this));
}
public async Task <FdbRangeChunk> GetRangeAsync(FdbKeySelector beginInclusive, FdbKeySelector endExclusive, FdbRangeOptions options, int iteration, bool snapshot, CancellationToken cancellationToken)
{
Contract.Requires(beginInclusive.Key.HasValue && endExclusive.Key.HasValue && options != null);
cancellationToken.ThrowIfCancellationRequested();
//TODO: check system keys
//Trace.WriteLine("## GetRange " + beginInclusive + " <= k < " + endExclusive + ", limit=" + options.Limit + ", reverse=" + options.Reverse + ", snapshot=" + snapshot);
lock (m_buffer)
{
beginInclusive = m_buffer.InternSelector(beginInclusive);
endExclusive = m_buffer.InternSelector(endExclusive);
}
// we need the read version
EnsureHasReadVersion();
options = FdbRangeOptions.EnsureDefaults(options, null, null, FdbStreamingMode.Iterator, false);
options.EnsureLegalValues();
var result = await m_db.GetRangeAtVersion(beginInclusive, endExclusive, options.Limit ?? 0, options.TargetBytes ?? 0, options.Mode.Value, iteration, options.Reverse.Value, m_readVersion.Value).ConfigureAwait(false);
if (!snapshot)
{
lock (m_lock)
{
//TODO: use the result to create the conflict range (between the resolver key and the returned key)
//AddReadConflict_NeedsLocking(range);
}
}
return(result);
}
public override Task <FdbRangeChunk> GetRangeAsync(FdbKeySelector beginInclusive, FdbKeySelector endExclusive, FdbRangeOptions options = null, int iteration = 0)
{
return(ExecuteAsync(
new FdbTransactionLog.GetRangeCommand(m_parent.Grab(beginInclusive), m_parent.Grab(endExclusive), options, iteration),
(tr, cmd) => tr.GetRangeAsync(cmd.Begin, cmd.End, cmd.Options, cmd.Iteration)
));
}
internal FdbQueryRangeExpression(KeySelectorPair range, FdbRangeOptions options)
{
this.Range = range;
this.Options = options;
}
public override FdbRangeQuery <KeyValuePair <Slice, Slice> > GetRange(FdbKeySelector beginInclusive, FdbKeySelector endExclusive, FdbRangeOptions options = null)
{
m_parent.ThrowIfDisposed();
var query = m_transaction.GetRange(beginInclusive, endExclusive, options);
return(query.UseTransaction(this));
}
public override Task <FdbRangeChunk> GetRangeAsync(KeySelector beginInclusive, KeySelector endExclusive, FdbRangeOptions options = null, int iteration = 0)
{
throw new NotImplementedException();
}
public override Task <FdbRangeChunk> GetRangeAsync(KeySelector beginInclusive, KeySelector endExclusive, FdbRangeOptions options = null, int iteration = 0)
{
return(ExecuteAsync(
new FdbTransactionLog.GetRangeCommand(Grab(beginInclusive), Grab(endExclusive), options, iteration),
(tr, cmd) => tr.GetRangeAsync(beginInclusive, endExclusive, options, iteration)
));
}
