/// <summary>Copy an existing set of options</summary>
/// <param name="options"></param>
public FdbRangeOptions(FdbRangeOptions options)
{
this.Limit = options.Limit;
this.Reverse = options.Reverse;
this.TargetBytes = options.TargetBytes;
this.Mode = options.Mode;
}
private static async Task <List <Slice> > GetBoundaryKeysInternalAsync([NotNull] IFdbReadOnlyTransaction trans, Slice begin, Slice end)
{
Contract.Requires(trans != null && end >= begin);
#if TRACE_COUNTING
trans.Annotate("Get boundary keys in range {0}", KeyRange.Create(begin, end));
#endif
trans.WithReadAccessToSystemKeys();
var results = new List <Slice>();
int iterations = 0;
var options = new FdbRangeOptions {
Mode = FdbStreamingMode.WantAll
};
while (begin < end)
{
FdbException error = null;
Slice lastBegin = begin;
try
{
var chunk = await trans.Snapshot.GetRangeAsync(KeyServers + begin, KeyServers + end, options, iterations).ConfigureAwait(false);
++iterations;
if (chunk.Count > 0)
{
foreach (var kvp in chunk.Chunk)
{
results.Add(kvp.Key.Substring(KeyServers.Count));
}
begin = chunk.Last.Key.Substring(KeyServers.Count) + 0;
}
if (!chunk.HasMore)
{
begin = end;
}
}
catch (FdbException e)
{
error = e;
}
if (error != null)
{
if (error.Code == FdbError.PastVersion && begin != lastBegin)
{ // if we get a PastVersion and *something* has happened, then we are no longer transactionnal
trans.Reset();
}
else
{
await trans.OnErrorAsync(error.Code).ConfigureAwait(false);
}
iterations = 0;
trans.WithReadAccessToSystemKeys();
}
}
#if TRACE_COUNTING
if (results.Count == 0)
{
trans.Annotate("There is no chunk boundary in range {0}", KeyRange.Create(begin, end));
}
else
{
trans.Annotate("Found {0} boundaries in {1} iteration(s)", results.Count, iterations);
}
#endif
return(results);
}
public static FdbRangeQuery <KeyValuePair <Slice, Slice> > GetRangeStartsWith(this IFdbReadOnlyTransaction trans, [NotNull] IKeySubspace subspace, FdbRangeOptions options = null)
{
//REVIEW: should we remove this method?
Contract.Requires(trans != null && subspace != null);
return(trans.GetRange(subspace.ToRange(), options));
}
/// <summary>Add all missing values from the provided defaults</summary>
/// <param name="options">Options provided by the caller (can be null)</param>
/// <param name="limit">Default value for Limit if not provided</param>
/// <param name="targetBytes">Default TargetBytes for limit if not provided</param>
/// <param name="mode">Default value for StreamingMode if not provided</param>
/// <param name="reverse">Default value for Reverse if not provided</param>
/// <returns>Options with all the values filled</returns>
public static FdbRangeOptions EnsureDefaults(FdbRangeOptions options, int? limit, int? targetBytes, FdbStreamingMode mode, bool reverse)
{
Contract.Requires((limit ?? 0) >= 0 && (targetBytes ?? 0) >= 0 && Enum.IsDefined(typeof(FdbStreamingMode), mode));
if (options == null)
{
options = new FdbRangeOptions()
{
Limit = limit,
TargetBytes = targetBytes,
Mode = mode,
Reverse = reverse
};
}
else if (options.Limit == null || options.TargetBytes == null || options.Mode == null || options.Reverse == null)
{
options = new FdbRangeOptions()
{
Limit = options.Limit ?? limit,
TargetBytes = options.TargetBytes ?? targetBytes,
Mode = options.Mode ?? mode,
Reverse = options.Reverse ?? reverse
};
}
Contract.Ensures(options.Mode != null && options.Reverse != null);
Contract.Ensures((options.Limit ?? 0) >= 0, "Limit cannot be negative");
Contract.Ensures((options.TargetBytes ?? 0) >= 0, "TargetBytes cannot be negative");
Contract.Ensures(options.Mode.HasValue && Enum.IsDefined(typeof(FdbStreamingMode), options.Mode.Value), "Streaming mode must be valid");
return options;
}
public FdbRangeQuery <KeyValuePair <Slice, Slice> > GetRange(FdbKeySelector beginInclusive, FdbKeySelector endExclusive, FdbRangeOptions options)
{
EnsureCanRead();
return(m_parent.GetRangeCore(beginInclusive, endExclusive, options, snapshot: true));
}
public override FdbRangeQuery<System.Collections.Generic.KeyValuePair<Slice, Slice>> GetRange(FdbKeySelector beginInclusive, FdbKeySelector endExclusive, FdbRangeOptions options = null)
{
throw new NotImplementedException();
}
private static async Task<List<Slice>> GetBoundaryKeysInternalAsync([NotNull] IFdbReadOnlyTransaction trans, Slice begin, Slice end)
{
Contract.Requires(trans != null && end >= begin);
#if TRACE_COUNTING
trans.Annotate("Get boundary keys in range {0}", FdbKeyRange.Create(begin, end));
#endif
trans.WithReadAccessToSystemKeys();
var results = new List<Slice>();
int iterations = 0;
var options = new FdbRangeOptions { Mode = FdbStreamingMode.WantAll };
while (begin < end)
{
FdbException error = null;
Slice lastBegin = begin;
try
{
var chunk = await trans.Snapshot.GetRangeAsync(KeyServers + begin, KeyServers + end, options, iterations).ConfigureAwait(false);
++iterations;
if (chunk.Count > 0)
{
foreach (var kvp in chunk.Chunk)
{
results.Add(kvp.Key.Substring(KeyServers.Count));
}
begin = chunk.Last.Key.Substring(KeyServers.Count) + (byte)0;
}
if (!chunk.HasMore)
{
begin = end;
}
}
catch (FdbException e)
{
error = e;
}
if (error != null)
{
if (error.Code == FdbError.PastVersion && begin != lastBegin)
{ // if we get a PastVersion and *something* has happened, then we are no longer transactionnal
trans.Reset();
}
else
{
await trans.OnErrorAsync(error.Code).ConfigureAwait(false);
}
iterations = 0;
trans.WithReadAccessToSystemKeys();
}
}
#if TRACE_COUNTING
if (results.Count == 0)
{
trans.Annotate("There is no chunk boundary in range {0}", FdbKeyRange.Create(begin, end));
}
else
{
trans.Annotate("Found {0} boundaries in {1} iteration(s)", results.Count, iterations);
}
#endif
return results;
}
/// <summary>Read a single page of a range query from the database, using a dedicated transaction.</summary>
/// <param name="db">Database instance</param>
/// <remarks>
/// Use this method only if you intend to perform a single operation inside your execution context (ex: HTTP request).
/// If you need to combine multiple read or write operations, consider using on of the multiple <see cref="IFdbReadOnlyRetryable.ReadAsync"/> or <see cref="IFdbRetryable.ReadWriteAsync"/> overrides.
/// </remarks>
public static Task <FdbRangeChunk> GetRangeAsync(this IFdbReadOnlyRetryable db, FdbKeySelector beginInclusive, FdbKeySelector endExclusive, FdbRangeOptions options, int iteration, CancellationToken cancellationToken)
{
if (db == null)
{
throw new ArgumentNullException("db");
}
return(db.ReadAsync((tr) => tr.GetRangeAsync(beginInclusive, endExclusive, options, iteration), cancellationToken));
}
internal FdbRangeQuery<KeyValuePair<Slice, Slice>> GetRangeCore(FdbKeySelector begin, FdbKeySelector end, FdbRangeOptions options, bool snapshot)
{
this.Database.EnsureKeyIsValid(begin.Key);
this.Database.EnsureKeyIsValid(end.Key, endExclusive: true);
options = FdbRangeOptions.EnsureDefaults(options, null, null, FdbStreamingMode.Iterator, false);
options.EnsureLegalValues();
#if DEBUG
if (Logging.On && Logging.IsVerbose) Logging.Verbose(this, "GetRangeCore", String.Format("Getting range '{0} <= x < {1}'", begin.ToString(), end.ToString()));
#endif
return new FdbRangeQuery<KeyValuePair<Slice, Slice>>(this, begin, end, TaskHelpers.Cache<KeyValuePair<Slice, Slice>>.Identity, snapshot, options);
}
/// <summary>
/// Reads all key-value pairs in the database snapshot represented by transaction (potentially limited by limit, target_bytes, or mode)
/// which have a key lexicographically greater than or equal to the key resolved by the begin key selector
/// and lexicographically less than the key resolved by the end key selector.
/// </summary>
/// <param name="beginInclusive">key selector defining the beginning of the range</param>
/// <param name="endExclusive">key selector defining the end of the range</param>
/// <param name="options">Optionnal query options (Limit, TargetBytes, StreamingMode, Reverse, ...)</param>
/// <param name="iteration">If streaming mode is FdbStreamingMode.Iterator, this parameter should start at 1 and be incremented by 1 for each successive call while reading this range. In all other cases it is ignored.</param>
/// <returns></returns>
public Task<FdbRangeChunk> GetRangeAsync(FdbKeySelector beginInclusive, FdbKeySelector endExclusive, FdbRangeOptions options = null, int iteration = 0)
{
EnsureCanRead();
m_database.EnsureKeyIsValid(beginInclusive.Key);
m_database.EnsureKeyIsValid(endExclusive.Key, endExclusive: true);
options = FdbRangeOptions.EnsureDefaults(options, null, null, FdbStreamingMode.Iterator, false);
options.EnsureLegalValues();
// The iteration value is only needed when in iterator mode, but then it should start from 1
if (iteration == 0) iteration = 1;
return m_handler.GetRangeAsync(beginInclusive, endExclusive, options, iteration, snapshot: false, cancellationToken: m_cancellation);
}
public virtual FdbRangeQuery<KeyValuePair<Slice, Slice>> GetRange(FdbKeySelector beginInclusive, FdbKeySelector endInclusive, FdbRangeOptions options = null)
{
return m_transaction.GetRange(beginInclusive, endInclusive, options);
}
public virtual Task<FdbRangeChunk> GetRangeAsync(FdbKeySelector beginInclusive, FdbKeySelector endExclusive, FdbRangeOptions options = null, int iteration = 0)
{
return m_transaction.GetRangeAsync(beginInclusive, endExclusive, options, iteration);
}
public FdbRangeQuery <TResult> GetRange <TResult>(KeySelector beginInclusive, KeySelector endExclusive, Func <KeyValuePair <Slice, Slice>, TResult> selector, FdbRangeOptions options = null)
{
return(m_parent.GetRangeCore(beginInclusive, endExclusive, options, snapshot: true, selector));
}
public FdbRangeQuery <KeyValuePair <Slice, Slice> > GetRange(KeySelector beginInclusive, KeySelector endExclusive, FdbRangeOptions options = null)
{
return(m_parent.GetRangeCore(beginInclusive, endExclusive, options, snapshot: true, kv => kv));
}
public async Task Run(IFdbDatabase db, TextWriter log, CancellationToken ct)
{
// estimate the number of machines...
Console.WriteLine("# Detecting cluster topology...");
var servers = await db.QueryAsync(tr => tr
.WithReadAccessToSystemKeys()
.GetRange(FdbKeyRange.StartsWith(Fdb.System.ServerList))
.Select(kvp => new
{
Node = kvp.Value.Substring(8, 16).ToHexaString(),
Machine = kvp.Value.Substring(24, 16).ToHexaString(),
DataCenter = kvp.Value.Substring(40, 16).ToHexaString()
}),
ct
);
var numNodes = servers.Select(s => s.Node).Distinct().Count();
var numMachines = servers.Select(s => s.Machine).Distinct().Count();
var numDCs = servers.Select(s => s.DataCenter).Distinct().Count();
Console.WriteLine("# > Found " + numNodes + " process(es) on " + numMachines + " machine(s) in " + numDCs + " datacenter(s)");
Console.WriteLine("# Reading list of shards...");
// dump keyServers
var ranges = await Fdb.System.GetChunksAsync(db, FdbKey.MinValue, FdbKey.MaxValue, ct);
Console.WriteLine("# > Found " + ranges.Count + " shards:");
// take a sample
var rnd = new Random(1234);
int sz = Math.Max((int)Math.Ceiling(this.Ratio * ranges.Count), 1);
if (sz > 500) sz = 500; //SAFETY
if (sz < 50) sz = Math.Max(sz, Math.Min(50, ranges.Count));
var samples = new List<FdbKeyRange>();
for (int i = 0; i < sz; i++)
{
int p = rnd.Next(ranges.Count);
samples.Add(ranges[p]);
ranges.RemoveAt(p);
}
Console.WriteLine("# Sampling " + sz + " out of " + ranges.Count + " shards (" + (100.0 * sz / ranges.Count).ToString("N1") + "%) ...");
Console.WriteLine("{0,9}{1,10}{2,10}{3,10} : K+V size distribution", "Count", "Keys", "Values", "Total");
var rangeOptions = new FdbRangeOptions { Mode = FdbStreamingMode.WantAll };
samples = samples.OrderBy(x => x.Begin).ToList();
long total = 0;
int workers = Math.Min(numMachines, 8);
var sw = Stopwatch.StartNew();
var tasks = new List<Task>();
while(samples.Count > 0)
{
while (tasks.Count < workers && samples.Count > 0)
{
var range = samples[0];
samples.RemoveAt(0);
tasks.Add(Task.Run(async () =>
{
var hh = new RobustHistogram(RobustHistogram.TimeScale.Ticks);
#region Method 1: get_range everything...
using (var tr = db.BeginTransaction(ct))
{
long keySize = 0;
long valueSize = 0;
long count = 0;
int iter = 0;
var beginSelector = FdbKeySelector.FirstGreaterOrEqual(range.Begin);
var endSelector = FdbKeySelector.FirstGreaterOrEqual(range.End);
while (true)
{
FdbRangeChunk data = default(FdbRangeChunk);
FdbException error = null;
try
{
data = await tr.Snapshot.GetRangeAsync(
beginSelector,
endSelector,
rangeOptions,
iter
).ConfigureAwait(false);
}
catch (FdbException e)
{
error = e;
}
if (error != null)
{
await tr.OnErrorAsync(error.Code).ConfigureAwait(false);
continue;
}
if (data.Count == 0) break;
count += data.Count;
foreach (var kvp in data.Chunk)
{
keySize += kvp.Key.Count;
valueSize += kvp.Value.Count;
hh.Add(TimeSpan.FromTicks(kvp.Key.Count + kvp.Value.Count));
}
if (!data.HasMore) break;
beginSelector = FdbKeySelector.FirstGreaterThan(data.Last.Key);
++iter;
}
long totalSize = keySize + valueSize;
Interlocked.Add(ref total, totalSize);
Console.WriteLine("{0,9}{1,10}{2,10}{3,10} : {4}", count.ToString("N0"), FormatSize(keySize), FormatSize(valueSize), FormatSize(totalSize), hh.GetDistribution(begin: 1, end: 10000, fold:2));
}
#endregion
#region Method 2: estimate the count using key selectors...
//long counter = await Fdb.System.EstimateCountAsync(db, range, ct);
//Console.WriteLine("COUNT = " + counter.ToString("N0"));
#endregion
}, ct));
}
var done = await Task.WhenAny(tasks);
tasks.Remove(done);
}
await Task.WhenAll(tasks);
sw.Stop();
Console.WriteLine("> Sampled " + FormatSize(total) + " (" + total.ToString("N0") + " bytes) in " + sw.Elapsed.TotalSeconds.ToString("N1") + " sec");
Console.WriteLine("> Estimated total size is " + FormatSize(total * ranges.Count / sz));
}
/// <summary>Construct a query with a set of initial settings</summary>
internal FdbRangeQuery([NotNull] IFdbReadOnlyTransaction transaction, KeySelector begin, KeySelector end, [NotNull] Func <KeyValuePair <Slice, Slice>, T> transform, bool snapshot, [CanBeNull] FdbRangeOptions options)
{
Contract.Requires(transaction != null && transform != null);
this.Transaction = transaction;
this.Begin = begin;
this.End = end;
this.Transform = transform;
this.Snapshot = snapshot;
this.Options = options ?? new FdbRangeOptions();
this.OriginalRange = KeySelectorPair.Create(begin, end);
}
/// <summary>
/// Create a new range query that will read all key-value pairs in the database snapshot represented by the transaction
/// </summary>
/// <param name="beginInclusive">key selector defining the beginning of the range</param>
/// <param name="endExclusive">key selector defining the end of the range</param>
/// <param name="options">Optionnal query options (Limit, TargetBytes, Mode, Reverse, ...)</param>
/// <returns>Range query that, once executed, will return all the key-value pairs matching the providing selector pair</returns>
public FdbRangeQuery<KeyValuePair<Slice, Slice>> GetRange(FdbKeySelector beginInclusive, FdbKeySelector endExclusive, FdbRangeOptions options = null)
{
EnsureCanRead();
return GetRangeCore(beginInclusive, endExclusive, options, snapshot: false);
}
public Task <FdbRangeChunk> GetRangeAsync(KeySelector beginInclusive, KeySelector endExclusive, FdbRangeOptions options, int iteration)
{
EnsureCanRead();
m_parent.m_database.EnsureKeyIsValid(beginInclusive.Key);
m_parent.m_database.EnsureKeyIsValid(endExclusive.Key);
options = FdbRangeOptions.EnsureDefaults(options, null, null, FdbStreamingMode.Iterator, FdbReadMode.Both, false);
options.EnsureLegalValues();
// The iteration value is only needed when in iterator mode, but then it should start from 1
if (iteration == 0)
{
iteration = 1;
}
return(m_parent.m_handler.GetRangeAsync(beginInclusive, endExclusive, options, iteration, snapshot: true, ct: m_parent.m_cancellation));
}
/// <summary>Asynchronously fetch a new page of results</summary>
/// <param name="cancellationToken"></param>
/// <returns>True if Chunk contains a new page of results. False if all results have been read.</returns>
private Task <bool> FetchNextPageAsync(CancellationToken cancellationToken)
{
Contract.Requires(!this.AtEnd);
Contract.Requires(this.Iteration >= 0);
cancellationToken.ThrowIfCancellationRequested();
this.Transaction.EnsureCanRead();
this.Iteration++;
#if DEBUG_RANGE_PAGING
Debug.WriteLine("FdbRangeQuery.PagingIterator.FetchNextPageAsync(iter=" + this.Iteration + ") started");
#endif
var options = new FdbRangeOptions
{
Limit = this.RemainingCount,
TargetBytes = this.RemainingSize,
Mode = this.Query.Mode,
Reverse = this.Query.Reversed
};
// select the appropriate streaming mode if purpose is not default
switch (m_mode)
{
case FdbAsyncMode.Iterator:
{
// the caller is responsible for calling MoveNext(..) and deciding if it wants to continue or not..
options.Mode = FdbStreamingMode.Iterator;
break;
}
case FdbAsyncMode.All:
{
// we are in a ToList or ForEach, we want to read everything in as few chunks as possible
options.Mode = FdbStreamingMode.WantAll;
break;
}
case FdbAsyncMode.Head:
{
// the caller only expect one (or zero) values
options.Mode = FdbStreamingMode.Iterator;
break;
}
}
var tr = this.Transaction;
if (this.Query.Snapshot)
{ // make sure we have the snapshot version !
tr = tr.Snapshot;
}
//BUGBUG: mix the custom cancellation token with the transaction, if it is diffent !
var task = tr
.GetRangeAsync(this.Begin, this.End, options, this.Iteration)
.Then((result) =>
{
this.Chunk = result.Chunk;
this.RowCount += result.Count;
this.HasMore = result.HasMore;
// subtract number of row from the remaining allowed
if (this.RemainingCount.HasValue)
{
this.RemainingCount = this.RemainingCount.Value - result.Count;
}
// subtract size of rows from the remaining allowed
if (this.RemainingSize.HasValue)
{
this.RemainingSize = this.RemainingSize.Value - result.GetSize();
}
this.AtEnd = !result.HasMore || (this.RemainingCount.HasValue && this.RemainingCount.Value <= 0) || (this.RemainingSize.HasValue && this.RemainingSize.Value <= 0);
if (!this.AtEnd)
{ // update begin..end so that next call will continue from where we left...
var lastKey = result.Last.Key;
if (this.Query.Reversed)
{
this.End = KeySelector.FirstGreaterOrEqual(lastKey);
}
else
{
this.Begin = KeySelector.FirstGreaterThan(lastKey);
}
}
#if DEBUG_RANGE_PAGING
Debug.WriteLine("FdbRangeQuery.PagingIterator.FetchNextPageAsync() returned " + this.Chunk.Length + " results (" + this.RowCount + " total) " + (hasMore ? " with more to come" : " and has no more data"));
#endif
if (!result.IsEmpty && this.Transaction != null)
{
return(Publish(result.Chunk));
}
return(Completed());
});
// keep track of this operation
this.PendingReadTask = task;
return(task);
}
public static async Task Sampling(string[] path, IFdbTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
double ratio = 0.1d;
bool auto = true;
if (extras.Count > 0)
{
double x = extras.Get<double>(0);
if (x > 0 && x <= 1) ratio = x;
auto = false;
}
var folder = await TryOpenCurrentDirectoryAsync(path, db, ct);
FdbKeyRange span;
if (folder is FdbDirectorySubspace)
{
span = FdbKeyRange.StartsWith((folder as FdbDirectorySubspace).Copy());
log.WriteLine("Reading list of shards for /{0} under {1} ...", String.Join("/", path), FdbKey.Dump(span.Begin));
}
else
{
log.WriteLine("Reading list of shards for the whole cluster ...");
span = FdbKeyRange.All;
}
// dump keyServers
var ranges = await Fdb.System.GetChunksAsync(db, span, ct);
log.WriteLine("> Found {0:N0} shard(s)", ranges.Count);
// take a sample
var samples = new List<FdbKeyRange>();
if (ranges.Count <= 32)
{ // small enough to scan it all
samples.AddRange(ranges);
log.WriteLine("Sampling all {0:N0} shards ...", samples.Count);
}
else
{ // need to take a random subset
var rnd = new Random();
int sz = Math.Max((int)Math.Ceiling(ratio * ranges.Count), 1);
if (auto)
{
if (sz > 100) sz = 100; //SAFETY
if (sz < 32) sz = Math.Max(sz, Math.Min(32, ranges.Count));
}
var population = new List<FdbKeyRange>(ranges);
for (int i = 0; i < sz; i++)
{
int p = rnd.Next(population.Count);
samples.Add(population[p]);
population.RemoveAt(p);
}
log.WriteLine("Sampling " + samples.Count + " out of " + ranges.Count + " shards (" + (100.0 * samples.Count / ranges.Count).ToString("N1") + "%) ...");
}
log.WriteLine();
const string FORMAT_STRING = "{0,9} ║{1,10}{6,6} {2,-29} ║{3,10}{7,7} {4,-37} ║{5,10}";
const string SCALE_KEY = "....--------========########M";
const string SCALE_VAL = "....--------========########@@@@@@@@M";
log.WriteLine(FORMAT_STRING, "Count", "Keys", SCALE_KEY, "Values", SCALE_VAL, "Total", "med.", "med.");
var rangeOptions = new FdbRangeOptions { Mode = FdbStreamingMode.WantAll };
samples = samples.OrderBy(x => x.Begin).ToList();
long globalSize = 0;
long globalCount = 0;
int workers = 8; // Math.Max(4, Environment.ProcessorCount);
var sw = Stopwatch.StartNew();
var tasks = new List<Task>();
int n = samples.Count;
while (samples.Count > 0)
{
while (tasks.Count < workers && samples.Count > 0)
{
var range = samples[0];
samples.RemoveAt(0);
tasks.Add(Task.Run(async () =>
{
var kk = new RobustHistogram(RobustHistogram.TimeScale.Ticks);
var vv = new RobustHistogram(RobustHistogram.TimeScale.Ticks);
#region Method 1: get_range everything...
using (var tr = db.BeginTransaction(ct))
{
long keySize = 0;
long valueSize = 0;
long count = 0;
int iter = 0;
var beginSelector = FdbKeySelector.FirstGreaterOrEqual(range.Begin);
var endSelector = FdbKeySelector.FirstGreaterOrEqual(range.End);
while (true)
{
FdbRangeChunk data = default(FdbRangeChunk);
FdbException error = null;
try
{
data = await tr.Snapshot.GetRangeAsync(
beginSelector,
endSelector,
rangeOptions,
iter
).ConfigureAwait(false);
}
catch (FdbException e)
{
error = e;
}
if (error != null)
{
await tr.OnErrorAsync(error.Code).ConfigureAwait(false);
continue;
}
if (data.Count == 0) break;
count += data.Count;
foreach (var kvp in data.Chunk)
{
keySize += kvp.Key.Count;
valueSize += kvp.Value.Count;
kk.Add(TimeSpan.FromTicks(kvp.Key.Count));
vv.Add(TimeSpan.FromTicks(kvp.Value.Count));
}
if (!data.HasMore) break;
beginSelector = FdbKeySelector.FirstGreaterThan(data.Last.Key);
++iter;
}
long totalSize = keySize + valueSize;
Interlocked.Add(ref globalSize, totalSize);
Interlocked.Add(ref globalCount, count);
lock (log)
{
log.WriteLine(FORMAT_STRING, count.ToString("N0"), FormatSize(keySize), kk.GetDistribution(begin: 1, end: 12000, fold: 2), FormatSize(valueSize), vv.GetDistribution(begin: 1, end: 120000, fold: 2), FormatSize(totalSize), FormatSize((int)Math.Ceiling(kk.Median)), FormatSize((int)Math.Ceiling(vv.Median)));
}
}
#endregion
#region Method 2: estimate the count using key selectors...
//long counter = await Fdb.System.EstimateCountAsync(db, range, ct);
//Console.WriteLine("COUNT = " + counter.ToString("N0"));
#endregion
}, ct));
}
var done = await Task.WhenAny(tasks);
tasks.Remove(done);
}
await Task.WhenAll(tasks);
sw.Stop();
log.WriteLine();
if (n != ranges.Count)
{
log.WriteLine("Sampled " + FormatSize(globalSize) + " (" + globalSize.ToString("N0") + " bytes) and " + globalCount.ToString("N0") + " keys in " + sw.Elapsed.TotalSeconds.ToString("N1") + " sec");
log.WriteLine("> Estimated total size is " + FormatSize(globalSize * ranges.Count / n));
}
else
{
log.WriteLine("Found " + FormatSize(globalSize) + " (" + globalSize.ToString("N0") + " bytes) and " + globalCount.ToString("N0") + " keys in " + sw.Elapsed.TotalSeconds.ToString("N1") + " sec");
// compare to the whole cluster
ranges = await Fdb.System.GetChunksAsync(db, FdbKey.MinValue, FdbKey.MaxValue, ct);
log.WriteLine("> This directory contains ~{0:N2}% of all data", (100.0 * n / ranges.Count));
}
log.WriteLine();
}
/// <summary>Read a single page of a range query from the database, using a dedicated transaction.</summary>
/// <param name="db">Database instance</param>
/// <remarks>
/// Use this method only if you intend to perform a single operation inside your execution context (ex: HTTP request).
/// If you need to combine multiple read or write operations, consider using on of the multiple <see cref="IFdbReadOnlyRetryable.ReadAsync"/> or <see cref="IFdbRetryable.ReadWriteAsync"/> overrides.
/// </remarks>
public static Task <FdbRangeChunk> GetRangeAsync([NotNull] this IFdbReadOnlyRetryable db, KeySelector beginInclusive, KeySelector endExclusive, FdbRangeOptions options, int iteration, CancellationToken ct)
{
Contract.NotNull(db, nameof(db));
return(db.ReadAsync((tr) => tr.GetRangeAsync(beginInclusive, endExclusive, options, iteration), ct));
}
public override Task<FdbRangeChunk> GetRangeAsync(FdbKeySelector beginInclusive, FdbKeySelector endExclusive, FdbRangeOptions options = null, int iteration = 0)
{
throw new NotImplementedException();
}
internal FdbRangeQuery <KeyValuePair <Slice, Slice> > GetRangeCore(FdbKeySelector begin, FdbKeySelector end, FdbRangeOptions options, bool snapshot)
{
this.Database.EnsureKeyIsValid(begin.Key);
this.Database.EnsureKeyIsValid(end.Key, endExclusive: true);
options = FdbRangeOptions.EnsureDefaults(options, null, null, FdbStreamingMode.Iterator, false);
options.EnsureLegalValues();
#if DEBUG
if (Logging.On && Logging.IsVerbose)
{
Logging.Verbose(this, "GetRangeCore", String.Format("Getting range '{0} <= x < {1}'", begin.ToString(), end.ToString()));
}
#endif
return(new FdbRangeQuery <KeyValuePair <Slice, Slice> >(this, begin, end, TaskHelpers.Cache <KeyValuePair <Slice, Slice> > .Identity, snapshot, options));
}
