public static async Task Shards(string[] path, IFdbTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
var ranges = await Fdb.System.GetChunksAsync(db, FdbKey.MinValue, FdbKey.MaxValue, ct);
Console.WriteLine("Found {0} shards in the whole cluster", ranges.Count);
// look if there is something under there
var folder = (await TryOpenCurrentDirectoryAsync(path, db, ct)) as FdbDirectorySubspace;
if (folder != null)
{
var r = FdbKeyRange.StartsWith(folder.Copy().Key);
Console.WriteLine("Searching for shards that intersect with /{0} ...", String.Join("/", path));
ranges = await Fdb.System.GetChunksAsync(db, r, ct);
Console.WriteLine("Found {0} ranges intersecting {1}:", ranges.Count, r);
var last = Slice.Empty;
foreach (var range in ranges)
{
Console.Write("> " + FdbKey.Dump(range.Begin) + " ...");
long count = await Fdb.System.EstimateCountAsync(db, range, ct);
Console.WriteLine(" {0:N0}", count);
last = range.End;
//TODO: we can probably get more details on this shard looking in the system keyspace (where it is, how many replicas, ...)
}
Console.WriteLine("> ... " + FdbKey.Dump(last));
}
//Console.WriteLine("Found " + ranges.Count + " shards in the cluster");
//TODO: shards that intersect the current directory
}
/// <summary>Return all fields of an hashset</summary>
/// <param name="trans">Transaction that will be used for this request</param>
/// <param name="id">Unique identifier of the hashset</param>
/// <returns>Dictionary containing, for all fields, their associated values</returns>
public async Task <IDictionary <string, Slice> > GetAsync(IFdbReadOnlyTransaction trans, IFdbTuple id)
{
if (trans == null)
{
throw new ArgumentNullException("trans");
}
if (id == null)
{
throw new ArgumentNullException("id");
}
var prefix = GetKey(id);
var results = new Dictionary <string, Slice>(StringComparer.OrdinalIgnoreCase);
await trans
.GetRange(FdbKeyRange.StartsWith(prefix))
.ForEachAsync((kvp) =>
{
string field = this.Subspace.UnpackLast <string>(kvp.Key);
results[field] = kvp.Value;
})
.ConfigureAwait(false);
return(results);
}
/// <summary>Remove all the values for a specific key</summary>
/// <param name="trans"></param>
/// <param name="key"></param>
/// <returns></returns>
public void Remove([NotNull] IFdbTransaction trans, TKey key)
{
if (trans == null)
{
throw new ArgumentNullException("trans");
}
trans.ClearRange(FdbKeyRange.StartsWith(this.Location.Partial.Keys.Encode(key)));
}
protected virtual Task <List <Slice> > LoadPartsAsync(IFdbReadOnlyTransaction trans, TId id)
{
var key = this.Location.Partial.EncodeKey(id);
return(trans
.GetRange(FdbKeyRange.StartsWith(key)) //TODO: options ?
.Select(kvp => kvp.Value)
.ToListAsync());
}
private void ClearTask(IFdbTransaction tr, Slice taskId)
{
tr.Annotate("Deleting task {0}", taskId.ToAsciiOrHexaString());
// clear all metadata about the task
tr.ClearRange(FdbKeyRange.StartsWith(this.TaskStore.Pack(taskId)));
// decrement pending number of tasks
this.Counters.Decrement(tr, COUNTER_PENDING_TASKS);
}
public FdbRangeQuery <TId> Lookup(IFdbReadOnlyTransaction trans, TValue value, bool reverse = false)
{
var prefix = this.Location.Partial.Keys.Encode(value);
return(trans
.GetRange(FdbKeyRange.StartsWith(prefix), new FdbRangeOptions {
Reverse = reverse
})
.Select((kvp) => this.Location.Keys.Decode(kvp.Key).Item2));
}
public void Test_FdbKey_PrettyPrint()
{
// verify that the pretty printing of keys produce a user friendly output
Assert.That(FdbKey.Dump(Slice.Nil), Is.EqualTo("<null>"));
Assert.That(FdbKey.Dump(Slice.Empty), Is.EqualTo("<empty>"));
Assert.That(FdbKey.Dump(Slice.FromByte(0)), Is.EqualTo("<00>"));
Assert.That(FdbKey.Dump(Slice.FromByte(255)), Is.EqualTo("<FF>"));
Assert.That(FdbKey.Dump(Slice.Create(new byte[] { 0, 1, 2, 3, 4, 5, 6, 7 })), Is.EqualTo("<00><01><02><03><04><05><06><07>"));
Assert.That(FdbKey.Dump(Slice.Create(new byte[] { 255, 254, 253, 252, 251, 250, 249, 248 })), Is.EqualTo("<FF><FE><FD><FC><FB><FA><F9><F8>"));
Assert.That(FdbKey.Dump(Slice.FromString("hello")), Is.EqualTo("hello"));
Assert.That(FdbKey.Dump(Slice.FromString("héllø")), Is.EqualTo("h<C3><A9>ll<C3><B8>"));
// tuples should be decoded properly
Assert.That(FdbKey.Dump(FdbTuple.Pack(123)), Is.EqualTo("(123,)"), "Singleton tuples should end with a ','");
Assert.That(FdbKey.Dump(FdbTuple.Pack(Slice.FromAscii("hello"))), Is.EqualTo("('hello',)"), "ASCII strings should use single quotes");
Assert.That(FdbKey.Dump(FdbTuple.Pack("héllø")), Is.EqualTo("(\"héllø\",)"), "Unicode strings should use double quotes");
Assert.That(FdbKey.Dump(FdbTuple.Pack(Slice.Create(new byte[] { 1, 2, 3 }))), Is.EqualTo("(<01 02 03>,)"));
Assert.That(FdbKey.Dump(FdbTuple.Pack(123, 456)), Is.EqualTo("(123, 456)"), "Elements should be separated with a space, and not end up with ','");
Assert.That(FdbKey.Dump(FdbTuple.Pack(true, false, default(object))), Is.EqualTo("(1, 0, null)"), "Booleans should be displayed as numbers, and null should be in lowercase"); //note: even though it's tempting to using Python's "Nil", it's not very ".NETty"
Assert.That(FdbKey.Dump(FdbTuple.Pack(1.0d, Math.PI, Math.E)), Is.EqualTo("(1, 3.1415926535897931, 2.7182818284590451)"), "Doubles should used dot and have full precision (17 digits)");
Assert.That(FdbKey.Dump(FdbTuple.Pack(1.0f, (float)Math.PI, (float)Math.E)), Is.EqualTo("(1, 3.14159274, 2.71828175)"), "Singles should used dot and have full precision (10 digits)");
var guid = Guid.NewGuid();
Assert.That(FdbKey.Dump(FdbTuple.Pack(guid)), Is.EqualTo(String.Format("({0},)", guid.ToString("D"))), "GUIDs should be displayed as a string literal, without quotes");
var uuid128 = Uuid128.NewUuid();
Assert.That(FdbKey.Dump(FdbTuple.Pack(uuid128)), Is.EqualTo(String.Format("({0},)", uuid128.ToString("D"))), "Uuid128s should be displayed as a string literal, without quotes");
var uuid64 = Uuid64.NewUuid();
Assert.That(FdbKey.Dump(FdbTuple.Pack(uuid64)), Is.EqualTo(String.Format("({0},)", uuid64.ToString("D"))), "Uuid64s should be displayed as a string literal, without quotes");
// ranges should be decoded when possible
var key = FdbTuple.ToRange(FdbTuple.Create("hello"));
// "<02>hello<00><00>" .. "<02>hello<00><FF>"
Assert.That(FdbKey.PrettyPrint(key.Begin, FdbKey.PrettyPrintMode.Begin), Is.EqualTo("(\"hello\",).<00>"));
Assert.That(FdbKey.PrettyPrint(key.End, FdbKey.PrettyPrintMode.End), Is.EqualTo("(\"hello\",).<FF>"));
key = FdbKeyRange.StartsWith(FdbTuple.Pack("hello"));
// "<02>hello<00>" .. "<02>hello<01>"
Assert.That(FdbKey.PrettyPrint(key.Begin, FdbKey.PrettyPrintMode.Begin), Is.EqualTo("(\"hello\",)"));
Assert.That(FdbKey.PrettyPrint(key.End, FdbKey.PrettyPrintMode.End), Is.EqualTo("(\"hello\",) + 1"));
var t = FdbTuple.Pack(123);
Assert.That(FdbKey.PrettyPrint(t, FdbKey.PrettyPrintMode.Single), Is.EqualTo("(123,)"));
Assert.That(FdbKey.PrettyPrint(FdbTuple.ToRange(t).Begin, FdbKey.PrettyPrintMode.Begin), Is.EqualTo("(123,).<00>"));
Assert.That(FdbKey.PrettyPrint(FdbTuple.ToRange(t).End, FdbKey.PrettyPrintMode.End), Is.EqualTo("(123,).<FF>"));
}
private Task Scenario2(IFdbTransaction tr)
{
var location = FdbSubspace.Create(Slice.FromAscii("TEST"));
tr.ClearRange(FdbKeyRange.StartsWith(location.Key));
for (int i = 0; i < 10; i++)
{
tr.Set(location.Pack(i), Slice.FromString("value of " + i));
}
return(Task.FromResult <object>(null));
}
/// <summary>Remove all fields of an hashset</summary>
/// <param name="id"></param>
public void Delete(IFdbTransaction trans, IFdbTuple id)
{
if (trans == null)
{
throw new ArgumentNullException("trans");
}
if (id == null)
{
throw new ArgumentNullException("id");
}
// remove all fields of the hash
trans.ClearRange(FdbKeyRange.StartsWith(GetKey(id)));
}
/// <summary>Insert a new document in the collection</summary>
public void Insert(IFdbTransaction trans, TDocument document)
{
if (trans == null)
{
throw new ArgumentNullException("trans");
}
if (document == null)
{
throw new ArgumentNullException("document");
}
var id = this.IdSelector(document);
if (id == null)
{
throw new InvalidOperationException("Cannot insert a document with a null identifier");
}
// encode the document
var packed = this.ValueEncoder.EncodeValue(document);
// Key Prefix = ...(id,)
var key = this.Location.Partial.EncodeKey(id);
// clear previous value
trans.ClearRange(FdbKeyRange.StartsWith(key));
int remaining = packed.Count;
if (remaining <= this.ChunkSize)
{ // stored as a single element
// Key = ...(id,)
trans.Set(key, packed);
}
else
{ // splits in as many chunks as necessary
// Key = ...(id, N) where N is the chunk index (0-based)
int p = 0;
int index = 0;
while (remaining > 0)
{
int sz = Math.Max(remaining, this.ChunkSize);
this.Location.Set(trans, FdbTuple.Create(id, index), packed.Substring(p, sz));
++index;
p += sz;
remaining -= sz;
}
}
}
/// <summary>Delete a document from the collection</summary>
/// <param name="trans"></param>
/// <param name="id"></param>
public void Delete(IFdbTransaction trans, TId id)
{
if (trans == null)
{
throw new ArgumentNullException("trans");
}
if (id == null)
{
throw new ArgumentNullException("id");
}
var key = this.Location.Partial.EncodeKey(id);
trans.ClearRange(FdbKeyRange.StartsWith(key));
}
/// <summary>Delete a document from the collection</summary>
/// <param name="trans"></param>
/// <param name="ids"></param>
public void DeleteMultiple(IFdbTransaction trans, IEnumerable <TId> ids)
{
if (trans == null)
{
throw new ArgumentNullException("trans");
}
if (ids == null)
{
throw new ArgumentNullException("ids");
}
foreach (var key in this.Location.Partial.EncodeKeyRange(ids))
{
trans.ClearRange(FdbKeyRange.StartsWith(key));
}
}
public decimal Count()
{
decimal total = 0M;
using (var tx = _le.BeginTransaction(TransactionBeginFlags.ReadOnly)) {
var prefix = FdbTuple.Create((byte)Tables.Quantity).ToSlice();
var range = FdbKeyRange.StartsWith(prefix);
var nums = InternalScan(tx, range, (slice, bytes) => ToDecimal(bytes));
foreach (var num in nums)
{
total += num;
}
}
return(total);
}
public IFdbAsyncEnumerable <KeyValuePair <TValue, long> > GetCounts([NotNull] IFdbReadOnlyTransaction trans, TKey key)
{
var range = FdbKeyRange.StartsWith(this.Location.Partial.Keys.Encode(key));
var query = trans
.GetRange(range)
.Select(kvp => new KeyValuePair <TValue, long>(this.Location.Keys.Decode(kvp.Key).Item2, kvp.Value.ToInt64()));
if (this.AllowNegativeValues)
{
return(query);
}
else
{
return(query.Where(kvp => kvp.Value > 0));
}
}
public void Test_FdbKeyRange_StartsWith()
{
FdbKeyRange range;
// "abc" => [ "abc", "abd" )
range = FdbKeyRange.StartsWith(Slice.FromAscii("abc"));
Assert.That(range.Begin, Is.EqualTo(Slice.FromAscii("abc")));
Assert.That(range.End, Is.EqualTo(Slice.FromAscii("abd")));
// "" => ArgumentException
Assert.That(() => FdbKeyRange.PrefixedBy(Slice.Empty), Throws.InstanceOf <ArgumentException>());
// "\xFF" => ArgumentException
Assert.That(() => FdbKeyRange.PrefixedBy(Slice.FromAscii("\xFF")), Throws.InstanceOf <ArgumentException>());
// null => ArgumentException
Assert.That(() => FdbKeyRange.PrefixedBy(Slice.Nil), Throws.InstanceOf <ArgumentException>());
}
static IEnumerable <TOut> InternalScan <TOut>(Tx tx, byte[] key,
Func <Slice, byte[], TOut> convert, string traceName, int skip = 0)
{
tx.TraceStart(traceName);
ushort x = 0;
var slice = Slice.Create(key);
var range = FdbKeyRange.StartsWith(slice);
using (var c = tx.CreateCursor()) {
if (!c.MoveToFirstAfter(key))
{
tx.TraceStop(traceName, x);
yield break;
}
var pair = c.Current;
for (var i = 0; i < int.MaxValue; i++)
{
var current = Slice.Create(pair.Key);
if (!range.Contains(current))
{
tx.TraceStop(traceName, x);
break;
}
if (i >= skip)
{
x += 1;
yield return(convert(current, pair.Value));
}
if (!c.MoveNext())
{
tx.TraceStop(traceName, x);
break;
}
pair = c.Current;
}
}
}
public static void DeleteRange(Tx tx, byte[] key, string traceName)
{
tx.TraceStart(traceName);
ushort x = 0;
var slice = Slice.Create(key);
var range = FdbKeyRange.StartsWith(slice);
using (var c = tx.CreateCursor()) {
if (!c.MoveToFirstAfter(key))
{
tx.TraceStop(traceName, x);
return;
}
var pair = c.Current;
for (var i = 0; i < int.MaxValue; i++)
{
var current = Slice.Create(pair.Key);
if (!range.Contains(current))
{
tx.TraceStop(traceName, x);
break;
}
x += 1;
tx.Delete(pair.Key);
if (!c.MoveNext())
{
tx.TraceStop(traceName, x);
break;
}
pair = c.Current;
}
}
tx.TraceStop(traceName);
}
/// <summary>Return the list the names of all fields of an hashset</summary>
/// <param name="trans">Transaction that will be used for this request</param>
/// <param name="id">Unique identifier of the hashset</param>
/// <returns>List of all fields. If the list is empty, the hashset does not exist</returns>
public Task <List <string> > GetKeys(IFdbReadOnlyTransaction trans, IFdbTuple id, CancellationToken cancellationToken = default(CancellationToken))
{
//note: As of Beta2, FDB does not have a fdb_get_range that only return the keys. That means that we will have to also read the values from the db, in order to just get the names of the fields :(
//TODO: find a way to optimize this ?
if (trans == null)
{
throw new ArgumentNullException("trans");
}
if (id == null)
{
throw new ArgumentNullException("id");
}
var prefix = GetKey(id);
var results = new Dictionary <string, Slice>(StringComparer.OrdinalIgnoreCase);
return(trans
.GetRange(FdbKeyRange.StartsWith(prefix))
.Select((kvp) => ParseFieldKey(FdbTuple.Unpack(kvp.Key)))
.ToListAsync(cancellationToken));
}
public IFdbAsyncEnumerable <TValue> Get([NotNull] IFdbReadOnlyTransaction trans, TKey key)
{
if (trans == null)
{
throw new ArgumentNullException("trans");
}
var range = FdbKeyRange.StartsWith(this.Location.Partial.Keys.Encode(key));
if (this.AllowNegativeValues)
{
return(trans
.GetRange(range)
.Select(kvp => this.Location.Keys.Decode(kvp.Key).Item2));
}
else
{
return(trans
.GetRange(range)
.Where(kvp => kvp.Value.ToInt64() > 0) // we need to filter out zero or negative values (possible artefacts)
.Select(kvp => this.Location.Keys.Decode(kvp.Key).Item2));
}
}
public static async Task DumpSubspace([NotNull] IFdbReadOnlyTransaction tr, [NotNull] FdbSubspace subspace)
{
Assert.That(tr, Is.Not.Null);
Console.WriteLine("Dumping content of subspace " + subspace.ToString() + " :");
int count = 0;
await tr
.GetRange(FdbKeyRange.StartsWith(subspace.Key))
.ForEachAsync((kvp) =>
{
var key = subspace.Extract(kvp.Key);
++count;
string keyDump = null;
try
{
// attemps decoding it as a tuple
keyDump = key.ToTuple().ToString();
}
catch (Exception)
{
// not a tuple, dump as bytes
keyDump = "'" + key.ToString() + "'";
}
Console.WriteLine("- " + keyDump + " = " + kvp.Value.ToString());
});
if (count == 0)
{
Console.WriteLine("> empty !");
}
else
{
Console.WriteLine("> Found " + count + " values");
}
}
public async Task Test_Can_Log_A_Transaction()
{
const int N = 10;
using (var db = await OpenTestPartitionAsync())
{
var location = await GetCleanDirectory(db, "Logging");
// note: ensure that all methods are JITed
await db.ReadWriteAsync(async (tr) =>
{
await tr.GetReadVersionAsync();
tr.Set(location.Pack("Warmup", 0), Slice.FromInt32(1));
tr.Clear(location.Pack("Warmup", 1));
await tr.GetAsync(location.Pack("Warmup", 2));
await tr.GetRange(FdbKeyRange.StartsWith(location.Pack("Warmup", 3))).ToListAsync();
tr.ClearRange(location.Pack("Warmup", 4), location.Pack("Warmup", 5));
}, this.Cancellation);
await db.WriteAsync((tr) =>
{
var rnd = new Random();
tr.Set(location.Pack("One"), Slice.FromString("111111"));
tr.Set(location.Pack("Two"), Slice.FromString("222222"));
for (int j = 0; j < 4; j++)
{
for (int i = 0; i < 100; i++)
{
tr.Set(location.Pack("Range", j, rnd.Next(1000)), Slice.Empty);
}
}
for (int j = 0; j < N; j++)
{
tr.Set(location.Pack("X", j), Slice.FromInt32(j));
tr.Set(location.Pack("Y", j), Slice.FromInt32(j));
tr.Set(location.Pack("Z", j), Slice.FromInt32(j));
tr.Set(location.Pack("W", j), Slice.FromInt32(j));
}
}, this.Cancellation);
bool first = true;
Action <FdbLoggedTransaction> logHandler = (tr) =>
{
if (first)
{
Console.WriteLine(tr.Log.GetCommandsReport());
first = false;
}
Console.WriteLine(tr.Log.GetTimingsReport(true));
};
// create a logged version of the database
var logged = new FdbLoggedDatabase(db, false, false, logHandler);
for (int k = 0; k < N; k++)
{
Console.WriteLine("==== " + k + " ==== ");
Console.WriteLine();
await logged.ReadWriteAsync(async (tr) =>
{
Assert.That(tr, Is.InstanceOf <FdbLoggedTransaction>());
//tr.SetOption(FdbTransactionOption.CausalReadRisky);
long ver = await tr.GetReadVersionAsync().ConfigureAwait(false);
await tr.GetAsync(location.Pack("One")).ConfigureAwait(false);
await tr.GetAsync(location.Pack("NotFound")).ConfigureAwait(false);
tr.Set(location.Pack("Write"), Slice.FromString("abcdef" + k.ToString()));
//tr.Annotate("BEFORE");
//await Task.Delay(TimeSpan.FromMilliseconds(10));
//tr.Annotate("AFTER");
//await tr.Snapshot.GetAsync(location.Pack("Snap")).ConfigureAwait(false);
tr.Annotate("This is a comment");
//await tr.GetRangeAsync(FdbKeySelector.LastLessOrEqual(location.Pack("A")), FdbKeySelector.FirstGreaterThan(location.Pack("Z"))).ConfigureAwait(false);
await Task.WhenAll(
tr.GetRange(FdbKeyRange.StartsWith(location.Pack("Range", 0))).ToListAsync(),
tr.GetRange(location.Pack("Range", 1, 0), location.Pack("Range", 1, 200)).ToListAsync(),
tr.GetRange(location.Pack("Range", 2, 400), location.Pack("Range", 2, 600)).ToListAsync(),
tr.GetRange(location.Pack("Range", 3, 800), location.Pack("Range", 3, 1000)).ToListAsync()
).ConfigureAwait(false);
await tr.GetAsync(location.Pack("Two")).ConfigureAwait(false);
await tr.GetValuesAsync(Enumerable.Range(0, N).Select(x => location.Pack("X", x))).ConfigureAwait(false);
for (int i = 0; i < N; i++)
{
await tr.GetAsync(location.Pack("Z", i)).ConfigureAwait(false);
}
await Task.WhenAll(Enumerable.Range(0, N / 2).Select(x => tr.GetAsync(location.Pack("Y", x)))).ConfigureAwait(false);
await Task.WhenAll(Enumerable.Range(N / 2, N / 2).Select(x => tr.GetAsync(location.Pack("Y", x)))).ConfigureAwait(false);
await Task.WhenAll(
tr.GetAsync(location.Pack("W", 1)),
tr.GetAsync(location.Pack("W", 2)),
tr.GetAsync(location.Pack("W", 3))
).ConfigureAwait(false);
tr.Set(location.Pack("Write2"), Slice.FromString("ghijkl" + k.ToString()));
tr.Clear(location.Pack("Clear", "0"));
tr.ClearRange(location.Pack("Clear", "A"), location.Pack("Clear", "Z"));
if (tr.Context.Retries == 0)
{
// make it fail
//throw new FdbException(FdbError.PastVersion, "fake timeout");
}
}, this.Cancellation);
}
}
}
public async Task Run(IFdbDatabase db, TextWriter log, CancellationToken ct)
{
// estimate the number of machines...
Console.WriteLine("# Detecting cluster topology...");
var servers = await db.QueryAsync(tr => tr
.WithReadAccessToSystemKeys()
.GetRange(FdbKeyRange.StartsWith(Fdb.System.ServerList))
.Select(kvp => new
{
Node = kvp.Value.Substring(8, 16).ToHexaString(),
Machine = kvp.Value.Substring(24, 16).ToHexaString(),
DataCenter = kvp.Value.Substring(40, 16).ToHexaString()
}),
ct
);
var numNodes = servers.Select(s => s.Node).Distinct().Count();
var numMachines = servers.Select(s => s.Machine).Distinct().Count();
var numDCs = servers.Select(s => s.DataCenter).Distinct().Count();
Console.WriteLine("# > Found " + numNodes + " process(es) on " + numMachines + " machine(s) in " + numDCs + " datacenter(s)");
Console.WriteLine("# Reading list of shards...");
// dump keyServers
var ranges = await Fdb.System.GetChunksAsync(db, FdbKey.MinValue, FdbKey.MaxValue, ct);
Console.WriteLine("# > Found " + ranges.Count + " shards:");
// take a sample
var rnd = new Random(1234);
int sz = Math.Max((int)Math.Ceiling(this.Ratio * ranges.Count), 1);
if (sz > 500)
{
sz = 500; //SAFETY
}
if (sz < 50)
{
sz = Math.Max(sz, Math.Min(50, ranges.Count));
}
var samples = new List <FdbKeyRange>();
for (int i = 0; i < sz; i++)
{
int p = rnd.Next(ranges.Count);
samples.Add(ranges[p]);
ranges.RemoveAt(p);
}
Console.WriteLine("# Sampling " + sz + " out of " + ranges.Count + " shards (" + (100.0 * sz / ranges.Count).ToString("N1") + "%) ...");
Console.WriteLine("{0,9}{1,10}{2,10}{3,10} : K+V size distribution", "Count", "Keys", "Values", "Total");
var rangeOptions = new FdbRangeOptions {
Mode = FdbStreamingMode.WantAll
};
samples = samples.OrderBy(x => x.Begin).ToList();
long total = 0;
int workers = Math.Min(numMachines, 8);
var sw = Stopwatch.StartNew();
var tasks = new List <Task>();
while (samples.Count > 0)
{
while (tasks.Count < workers && samples.Count > 0)
{
var range = samples[0];
samples.RemoveAt(0);
tasks.Add(Task.Run(async() =>
{
var hh = new RobustHistogram(RobustHistogram.TimeScale.Ticks);
#region Method 1: get_range everything...
using (var tr = db.BeginTransaction(ct))
{
long keySize = 0;
long valueSize = 0;
long count = 0;
int iter = 0;
var beginSelector = FdbKeySelector.FirstGreaterOrEqual(range.Begin);
var endSelector = FdbKeySelector.FirstGreaterOrEqual(range.End);
while (true)
{
FdbRangeChunk data = default(FdbRangeChunk);
FdbException error = null;
try
{
data = await tr.Snapshot.GetRangeAsync(
beginSelector,
endSelector,
rangeOptions,
iter
).ConfigureAwait(false);
}
catch (FdbException e)
{
error = e;
}
if (error != null)
{
await tr.OnErrorAsync(error.Code).ConfigureAwait(false);
continue;
}
if (data.Count == 0)
{
break;
}
count += data.Count;
foreach (var kvp in data.Chunk)
{
keySize += kvp.Key.Count;
valueSize += kvp.Value.Count;
hh.Add(TimeSpan.FromTicks(kvp.Key.Count + kvp.Value.Count));
}
if (!data.HasMore)
{
break;
}
beginSelector = FdbKeySelector.FirstGreaterThan(data.Last.Key);
++iter;
}
long totalSize = keySize + valueSize;
Interlocked.Add(ref total, totalSize);
Console.WriteLine("{0,9}{1,10}{2,10}{3,10} : {4}", count.ToString("N0"), FormatSize(keySize), FormatSize(valueSize), FormatSize(totalSize), hh.GetDistribution(begin: 1, end: 10000, fold: 2));
}
#endregion
#region Method 2: estimate the count using key selectors...
//long counter = await Fdb.System.EstimateCountAsync(db, range, ct);
//Console.WriteLine("COUNT = " + counter.ToString("N0"));
#endregion
}, ct));
}
var done = await Task.WhenAny(tasks);
tasks.Remove(done);
}
await Task.WhenAll(tasks);
sw.Stop();
Console.WriteLine("> Sampled " + FormatSize(total) + " (" + total.ToString("N0") + " bytes) in " + sw.Elapsed.TotalSeconds.ToString("N1") + " sec");
Console.WriteLine("> Estimated total size is " + FormatSize(total * ranges.Count / sz));
}
private async Task RunAsync(IFdbDatabase db, FdbSubspace location, CancellationToken ct, Action done, int N, int K, int W)
{
if (db == null)
{
throw new ArgumentNullException("db");
}
StringBuilder sb = new StringBuilder();
db = new FdbLoggedDatabase(db, false, false, (log) =>
{
sb.AppendLine(log.Log.GetTimingsReport(true));
//Console.WriteLine(log.Log.GetTimingsReport(true));
});
try
{
var workerPool = new FdbWorkerPool(location);
Console.WriteLine("workerPool at " + location.Key.ToAsciiOrHexaString());
var workerSignal = new AsyncCancelableMutex(ct);
var clientSignal = new AsyncCancelableMutex(ct);
int taskCounter = 0;
int msgSent = 0;
int msgReceived = 0;
Func <FdbWorkerMessage, CancellationToken, Task> handler = async(msg, _ct) =>
{
Interlocked.Increment(ref msgReceived);
//await Task.Delay(10 + Math.Abs(msg.Id.GetHashCode()) % 50);
await Task.Delay(10).ConfigureAwait(false);
};
Func <int, Task> worker = async(id) =>
{
await workerSignal.Task.ConfigureAwait(false);
Console.WriteLine("Worker #" + id + " is starting");
try
{
await workerPool.RunWorkerAsync(db, handler, ct).ConfigureAwait(false);
}
finally
{
Console.WriteLine("Worker #" + id + " has stopped");
}
};
Func <int, Task> client = async(id) =>
{
await clientSignal.Task.ConfigureAwait(false);
await Task.Delay(10).ConfigureAwait(false);
var rnd = new Random(id * 111);
for (int i = 0; i < N; i++)
{
var taskId = Slice.FromString("T" + Interlocked.Increment(ref taskCounter));
var taskBody = Slice.FromString("Message " + (i + 1) + " of " + N + " from client #" + id);
await workerPool.ScheduleTaskAsync(db, taskId, taskBody, ct).ConfigureAwait(false);
Interlocked.Increment(ref msgSent);
//if (i > 0 && i % 10 == 0) Console.WriteLine("@@@ Client#" + id + " pushed " + (i + 1) + " / " + N + " messages");
switch (rnd.Next(5))
{
case 0: await Task.Delay(10).ConfigureAwait(false); break;
case 1: await Task.Delay(100).ConfigureAwait(false); break;
case 2: await Task.Delay(500).ConfigureAwait(false); break;
}
}
Console.WriteLine("@@@ Client#" + id + " has finished!");
};
Func <string, Task> dump = async(label) =>
{
Console.WriteLine("<dump label='" + label + "' key='" + location.Key.ToAsciiOrHexaString() + "'>");
using (var tr = db.BeginTransaction(ct))
{
await tr.Snapshot
.GetRange(FdbKeyRange.StartsWith(location.Key))
.ForEachAsync((kvp) =>
{
Console.WriteLine(" - " + FdbTuple.Unpack(location.Extract(kvp.Key)) + " = " + kvp.Value.ToAsciiOrHexaString());
}).ConfigureAwait(false);
}
Console.WriteLine("</dump>");
};
var workers = Enumerable.Range(0, W).Select((i) => worker(i)).ToArray();
var clients = Enumerable.Range(0, K).Select((i) => client(i)).ToArray();
DateTime start = DateTime.Now;
DateTime last = start;
int lastHandled = -1;
using (var timer = new Timer((_) =>
{
var now = DateTime.Now;
Console.WriteLine("@@@ T=" + now.Subtract(start) + ", sent: " + msgSent.ToString("N0") + ", recv: " + msgReceived.ToString("N0"));
Console.WriteLine("### Workers: " + workerPool.IdleWorkers + " / " + workerPool.ActiveWorkers + " (" + new string('#', workerPool.IdleWorkers) + new string('.', workerPool.ActiveWorkers - workerPool.IdleWorkers) + "), sent: " + workerPool.MessageScheduled.ToString("N0") + ", recv: " + workerPool.MessageReceived.ToString("N0") + ", delta: " + (workerPool.MessageScheduled - workerPool.MessageReceived).ToString("N0") + ", busy: " + workerPool.WorkerBusyTime + " (avg " + workerPool.WorkerAverageBusyDuration.TotalMilliseconds.ToString("N3") + " ms)");
if (now.Subtract(last).TotalSeconds >= 10)
{
//dump("timer").GetAwaiter().GetResult();
last = now;
if (lastHandled == msgReceived)
{ // STALL ?
Console.WriteLine("STALL! ");
done();
}
lastHandled = msgReceived;
}
if (msgReceived >= K * N)
{
dump("complete").GetAwaiter().GetResult();
done();
}
}, null, 1000, 1000))
{
var sw = Stopwatch.StartNew();
// start the workers
workerSignal.Set(async: true);
await Task.Delay(500);
await dump("workers started");
// start the clients
clientSignal.Set(async: true);
await Task.WhenAll(clients);
Console.WriteLine("Clients completed after " + sw.Elapsed);
await Task.WhenAll(workers);
Console.WriteLine("Workers completed after " + sw.Elapsed);
}
}
finally
{
Console.WriteLine("---------------------------------------------------------------------------");
Console.WriteLine("Transaction logs:");
Console.WriteLine();
Console.WriteLine(sb.ToString());
}
}
public static async Task Sampling(string[] path, IFdbTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
double ratio = 0.1d;
bool auto = true;
if (extras.Count > 0)
{
double x = extras.Get <double>(0);
if (x > 0 && x <= 1)
{
ratio = x;
}
auto = false;
}
var folder = await TryOpenCurrentDirectoryAsync(path, db, ct);
FdbKeyRange span;
if (folder is FdbDirectorySubspace)
{
span = FdbKeyRange.StartsWith((folder as FdbDirectorySubspace).Copy());
log.WriteLine("Reading list of shards for /{0} under {1} ...", String.Join("/", path), FdbKey.Dump(span.Begin));
}
else
{
log.WriteLine("Reading list of shards for the whole cluster ...");
span = FdbKeyRange.All;
}
// dump keyServers
var ranges = await Fdb.System.GetChunksAsync(db, span, ct);
log.WriteLine("> Found {0:N0} shard(s)", ranges.Count);
// take a sample
var samples = new List <FdbKeyRange>();
if (ranges.Count <= 32)
{ // small enough to scan it all
samples.AddRange(ranges);
log.WriteLine("Sampling all {0:N0} shards ...", samples.Count);
}
else
{ // need to take a random subset
var rnd = new Random();
int sz = Math.Max((int)Math.Ceiling(ratio * ranges.Count), 1);
if (auto)
{
if (sz > 100)
{
sz = 100; //SAFETY
}
if (sz < 32)
{
sz = Math.Max(sz, Math.Min(32, ranges.Count));
}
}
var population = new List <FdbKeyRange>(ranges);
for (int i = 0; i < sz; i++)
{
int p = rnd.Next(population.Count);
samples.Add(population[p]);
population.RemoveAt(p);
}
log.WriteLine("Sampling " + samples.Count + " out of " + ranges.Count + " shards (" + (100.0 * samples.Count / ranges.Count).ToString("N1") + "%) ...");
}
log.WriteLine();
const string FORMAT_STRING = "{0,9} ║{1,10}{6,6} {2,-29} ║{3,10}{7,7} {4,-37} ║{5,10}";
const string SCALE_KEY = "....--------========########M";
const string SCALE_VAL = "....--------========########@@@@@@@@M";
log.WriteLine(FORMAT_STRING, "Count", "Keys", SCALE_KEY, "Values", SCALE_VAL, "Total", "med.", "med.");
var rangeOptions = new FdbRangeOptions {
Mode = FdbStreamingMode.WantAll
};
samples = samples.OrderBy(x => x.Begin).ToList();
long globalSize = 0;
long globalCount = 0;
int workers = 8; // Math.Max(4, Environment.ProcessorCount);
var sw = Stopwatch.StartNew();
var tasks = new List <Task>();
int n = samples.Count;
while (samples.Count > 0)
{
while (tasks.Count < workers && samples.Count > 0)
{
var range = samples[0];
samples.RemoveAt(0);
tasks.Add(Task.Run(async() =>
{
var kk = new RobustHistogram(RobustHistogram.TimeScale.Ticks);
var vv = new RobustHistogram(RobustHistogram.TimeScale.Ticks);
#region Method 1: get_range everything...
using (var tr = db.BeginTransaction(ct))
{
long keySize = 0;
long valueSize = 0;
long count = 0;
int iter = 0;
var beginSelector = FdbKeySelector.FirstGreaterOrEqual(range.Begin);
var endSelector = FdbKeySelector.FirstGreaterOrEqual(range.End);
while (true)
{
FdbRangeChunk data = default(FdbRangeChunk);
FdbException error = null;
try
{
data = await tr.Snapshot.GetRangeAsync(
beginSelector,
endSelector,
rangeOptions,
iter
).ConfigureAwait(false);
}
catch (FdbException e)
{
error = e;
}
if (error != null)
{
await tr.OnErrorAsync(error.Code).ConfigureAwait(false);
continue;
}
if (data.Count == 0)
{
break;
}
count += data.Count;
foreach (var kvp in data.Chunk)
{
keySize += kvp.Key.Count;
valueSize += kvp.Value.Count;
kk.Add(TimeSpan.FromTicks(kvp.Key.Count));
vv.Add(TimeSpan.FromTicks(kvp.Value.Count));
}
if (!data.HasMore)
{
break;
}
beginSelector = FdbKeySelector.FirstGreaterThan(data.Last.Key);
++iter;
}
long totalSize = keySize + valueSize;
Interlocked.Add(ref globalSize, totalSize);
Interlocked.Add(ref globalCount, count);
lock (log)
{
log.WriteLine(FORMAT_STRING, count.ToString("N0"), FormatSize(keySize), kk.GetDistribution(begin: 1, end: 12000, fold: 2), FormatSize(valueSize), vv.GetDistribution(begin: 1, end: 120000, fold: 2), FormatSize(totalSize), FormatSize((int)Math.Ceiling(kk.Median)), FormatSize((int)Math.Ceiling(vv.Median)));
}
}
#endregion
#region Method 2: estimate the count using key selectors...
//long counter = await Fdb.System.EstimateCountAsync(db, range, ct);
//Console.WriteLine("COUNT = " + counter.ToString("N0"));
#endregion
}, ct));
}
var done = await Task.WhenAny(tasks);
tasks.Remove(done);
}
await Task.WhenAll(tasks);
sw.Stop();
log.WriteLine();
if (n != ranges.Count)
{
log.WriteLine("Sampled " + FormatSize(globalSize) + " (" + globalSize.ToString("N0") + " bytes) and " + globalCount.ToString("N0") + " keys in " + sw.Elapsed.TotalSeconds.ToString("N1") + " sec");
log.WriteLine("> Estimated total size is " + FormatSize(globalSize * ranges.Count / n));
}
else
{
log.WriteLine("Found " + FormatSize(globalSize) + " (" + globalSize.ToString("N0") + " bytes) and " + globalCount.ToString("N0") + " keys in " + sw.Elapsed.TotalSeconds.ToString("N1") + " sec");
// compare to the whole cluster
ranges = await Fdb.System.GetChunksAsync(db, FdbKey.MinValue, FdbKey.MaxValue, ct);
log.WriteLine("> This directory contains ~{0:N2}% of all data", (100.0 * n / ranges.Count));
}
log.WriteLine();
}
public static async Task Map(string[] path, IFdbTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
// we want to merge the map of shards, with the map of directories from the Directory Layer, and count for each directory how many shards intersect
var folder = await TryOpenCurrentDirectoryAsync(path, db, ct);
if (folder == null)
{
log.WriteLine("# Directory not found");
return;
}
var span = folder.DirectoryLayer.ContentSubspace.ToRange();
// note: this may break in future versions of the DL! Maybe we need a custom API to get a flat list of all directories in a DL that span a specific range ?
var shards = await Fdb.System.GetChunksAsync(db, span, ct);
int totalShards = shards.Count;
log.WriteLine("Found {0} shard(s) in partition /{1}", totalShards, folder.DirectoryLayer.FullName);
log.WriteLine("Listing all directories...");
var map = new Dictionary <string, int>(StringComparer.Ordinal);
Action <string[], int> account = (p, c) =>
{
for (int i = 1; i <= p.Length; i++)
{
var s = "/" + String.Join("/", p, 0, i);
int x;
map[s] = map.TryGetValue(s, out x) ? (x + c) : c;
}
};
var work = new Stack <IFdbDirectory>();
work.Push(folder);
var dirs = new List <IFdbDirectory>();
int n = 0;
while (work.Count > 0)
{
var cur = work.Pop();
// skip sub partitions
var names = await cur.ListAsync(db, ct);
foreach (var name in names)
{
var sub = await cur.TryOpenAsync(db, name, ct);
if (sub != null)
{
var p = sub.FullName;
if (sub is FdbDirectoryPartition)
{
log.WriteLine("\r! Skipping partition {0} ", sub.Name);
n = 0;
continue;
}
log.Write("\r/{0}{1}", p, p.Length > n ? String.Empty : new string(' ', n - p.Length));
n = p.Length;
work.Push(sub);
dirs.Add(sub);
}
}
}
log.Write("\r" + new string(' ', n + 2));
log.WriteLine("\r> Found {0} sub-directories", dirs.Count);
log.WriteLine();
log.WriteLine("Estimating size of each directory...");
int foundShards = 0;
n = 0;
int max = 0;
IFdbDirectory bigBad = null;
foreach (var dir in dirs)
{
log.Write("\r> {0}{1}", dir.Name, dir.Name.Length > n ? String.Empty : new string(' ', n - dir.Name.Length));
n = dir.Name.Length;
var p = dir.Path.ToArray();
var key = ((FdbSubspace)dir).Key;
// verify that the subspace has at least one key inside
var bounds = await db.ReadAsync(async (tr) =>
{
var kvs = await Task.WhenAll(
tr.GetRange(FdbKeyRange.StartsWith(key)).FirstOrDefaultAsync(),
tr.GetRange(FdbKeyRange.StartsWith(key)).LastOrDefaultAsync()
);
return(new { Min = kvs[0].Key, Max = kvs[1].Key });
}, ct);
if (bounds.Min.HasValue)
{ // folder is not empty
shards = await Fdb.System.GetChunksAsync(db, FdbKeyRange.StartsWith(key), ct);
//TODO: we still need to check if the first and last shard really intersect the subspace
// we need to check if the shards actually contain data
//Console.WriteLine("/{0} under {1} with {2} shard(s)", string.Join("/", p), FdbKey.Dump(key), shards.Count);
foundShards += shards.Count;
account(p, shards.Count);
if (shards.Count > max)
{
max = shards.Count; bigBad = dir;
}
}
else
{
account(p, 0);
}
}
log.Write("\r" + new string(' ', n + 2));
log.WriteLine("\rFound a total of {0} shard(s) in {1} folder(s)", foundShards, dirs.Count);
log.WriteLine();
log.WriteLine("Shards %Total Path");
foreach (var kvp in map.OrderBy(x => x.Key))
{
log.WriteLine("{0,6} {1,-20} {2}", kvp.Value, RobustHistogram.FormatHistoBar((double)kvp.Value / foundShards, 20), kvp.Key);
}
log.WriteLine();
if (bigBad != null)
{
log.WriteLine("Biggest folder is /{0} with {1} shards ({2:N1}% total, {3:N1}% subtree)", bigBad.FullName, max, 100.0 * max / totalShards, 100.0 * max / foundShards);
log.WriteLine();
}
}
/// <summary>Find the DCs, machines and processes in the cluster</summary>
public static async Task Topology(string[] path, IFdbTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
var coords = await Fdb.System.GetCoordinatorsAsync(db, ct);
log.WriteLine("[Cluster] {0}", coords.Id);
var servers = await db.QueryAsync(tr => tr
.WithReadAccessToSystemKeys()
.GetRange(FdbKeyRange.StartsWith(Fdb.System.ServerList))
.Select(kvp => new
{
// Offsets Size Type Name Description
// 0 2 Word Version? 0100 (1.0 ?)
// 2 4 DWord ??? 0x00 0x20 0xA2 0x00
// 6 2 Word FDBMagic 0xDB 0x0F "FDB"
// 8 16 Guid NodeId Unique Process ID
// 24 16 Guid Machine "machine_id" field in foundationdb.conf (ends with 8x0 if manually specified)
// 40 16 Guid DataCenter "datacenter_id" field in foundationdb.conf (ends with 8x0 if manually specified)
// 56 4 ??? ?? 4 x 0
// 60 12 x24 ARRAY[] ?? array of 12x the same 24-byte struct defined below
// ...0 4 DWord IPAddress 01 00 00 7F => 127.0.0.1
// ...4 4 DWord Port 94 11 00 00 -> 4500
// ...8 4 DWord ?? randomish, changes every reboot
// ..12 4 DWord ?? randomish, changes every reboot
// ..16 4 DWord Size? small L-E integer, usually between 0x20 and 0x40...
// ..20 4 DWord ?? randmoish, changes every reboot
ProcessId = kvp.Value.Substring(8, 16).ToHexaString(),
MachineId = kvp.Value.Substring(24, 16).ToHexaString(),
DataCenterId = kvp.Value.Substring(40, 16).ToHexaString(),
Parts = Enumerable.Range(0, 12).Select(i =>
{
int p = 60 + 24 * i;
return(new
{
Address = new IPAddress(kvp.Value.Substring(p, 4).GetBytes().Reverse().ToArray()),
Port = kvp.Value.Substring(p + 4, 4).ToInt32(),
Unknown1 = kvp.Value.Substring(p + 8, 4).ToInt32(),
Unknown2 = kvp.Value.Substring(p + 12, 4).ToInt32(),
Unknown3 = kvp.Value.Substring(p + 16, 4).ToInt32(),
Unknown4 = kvp.Value.Substring(p + 20, 4).ToInt32(),
});
}).ToList(),
Raw = kvp.Value,
}),
ct
);
var numNodes = servers.Select(s => s.ProcessId).Distinct().Count();
var numMachines = servers.Select(s => s.MachineId).Distinct().Count();
var numDCs = servers.Select(s => s.DataCenterId).Distinct().Count();
var dcs = servers.GroupBy(x => x.DataCenterId).ToArray();
for (int dcIndex = 0; dcIndex < dcs.Length; dcIndex++)
{
var dc = dcs[dcIndex];
bool lastDc = dcIndex == dcs.Length - 1;
string dcId = dc.Key.EndsWith("0000000000000000") ? dc.Key.Substring(0, 16) : dc.Key;
log.WriteLine((lastDc ? "`- " : "|- ") + "[DataCenter] {0} (#{1})", dcId, dcIndex);
var machines = dc.GroupBy(x => x.MachineId).ToArray();
string dcPrefix = lastDc ? " " : "| ";
for (int machineIndex = 0; machineIndex < machines.Length; machineIndex++)
{
var machine = machines[machineIndex];
var lastMachine = machineIndex == machines.Length - 1;
string machineId = machine.Key.EndsWith("0000000000000000") ? machine.Key.Substring(0, 16) : machine.Key;
log.WriteLine(dcPrefix + (lastMachine ? "`- " : "|- ") + "[Machine] {0}, {1}", machine.First().Parts[0].Address, machineId);
var procs = machine.ToArray();
string machinePrefix = dcPrefix + (lastMachine ? " " : "| ");
for (int procIndex = 0; procIndex < procs.Length; procIndex++)
{
var proc = procs[procIndex];
bool lastProc = procIndex == procs.Length - 1;
log.WriteLine(machinePrefix + (lastProc ? "`- " : "|- ") + "[Process] {0}:{1}, {2}", proc.Parts[0].Address, proc.Parts[0].Port, proc.ProcessId);
//foreach (var part in proc.Parts)
//{
// log.WriteLine(machinePrefix + "| -> {0}, {1}, {2:X8}, {3:X8}, {4}, {5:X8}", part.Address, part.Port, part.Unknown1, part.Unknown2, part.Unknown3, part.Unknown4);
//}
}
}
}
log.WriteLine();
log.WriteLine("Found {0} process(es) on {1} machine(s) in {2} datacenter(s)", numNodes, numMachines, numDCs);
log.WriteLine();
}