public void Test_FdbKeyRange_Contains()
{
FdbKeyRange range;
// ["", "")
range = FdbKeyRange.Empty;
Assert.That(range.Contains(Slice.Empty), Is.False);
Assert.That(range.Contains(Slice.FromAscii("\x00")), Is.False);
Assert.That(range.Contains(Slice.FromAscii("hello")), Is.False);
Assert.That(range.Contains(Slice.FromAscii("\xFF")), Is.False);
// ["", "\xFF" )
range = FdbKeyRange.Create(Slice.Empty, Slice.FromAscii("\xFF"));
Assert.That(range.Contains(Slice.Empty), Is.True);
Assert.That(range.Contains(Slice.FromAscii("\x00")), Is.True);
Assert.That(range.Contains(Slice.FromAscii("hello")), Is.True);
Assert.That(range.Contains(Slice.FromAscii("\xFF")), Is.False);
// ["\x00", "\xFF" )
range = FdbKeyRange.Create(Slice.FromAscii("\x00"), Slice.FromAscii("\xFF"));
Assert.That(range.Contains(Slice.Empty), Is.False);
Assert.That(range.Contains(Slice.FromAscii("\x00")), Is.True);
Assert.That(range.Contains(Slice.FromAscii("hello")), Is.True);
Assert.That(range.Contains(Slice.FromAscii("\xFF")), Is.False);
// corner cases
Assert.That(FdbKeyRange.Create(Slice.FromAscii("A"), Slice.FromAscii("A")).Contains(Slice.FromAscii("A")), Is.False, "Equal bounds");
}
public void Test_FdbKeyRange_Test()
{
const int BEFORE = -1, INSIDE = 0, AFTER = +1;
FdbKeyRange range;
// range: [ "A", "Z" )
range = FdbKeyRange.Create(Slice.FromAscii("A"), Slice.FromAscii("Z"));
// Excluding the end: < "Z"
Assert.That(range.Test(Slice.FromAscii("\x00"), endIncluded: false), Is.EqualTo(BEFORE));
Assert.That(range.Test(Slice.FromAscii("@"), endIncluded: false), Is.EqualTo(BEFORE));
Assert.That(range.Test(Slice.FromAscii("A"), endIncluded: false), Is.EqualTo(INSIDE));
Assert.That(range.Test(Slice.FromAscii("Z"), endIncluded: false), Is.EqualTo(AFTER));
Assert.That(range.Test(Slice.FromAscii("Z\x00"), endIncluded: false), Is.EqualTo(AFTER));
Assert.That(range.Test(Slice.FromAscii("\xFF"), endIncluded: false), Is.EqualTo(AFTER));
// Including the end: <= "Z"
Assert.That(range.Test(Slice.FromAscii("\x00"), endIncluded: true), Is.EqualTo(BEFORE));
Assert.That(range.Test(Slice.FromAscii("@"), endIncluded: true), Is.EqualTo(BEFORE));
Assert.That(range.Test(Slice.FromAscii("A"), endIncluded: true), Is.EqualTo(INSIDE));
Assert.That(range.Test(Slice.FromAscii("Z"), endIncluded: true), Is.EqualTo(INSIDE));
Assert.That(range.Test(Slice.FromAscii("Z\x00"), endIncluded: true), Is.EqualTo(AFTER));
Assert.That(range.Test(Slice.FromAscii("\xFF"), endIncluded: true), Is.EqualTo(AFTER));
range = FdbKeyRange.Create(FdbTuple.Pack("A"), FdbTuple.Pack("Z"));
Assert.That(range.Test(FdbTuple.Create("@")), Is.EqualTo((BEFORE)));
Assert.That(range.Test(FdbTuple.Create("A")), Is.EqualTo((INSIDE)));
Assert.That(range.Test(FdbTuple.Create("Z")), Is.EqualTo((AFTER)));
Assert.That(range.Test(FdbTuple.Create("Z"), endIncluded: true), Is.EqualTo(INSIDE));
}
/// <summary>Return all fields of an hashset</summary>
/// <param name="trans">Transaction that will be used for this request</param>
/// <param name="id">Unique identifier of the hashset</param>
/// <returns>Dictionary containing, for all fields, their associated values</returns>
public async Task <IDictionary <string, Slice> > GetAsync(IFdbReadOnlyTransaction trans, IFdbTuple id)
{
if (trans == null)
{
throw new ArgumentNullException("trans");
}
if (id == null)
{
throw new ArgumentNullException("id");
}
var prefix = GetKey(id);
var results = new Dictionary <string, Slice>(StringComparer.OrdinalIgnoreCase);
await trans
.GetRange(FdbKeyRange.StartsWith(prefix))
.ForEachAsync((kvp) =>
{
string field = this.Subspace.UnpackLast <string>(kvp.Key);
results[field] = kvp.Value;
})
.ConfigureAwait(false);
return(results);
}
IEnumerable <TOut> InternalScan <TOut>(LightningTransaction lt, FdbKeyRange range,
Func <Slice, byte[], TOut> convert)
{
ushort x = 0;
using (var c = lt.CreateCursor(_ld)) {
if (!c.MoveToFirstAfter(range.Begin.GetBytes()))
{
yield break;
}
var pair = c.Current;
for (var i = 0; i < int.MaxValue; i++)
{
var current = Slice.Create(pair.Key);
if (!range.Contains(current))
{
break;
}
x += 1;
yield return(convert(current, pair.Value));
if (!c.MoveNext())
{
break;
}
pair = c.Current;
}
}
}
public static async Task Shards(string[] path, IFdbTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
var ranges = await Fdb.System.GetChunksAsync(db, FdbKey.MinValue, FdbKey.MaxValue, ct);
Console.WriteLine("Found {0} shards in the whole cluster", ranges.Count);
// look if there is something under there
var folder = (await TryOpenCurrentDirectoryAsync(path, db, ct)) as FdbDirectorySubspace;
if (folder != null)
{
var r = FdbKeyRange.StartsWith(folder.Copy().Key);
Console.WriteLine("Searching for shards that intersect with /{0} ...", String.Join("/", path));
ranges = await Fdb.System.GetChunksAsync(db, r, ct);
Console.WriteLine("Found {0} ranges intersecting {1}:", ranges.Count, r);
var last = Slice.Empty;
foreach (var range in ranges)
{
Console.Write("> " + FdbKey.Dump(range.Begin) + " ...");
long count = await Fdb.System.EstimateCountAsync(db, range, ct);
Console.WriteLine(" {0:N0}", count);
last = range.End;
//TODO: we can probably get more details on this shard looking in the system keyspace (where it is, how many replicas, ...)
}
Console.WriteLine("> ... " + FdbKey.Dump(last));
}
//Console.WriteLine("Found " + ranges.Count + " shards in the cluster");
//TODO: shards that intersect the current directory
}
/// <summary>Copy a pair of keys into the buffer, and return a new identical pair</summary>
/// <param name="range">Key range</param>
/// <returns>Equivalent pair of keys, that are backed by the buffer.</returns>
public FdbKeyRange InternRange(FdbKeyRange range)
{
//TODO: if end is prefixed by begin, we could merge both keys (frequent when dealing with ranges on tuples that add \xFF
return new FdbKeyRange(
Intern(range.Begin, aligned: true),
Intern(range.End, aligned: true)
);
}
private void ClearTask(IFdbTransaction tr, Slice taskId)
{
tr.Annotate("Deleting task {0}", taskId.ToAsciiOrHexaString());
// clear all metadata about the task
tr.ClearRange(FdbKeyRange.StartsWith(this.TaskStore.Pack(taskId)));
// decrement pending number of tasks
this.Counters.Decrement(tr, COUNTER_PENDING_TASKS);
}
public void Test_FdbKeyRange_Disjoint()
{
Func <byte, byte, FdbKeyRange> range = (x, y) => FdbKeyRange.Create(Slice.FromByte(x), Slice.FromByte(y));
#region Disjoint...
// [0, 1) [2, 3)
// #X
// #X
Assert.That(range(0, 1).Disjoint(range(2, 3)), Is.True);
// [2, 3) [0, 1)
// #X
// #X
Assert.That(range(2, 3).Disjoint(range(0, 1)), Is.True);
#endregion
#region Not Disjoint...
// [0, 1) [1, 2)
// #X
// #X
Assert.That(range(0, 1).Disjoint(range(1, 2)), Is.False);
// [1, 2) [0, 1)
// #X
// #X
Assert.That(range(1, 2).Disjoint(range(0, 1)), Is.False);
// [0, 2) [1, 3)
// ##X
// ##X
Assert.That(range(0, 2).Disjoint(range(1, 3)), Is.False);
// [1, 3) [0, 2)
// ##X
// ##X
Assert.That(range(1, 3).Disjoint(range(0, 2)), Is.False);
// [0, 1) [0, 2)
// #X
// ##X
Assert.That(range(0, 1).Disjoint(range(0, 2)), Is.False);
// [0, 2) [0, 1)
// ##X
// #X
Assert.That(range(0, 2).Disjoint(range(0, 1)), Is.False);
// [0, 2) [1, 2)
// ##X
// #X
Assert.That(range(0, 2).Disjoint(range(1, 2)), Is.False);
// [1, 2) [0, 2)
// #X
// ##X
Assert.That(range(1, 2).Disjoint(range(0, 2)), Is.False);
#endregion
}
protected virtual Task <List <Slice> > LoadPartsAsync(IFdbReadOnlyTransaction trans, TId id)
{
var key = this.Location.Partial.EncodeKey(id);
return(trans
.GetRange(FdbKeyRange.StartsWith(key)) //TODO: options ?
.Select(kvp => kvp.Value)
.ToListAsync());
}
/// <summary>Remove all the values for a specific key</summary>
/// <param name="trans"></param>
/// <param name="key"></param>
/// <returns></returns>
public void Remove([NotNull] IFdbTransaction trans, TKey key)
{
if (trans == null)
{
throw new ArgumentNullException("trans");
}
trans.ClearRange(FdbKeyRange.StartsWith(this.Location.Partial.Keys.Encode(key)));
}
public FdbRangeQuery <TId> Lookup(IFdbReadOnlyTransaction trans, TValue value, bool reverse = false)
{
var prefix = this.Location.Partial.Keys.Encode(value);
return(trans
.GetRange(FdbKeyRange.StartsWith(prefix), new FdbRangeOptions {
Reverse = reverse
})
.Select((kvp) => this.Location.Keys.Decode(kvp.Key).Item2));
}
private Task Scenario2(IFdbTransaction tr)
{
var location = FdbSubspace.Create(Slice.FromAscii("TEST"));
tr.ClearRange(FdbKeyRange.StartsWith(location.Key));
for (int i = 0; i < 10; i++)
{
tr.Set(location.Pack(i), Slice.FromString("value of " + i));
}
return(Task.FromResult <object>(null));
}
public void Test_FdbKey_PrettyPrint()
{
// verify that the pretty printing of keys produce a user friendly output
Assert.That(FdbKey.Dump(Slice.Nil), Is.EqualTo("<null>"));
Assert.That(FdbKey.Dump(Slice.Empty), Is.EqualTo("<empty>"));
Assert.That(FdbKey.Dump(Slice.FromByte(0)), Is.EqualTo("<00>"));
Assert.That(FdbKey.Dump(Slice.FromByte(255)), Is.EqualTo("<FF>"));
Assert.That(FdbKey.Dump(Slice.Create(new byte[] { 0, 1, 2, 3, 4, 5, 6, 7 })), Is.EqualTo("<00><01><02><03><04><05><06><07>"));
Assert.That(FdbKey.Dump(Slice.Create(new byte[] { 255, 254, 253, 252, 251, 250, 249, 248 })), Is.EqualTo("<FF><FE><FD><FC><FB><FA><F9><F8>"));
Assert.That(FdbKey.Dump(Slice.FromString("hello")), Is.EqualTo("hello"));
Assert.That(FdbKey.Dump(Slice.FromString("héllø")), Is.EqualTo("h<C3><A9>ll<C3><B8>"));
// tuples should be decoded properly
Assert.That(FdbKey.Dump(FdbTuple.Pack(123)), Is.EqualTo("(123,)"), "Singleton tuples should end with a ','");
Assert.That(FdbKey.Dump(FdbTuple.Pack(Slice.FromAscii("hello"))), Is.EqualTo("('hello',)"), "ASCII strings should use single quotes");
Assert.That(FdbKey.Dump(FdbTuple.Pack("héllø")), Is.EqualTo("(\"héllø\",)"), "Unicode strings should use double quotes");
Assert.That(FdbKey.Dump(FdbTuple.Pack(Slice.Create(new byte[] { 1, 2, 3 }))), Is.EqualTo("(<01 02 03>,)"));
Assert.That(FdbKey.Dump(FdbTuple.Pack(123, 456)), Is.EqualTo("(123, 456)"), "Elements should be separated with a space, and not end up with ','");
Assert.That(FdbKey.Dump(FdbTuple.Pack(true, false, default(object))), Is.EqualTo("(1, 0, null)"), "Booleans should be displayed as numbers, and null should be in lowercase"); //note: even though it's tempting to using Python's "Nil", it's not very ".NETty"
Assert.That(FdbKey.Dump(FdbTuple.Pack(1.0d, Math.PI, Math.E)), Is.EqualTo("(1, 3.1415926535897931, 2.7182818284590451)"), "Doubles should used dot and have full precision (17 digits)");
Assert.That(FdbKey.Dump(FdbTuple.Pack(1.0f, (float)Math.PI, (float)Math.E)), Is.EqualTo("(1, 3.14159274, 2.71828175)"), "Singles should used dot and have full precision (10 digits)");
var guid = Guid.NewGuid();
Assert.That(FdbKey.Dump(FdbTuple.Pack(guid)), Is.EqualTo(String.Format("({0},)", guid.ToString("D"))), "GUIDs should be displayed as a string literal, without quotes");
var uuid128 = Uuid128.NewUuid();
Assert.That(FdbKey.Dump(FdbTuple.Pack(uuid128)), Is.EqualTo(String.Format("({0},)", uuid128.ToString("D"))), "Uuid128s should be displayed as a string literal, without quotes");
var uuid64 = Uuid64.NewUuid();
Assert.That(FdbKey.Dump(FdbTuple.Pack(uuid64)), Is.EqualTo(String.Format("({0},)", uuid64.ToString("D"))), "Uuid64s should be displayed as a string literal, without quotes");
// ranges should be decoded when possible
var key = FdbTuple.ToRange(FdbTuple.Create("hello"));
// "<02>hello<00><00>" .. "<02>hello<00><FF>"
Assert.That(FdbKey.PrettyPrint(key.Begin, FdbKey.PrettyPrintMode.Begin), Is.EqualTo("(\"hello\",).<00>"));
Assert.That(FdbKey.PrettyPrint(key.End, FdbKey.PrettyPrintMode.End), Is.EqualTo("(\"hello\",).<FF>"));
key = FdbKeyRange.StartsWith(FdbTuple.Pack("hello"));
// "<02>hello<00>" .. "<02>hello<01>"
Assert.That(FdbKey.PrettyPrint(key.Begin, FdbKey.PrettyPrintMode.Begin), Is.EqualTo("(\"hello\",)"));
Assert.That(FdbKey.PrettyPrint(key.End, FdbKey.PrettyPrintMode.End), Is.EqualTo("(\"hello\",) + 1"));
var t = FdbTuple.Pack(123);
Assert.That(FdbKey.PrettyPrint(t, FdbKey.PrettyPrintMode.Single), Is.EqualTo("(123,)"));
Assert.That(FdbKey.PrettyPrint(FdbTuple.ToRange(t).Begin, FdbKey.PrettyPrintMode.Begin), Is.EqualTo("(123,).<00>"));
Assert.That(FdbKey.PrettyPrint(FdbTuple.ToRange(t).End, FdbKey.PrettyPrintMode.End), Is.EqualTo("(123,).<FF>"));
}
/// <summary>Remove all fields of an hashset</summary>
/// <param name="id"></param>
public void Delete(IFdbTransaction trans, IFdbTuple id)
{
if (trans == null)
{
throw new ArgumentNullException("trans");
}
if (id == null)
{
throw new ArgumentNullException("id");
}
// remove all fields of the hash
trans.ClearRange(FdbKeyRange.StartsWith(GetKey(id)));
}
/// <summary>Insert a new document in the collection</summary>
public void Insert(IFdbTransaction trans, TDocument document)
{
if (trans == null)
{
throw new ArgumentNullException("trans");
}
if (document == null)
{
throw new ArgumentNullException("document");
}
var id = this.IdSelector(document);
if (id == null)
{
throw new InvalidOperationException("Cannot insert a document with a null identifier");
}
// encode the document
var packed = this.ValueEncoder.EncodeValue(document);
// Key Prefix = ...(id,)
var key = this.Location.Partial.EncodeKey(id);
// clear previous value
trans.ClearRange(FdbKeyRange.StartsWith(key));
int remaining = packed.Count;
if (remaining <= this.ChunkSize)
{ // stored as a single element
// Key = ...(id,)
trans.Set(key, packed);
}
else
{ // splits in as many chunks as necessary
// Key = ...(id, N) where N is the chunk index (0-based)
int p = 0;
int index = 0;
while (remaining > 0)
{
int sz = Math.Max(remaining, this.ChunkSize);
this.Location.Set(trans, FdbTuple.Create(id, index), packed.Substring(p, sz));
++index;
p += sz;
remaining -= sz;
}
}
}
/// <summary>Delete a document from the collection</summary>
/// <param name="trans"></param>
/// <param name="id"></param>
public void Delete(IFdbTransaction trans, TId id)
{
if (trans == null)
{
throw new ArgumentNullException("trans");
}
if (id == null)
{
throw new ArgumentNullException("id");
}
var key = this.Location.Partial.EncodeKey(id);
trans.ClearRange(FdbKeyRange.StartsWith(key));
}
/// <summary>Delete a document from the collection</summary>
/// <param name="trans"></param>
/// <param name="ids"></param>
public void DeleteMultiple(IFdbTransaction trans, IEnumerable <TId> ids)
{
if (trans == null)
{
throw new ArgumentNullException("trans");
}
if (ids == null)
{
throw new ArgumentNullException("ids");
}
foreach (var key in this.Location.Partial.EncodeKeyRange(ids))
{
trans.ClearRange(FdbKeyRange.StartsWith(key));
}
}
/// <summary>Adds a range to teh clear list of this transaction</summary>
/// <remarks>Must be called with m_lock taken</remarks>
private void AddClearCommand_NeedsLocking(FdbKeyRange range)
{
// merge the cleared range with the others
m_clears.Mark(range.Begin, range.End);
// remove all writes that where in this range
var keys = m_writes.FindBetween(range.Begin, true, range.End, false).ToList();
if (keys.Count > 0)
{
foreach (var key in keys)
{
m_writes.Remove(key);
}
}
}
public IFdbAsyncEnumerable <KeyValuePair <TValue, long> > GetCounts([NotNull] IFdbReadOnlyTransaction trans, TKey key)
{
var range = FdbKeyRange.StartsWith(this.Location.Partial.Keys.Encode(key));
var query = trans
.GetRange(range)
.Select(kvp => new KeyValuePair <TValue, long>(this.Location.Keys.Decode(kvp.Key).Item2, kvp.Value.ToInt64()));
if (this.AllowNegativeValues)
{
return(query);
}
else
{
return(query.Where(kvp => kvp.Value > 0));
}
}
public decimal Count()
{
decimal total = 0M;
using (var tx = _le.BeginTransaction(TransactionBeginFlags.ReadOnly)) {
var prefix = FdbTuple.Create((byte)Tables.Quantity).ToSlice();
var range = FdbKeyRange.StartsWith(prefix);
var nums = InternalScan(tx, range, (slice, bytes) => ToDecimal(bytes));
foreach (var num in nums)
{
total += num;
}
}
return(total);
}
public void Test_FdbKeyRange_PrefixedBy()
{
FdbKeyRange range;
// "abc" => [ "abc\x00", "abd" )
range = FdbKeyRange.PrefixedBy(Slice.FromAscii("abc"));
Assert.That(range.Begin, Is.EqualTo(Slice.FromAscii("abc\x00")));
Assert.That(range.End, Is.EqualTo(Slice.FromAscii("abd")));
// "" => ArgumentException
Assert.That(() => FdbKeyRange.PrefixedBy(Slice.Empty), Throws.InstanceOf <ArgumentException>());
// "\xFF" => ArgumentException
Assert.That(() => FdbKeyRange.PrefixedBy(Slice.FromAscii("\xFF")), Throws.InstanceOf <ArgumentException>());
// null => ArgumentException
Assert.That(() => FdbKeyRange.PrefixedBy(Slice.Nil), Throws.InstanceOf <ArgumentException>());
}
static IEnumerable <TOut> InternalScan <TOut>(Tx tx, byte[] key,
Func <Slice, byte[], TOut> convert, string traceName, int skip = 0)
{
tx.TraceStart(traceName);
ushort x = 0;
var slice = Slice.Create(key);
var range = FdbKeyRange.StartsWith(slice);
using (var c = tx.CreateCursor()) {
if (!c.MoveToFirstAfter(key))
{
tx.TraceStop(traceName, x);
yield break;
}
var pair = c.Current;
for (var i = 0; i < int.MaxValue; i++)
{
var current = Slice.Create(pair.Key);
if (!range.Contains(current))
{
tx.TraceStop(traceName, x);
break;
}
if (i >= skip)
{
x += 1;
yield return(convert(current, pair.Value));
}
if (!c.MoveNext())
{
tx.TraceStop(traceName, x);
break;
}
pair = c.Current;
}
}
}
public static void DeleteRange(Tx tx, byte[] key, string traceName)
{
tx.TraceStart(traceName);
ushort x = 0;
var slice = Slice.Create(key);
var range = FdbKeyRange.StartsWith(slice);
using (var c = tx.CreateCursor()) {
if (!c.MoveToFirstAfter(key))
{
tx.TraceStop(traceName, x);
return;
}
var pair = c.Current;
for (var i = 0; i < int.MaxValue; i++)
{
var current = Slice.Create(pair.Key);
if (!range.Contains(current))
{
tx.TraceStop(traceName, x);
break;
}
x += 1;
tx.Delete(pair.Key);
if (!c.MoveNext())
{
tx.TraceStop(traceName, x);
break;
}
pair = c.Current;
}
}
tx.TraceStop(traceName);
}
/// <summary>Return the list the names of all fields of an hashset</summary>
/// <param name="trans">Transaction that will be used for this request</param>
/// <param name="id">Unique identifier of the hashset</param>
/// <returns>List of all fields. If the list is empty, the hashset does not exist</returns>
public Task <List <string> > GetKeys(IFdbReadOnlyTransaction trans, IFdbTuple id, CancellationToken cancellationToken = default(CancellationToken))
{
//note: As of Beta2, FDB does not have a fdb_get_range that only return the keys. That means that we will have to also read the values from the db, in order to just get the names of the fields :(
//TODO: find a way to optimize this ?
if (trans == null)
{
throw new ArgumentNullException("trans");
}
if (id == null)
{
throw new ArgumentNullException("id");
}
var prefix = GetKey(id);
var results = new Dictionary <string, Slice>(StringComparer.OrdinalIgnoreCase);
return(trans
.GetRange(FdbKeyRange.StartsWith(prefix))
.Select((kvp) => ParseFieldKey(FdbTuple.Unpack(kvp.Key)))
.ToListAsync(cancellationToken));
}
public IFdbAsyncEnumerable <TValue> Get([NotNull] IFdbReadOnlyTransaction trans, TKey key)
{
if (trans == null)
{
throw new ArgumentNullException("trans");
}
var range = FdbKeyRange.StartsWith(this.Location.Partial.Keys.Encode(key));
if (this.AllowNegativeValues)
{
return(trans
.GetRange(range)
.Select(kvp => this.Location.Keys.Decode(kvp.Key).Item2));
}
else
{
return(trans
.GetRange(range)
.Where(kvp => kvp.Value.ToInt64() > 0) // we need to filter out zero or negative values (possible artefacts)
.Select(kvp => this.Location.Keys.Decode(kvp.Key).Item2));
}
}
public static async Task DumpSubspace([NotNull] IFdbReadOnlyTransaction tr, [NotNull] FdbSubspace subspace)
{
Assert.That(tr, Is.Not.Null);
Console.WriteLine("Dumping content of subspace " + subspace.ToString() + " :");
int count = 0;
await tr
.GetRange(FdbKeyRange.StartsWith(subspace.Key))
.ForEachAsync((kvp) =>
{
var key = subspace.Extract(kvp.Key);
++count;
string keyDump = null;
try
{
// attemps decoding it as a tuple
keyDump = key.ToTuple().ToString();
}
catch (Exception)
{
// not a tuple, dump as bytes
keyDump = "'" + key.ToString() + "'";
}
Console.WriteLine("- " + keyDump + " = " + kvp.Value.ToString());
});
if (count == 0)
{
Console.WriteLine("> empty !");
}
else
{
Console.WriteLine("> Found " + count + " values");
}
}
private async Task RunAsync(IFdbDatabase db, FdbSubspace location, CancellationToken ct, Action done, int N, int K, int W)
{
if (db == null)
{
throw new ArgumentNullException("db");
}
StringBuilder sb = new StringBuilder();
db = new FdbLoggedDatabase(db, false, false, (log) =>
{
sb.AppendLine(log.Log.GetTimingsReport(true));
//Console.WriteLine(log.Log.GetTimingsReport(true));
});
try
{
var workerPool = new FdbWorkerPool(location);
Console.WriteLine("workerPool at " + location.Key.ToAsciiOrHexaString());
var workerSignal = new AsyncCancelableMutex(ct);
var clientSignal = new AsyncCancelableMutex(ct);
int taskCounter = 0;
int msgSent = 0;
int msgReceived = 0;
Func <FdbWorkerMessage, CancellationToken, Task> handler = async(msg, _ct) =>
{
Interlocked.Increment(ref msgReceived);
//await Task.Delay(10 + Math.Abs(msg.Id.GetHashCode()) % 50);
await Task.Delay(10).ConfigureAwait(false);
};
Func <int, Task> worker = async(id) =>
{
await workerSignal.Task.ConfigureAwait(false);
Console.WriteLine("Worker #" + id + " is starting");
try
{
await workerPool.RunWorkerAsync(db, handler, ct).ConfigureAwait(false);
}
finally
{
Console.WriteLine("Worker #" + id + " has stopped");
}
};
Func <int, Task> client = async(id) =>
{
await clientSignal.Task.ConfigureAwait(false);
await Task.Delay(10).ConfigureAwait(false);
var rnd = new Random(id * 111);
for (int i = 0; i < N; i++)
{
var taskId = Slice.FromString("T" + Interlocked.Increment(ref taskCounter));
var taskBody = Slice.FromString("Message " + (i + 1) + " of " + N + " from client #" + id);
await workerPool.ScheduleTaskAsync(db, taskId, taskBody, ct).ConfigureAwait(false);
Interlocked.Increment(ref msgSent);
//if (i > 0 && i % 10 == 0) Console.WriteLine("@@@ Client#" + id + " pushed " + (i + 1) + " / " + N + " messages");
switch (rnd.Next(5))
{
case 0: await Task.Delay(10).ConfigureAwait(false); break;
case 1: await Task.Delay(100).ConfigureAwait(false); break;
case 2: await Task.Delay(500).ConfigureAwait(false); break;
}
}
Console.WriteLine("@@@ Client#" + id + " has finished!");
};
Func <string, Task> dump = async(label) =>
{
Console.WriteLine("<dump label='" + label + "' key='" + location.Key.ToAsciiOrHexaString() + "'>");
using (var tr = db.BeginTransaction(ct))
{
await tr.Snapshot
.GetRange(FdbKeyRange.StartsWith(location.Key))
.ForEachAsync((kvp) =>
{
Console.WriteLine(" - " + FdbTuple.Unpack(location.Extract(kvp.Key)) + " = " + kvp.Value.ToAsciiOrHexaString());
}).ConfigureAwait(false);
}
Console.WriteLine("</dump>");
};
var workers = Enumerable.Range(0, W).Select((i) => worker(i)).ToArray();
var clients = Enumerable.Range(0, K).Select((i) => client(i)).ToArray();
DateTime start = DateTime.Now;
DateTime last = start;
int lastHandled = -1;
using (var timer = new Timer((_) =>
{
var now = DateTime.Now;
Console.WriteLine("@@@ T=" + now.Subtract(start) + ", sent: " + msgSent.ToString("N0") + ", recv: " + msgReceived.ToString("N0"));
Console.WriteLine("### Workers: " + workerPool.IdleWorkers + " / " + workerPool.ActiveWorkers + " (" + new string('#', workerPool.IdleWorkers) + new string('.', workerPool.ActiveWorkers - workerPool.IdleWorkers) + "), sent: " + workerPool.MessageScheduled.ToString("N0") + ", recv: " + workerPool.MessageReceived.ToString("N0") + ", delta: " + (workerPool.MessageScheduled - workerPool.MessageReceived).ToString("N0") + ", busy: " + workerPool.WorkerBusyTime + " (avg " + workerPool.WorkerAverageBusyDuration.TotalMilliseconds.ToString("N3") + " ms)");
if (now.Subtract(last).TotalSeconds >= 10)
{
//dump("timer").GetAwaiter().GetResult();
last = now;
if (lastHandled == msgReceived)
{ // STALL ?
Console.WriteLine("STALL! ");
done();
}
lastHandled = msgReceived;
}
if (msgReceived >= K * N)
{
dump("complete").GetAwaiter().GetResult();
done();
}
}, null, 1000, 1000))
{
var sw = Stopwatch.StartNew();
// start the workers
workerSignal.Set(async: true);
await Task.Delay(500);
await dump("workers started");
// start the clients
clientSignal.Set(async: true);
await Task.WhenAll(clients);
Console.WriteLine("Clients completed after " + sw.Elapsed);
await Task.WhenAll(workers);
Console.WriteLine("Workers completed after " + sw.Elapsed);
}
}
finally
{
Console.WriteLine("---------------------------------------------------------------------------");
Console.WriteLine("Transaction logs:");
Console.WriteLine();
Console.WriteLine(sb.ToString());
}
}
public static async Task Sampling(string[] path, IFdbTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
double ratio = 0.1d;
bool auto = true;
if (extras.Count > 0)
{
double x = extras.Get <double>(0);
if (x > 0 && x <= 1)
{
ratio = x;
}
auto = false;
}
var folder = await TryOpenCurrentDirectoryAsync(path, db, ct);
FdbKeyRange span;
if (folder is FdbDirectorySubspace)
{
span = FdbKeyRange.StartsWith((folder as FdbDirectorySubspace).Copy());
log.WriteLine("Reading list of shards for /{0} under {1} ...", String.Join("/", path), FdbKey.Dump(span.Begin));
}
else
{
log.WriteLine("Reading list of shards for the whole cluster ...");
span = FdbKeyRange.All;
}
// dump keyServers
var ranges = await Fdb.System.GetChunksAsync(db, span, ct);
log.WriteLine("> Found {0:N0} shard(s)", ranges.Count);
// take a sample
var samples = new List <FdbKeyRange>();
if (ranges.Count <= 32)
{ // small enough to scan it all
samples.AddRange(ranges);
log.WriteLine("Sampling all {0:N0} shards ...", samples.Count);
}
else
{ // need to take a random subset
var rnd = new Random();
int sz = Math.Max((int)Math.Ceiling(ratio * ranges.Count), 1);
if (auto)
{
if (sz > 100)
{
sz = 100; //SAFETY
}
if (sz < 32)
{
sz = Math.Max(sz, Math.Min(32, ranges.Count));
}
}
var population = new List <FdbKeyRange>(ranges);
for (int i = 0; i < sz; i++)
{
int p = rnd.Next(population.Count);
samples.Add(population[p]);
population.RemoveAt(p);
}
log.WriteLine("Sampling " + samples.Count + " out of " + ranges.Count + " shards (" + (100.0 * samples.Count / ranges.Count).ToString("N1") + "%) ...");
}
log.WriteLine();
const string FORMAT_STRING = "{0,9} ║{1,10}{6,6} {2,-29} ║{3,10}{7,7} {4,-37} ║{5,10}";
const string SCALE_KEY = "....--------========########M";
const string SCALE_VAL = "....--------========########@@@@@@@@M";
log.WriteLine(FORMAT_STRING, "Count", "Keys", SCALE_KEY, "Values", SCALE_VAL, "Total", "med.", "med.");
var rangeOptions = new FdbRangeOptions {
Mode = FdbStreamingMode.WantAll
};
samples = samples.OrderBy(x => x.Begin).ToList();
long globalSize = 0;
long globalCount = 0;
int workers = 8; // Math.Max(4, Environment.ProcessorCount);
var sw = Stopwatch.StartNew();
var tasks = new List <Task>();
int n = samples.Count;
while (samples.Count > 0)
{
while (tasks.Count < workers && samples.Count > 0)
{
var range = samples[0];
samples.RemoveAt(0);
tasks.Add(Task.Run(async() =>
{
var kk = new RobustHistogram(RobustHistogram.TimeScale.Ticks);
var vv = new RobustHistogram(RobustHistogram.TimeScale.Ticks);
#region Method 1: get_range everything...
using (var tr = db.BeginTransaction(ct))
{
long keySize = 0;
long valueSize = 0;
long count = 0;
int iter = 0;
var beginSelector = FdbKeySelector.FirstGreaterOrEqual(range.Begin);
var endSelector = FdbKeySelector.FirstGreaterOrEqual(range.End);
while (true)
{
FdbRangeChunk data = default(FdbRangeChunk);
FdbException error = null;
try
{
data = await tr.Snapshot.GetRangeAsync(
beginSelector,
endSelector,
rangeOptions,
iter
).ConfigureAwait(false);
}
catch (FdbException e)
{
error = e;
}
if (error != null)
{
await tr.OnErrorAsync(error.Code).ConfigureAwait(false);
continue;
}
if (data.Count == 0)
{
break;
}
count += data.Count;
foreach (var kvp in data.Chunk)
{
keySize += kvp.Key.Count;
valueSize += kvp.Value.Count;
kk.Add(TimeSpan.FromTicks(kvp.Key.Count));
vv.Add(TimeSpan.FromTicks(kvp.Value.Count));
}
if (!data.HasMore)
{
break;
}
beginSelector = FdbKeySelector.FirstGreaterThan(data.Last.Key);
++iter;
}
long totalSize = keySize + valueSize;
Interlocked.Add(ref globalSize, totalSize);
Interlocked.Add(ref globalCount, count);
lock (log)
{
log.WriteLine(FORMAT_STRING, count.ToString("N0"), FormatSize(keySize), kk.GetDistribution(begin: 1, end: 12000, fold: 2), FormatSize(valueSize), vv.GetDistribution(begin: 1, end: 120000, fold: 2), FormatSize(totalSize), FormatSize((int)Math.Ceiling(kk.Median)), FormatSize((int)Math.Ceiling(vv.Median)));
}
}
#endregion
#region Method 2: estimate the count using key selectors...
//long counter = await Fdb.System.EstimateCountAsync(db, range, ct);
//Console.WriteLine("COUNT = " + counter.ToString("N0"));
#endregion
}, ct));
}
var done = await Task.WhenAny(tasks);
tasks.Remove(done);
}
await Task.WhenAll(tasks);
sw.Stop();
log.WriteLine();
if (n != ranges.Count)
{
log.WriteLine("Sampled " + FormatSize(globalSize) + " (" + globalSize.ToString("N0") + " bytes) and " + globalCount.ToString("N0") + " keys in " + sw.Elapsed.TotalSeconds.ToString("N1") + " sec");
log.WriteLine("> Estimated total size is " + FormatSize(globalSize * ranges.Count / n));
}
else
{
log.WriteLine("Found " + FormatSize(globalSize) + " (" + globalSize.ToString("N0") + " bytes) and " + globalCount.ToString("N0") + " keys in " + sw.Elapsed.TotalSeconds.ToString("N1") + " sec");
// compare to the whole cluster
ranges = await Fdb.System.GetChunksAsync(db, FdbKey.MinValue, FdbKey.MaxValue, ct);
log.WriteLine("> This directory contains ~{0:N2}% of all data", (100.0 * n / ranges.Count));
}
log.WriteLine();
}
/// <summary>Find the DCs, machines and processes in the cluster</summary>
public static async Task Topology(string[] path, IFdbTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
var coords = await Fdb.System.GetCoordinatorsAsync(db, ct);
log.WriteLine("[Cluster] {0}", coords.Id);
var servers = await db.QueryAsync(tr => tr
.WithReadAccessToSystemKeys()
.GetRange(FdbKeyRange.StartsWith(Fdb.System.ServerList))
.Select(kvp => new
{
// Offsets Size Type Name Description
// 0 2 Word Version? 0100 (1.0 ?)
// 2 4 DWord ??? 0x00 0x20 0xA2 0x00
// 6 2 Word FDBMagic 0xDB 0x0F "FDB"
// 8 16 Guid NodeId Unique Process ID
// 24 16 Guid Machine "machine_id" field in foundationdb.conf (ends with 8x0 if manually specified)
// 40 16 Guid DataCenter "datacenter_id" field in foundationdb.conf (ends with 8x0 if manually specified)
// 56 4 ??? ?? 4 x 0
// 60 12 x24 ARRAY[] ?? array of 12x the same 24-byte struct defined below
// ...0 4 DWord IPAddress 01 00 00 7F => 127.0.0.1
// ...4 4 DWord Port 94 11 00 00 -> 4500
// ...8 4 DWord ?? randomish, changes every reboot
// ..12 4 DWord ?? randomish, changes every reboot
// ..16 4 DWord Size? small L-E integer, usually between 0x20 and 0x40...
// ..20 4 DWord ?? randmoish, changes every reboot
ProcessId = kvp.Value.Substring(8, 16).ToHexaString(),
MachineId = kvp.Value.Substring(24, 16).ToHexaString(),
DataCenterId = kvp.Value.Substring(40, 16).ToHexaString(),
Parts = Enumerable.Range(0, 12).Select(i =>
{
int p = 60 + 24 * i;
return(new
{
Address = new IPAddress(kvp.Value.Substring(p, 4).GetBytes().Reverse().ToArray()),
Port = kvp.Value.Substring(p + 4, 4).ToInt32(),
Unknown1 = kvp.Value.Substring(p + 8, 4).ToInt32(),
Unknown2 = kvp.Value.Substring(p + 12, 4).ToInt32(),
Unknown3 = kvp.Value.Substring(p + 16, 4).ToInt32(),
Unknown4 = kvp.Value.Substring(p + 20, 4).ToInt32(),
});
}).ToList(),
Raw = kvp.Value,
}),
ct
);
var numNodes = servers.Select(s => s.ProcessId).Distinct().Count();
var numMachines = servers.Select(s => s.MachineId).Distinct().Count();
var numDCs = servers.Select(s => s.DataCenterId).Distinct().Count();
var dcs = servers.GroupBy(x => x.DataCenterId).ToArray();
for (int dcIndex = 0; dcIndex < dcs.Length; dcIndex++)
{
var dc = dcs[dcIndex];
bool lastDc = dcIndex == dcs.Length - 1;
string dcId = dc.Key.EndsWith("0000000000000000") ? dc.Key.Substring(0, 16) : dc.Key;
log.WriteLine((lastDc ? "`- " : "|- ") + "[DataCenter] {0} (#{1})", dcId, dcIndex);
var machines = dc.GroupBy(x => x.MachineId).ToArray();
string dcPrefix = lastDc ? " " : "| ";
for (int machineIndex = 0; machineIndex < machines.Length; machineIndex++)
{
var machine = machines[machineIndex];
var lastMachine = machineIndex == machines.Length - 1;
string machineId = machine.Key.EndsWith("0000000000000000") ? machine.Key.Substring(0, 16) : machine.Key;
log.WriteLine(dcPrefix + (lastMachine ? "`- " : "|- ") + "[Machine] {0}, {1}", machine.First().Parts[0].Address, machineId);
var procs = machine.ToArray();
string machinePrefix = dcPrefix + (lastMachine ? " " : "| ");
for (int procIndex = 0; procIndex < procs.Length; procIndex++)
{
var proc = procs[procIndex];
bool lastProc = procIndex == procs.Length - 1;
log.WriteLine(machinePrefix + (lastProc ? "`- " : "|- ") + "[Process] {0}:{1}, {2}", proc.Parts[0].Address, proc.Parts[0].Port, proc.ProcessId);
//foreach (var part in proc.Parts)
//{
// log.WriteLine(machinePrefix + "| -> {0}, {1}, {2:X8}, {3:X8}, {4}, {5:X8}", part.Address, part.Port, part.Unknown1, part.Unknown2, part.Unknown3, part.Unknown4);
//}
}
}
}
log.WriteLine();
log.WriteLine("Found {0} process(es) on {1} machine(s) in {2} datacenter(s)", numNodes, numMachines, numDCs);
log.WriteLine();
}
public static async Task Map(string[] path, IFdbTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
// we want to merge the map of shards, with the map of directories from the Directory Layer, and count for each directory how many shards intersect
var folder = await TryOpenCurrentDirectoryAsync(path, db, ct);
if (folder == null)
{
log.WriteLine("# Directory not found");
return;
}
var span = folder.DirectoryLayer.ContentSubspace.ToRange();
// note: this may break in future versions of the DL! Maybe we need a custom API to get a flat list of all directories in a DL that span a specific range ?
var shards = await Fdb.System.GetChunksAsync(db, span, ct);
int totalShards = shards.Count;
log.WriteLine("Found {0} shard(s) in partition /{1}", totalShards, folder.DirectoryLayer.FullName);
log.WriteLine("Listing all directories...");
var map = new Dictionary <string, int>(StringComparer.Ordinal);
Action <string[], int> account = (p, c) =>
{
for (int i = 1; i <= p.Length; i++)
{
var s = "/" + String.Join("/", p, 0, i);
int x;
map[s] = map.TryGetValue(s, out x) ? (x + c) : c;
}
};
var work = new Stack <IFdbDirectory>();
work.Push(folder);
var dirs = new List <IFdbDirectory>();
int n = 0;
while (work.Count > 0)
{
var cur = work.Pop();
// skip sub partitions
var names = await cur.ListAsync(db, ct);
foreach (var name in names)
{
var sub = await cur.TryOpenAsync(db, name, ct);
if (sub != null)
{
var p = sub.FullName;
if (sub is FdbDirectoryPartition)
{
log.WriteLine("\r! Skipping partition {0} ", sub.Name);
n = 0;
continue;
}
log.Write("\r/{0}{1}", p, p.Length > n ? String.Empty : new string(' ', n - p.Length));
n = p.Length;
work.Push(sub);
dirs.Add(sub);
}
}
}
log.Write("\r" + new string(' ', n + 2));
log.WriteLine("\r> Found {0} sub-directories", dirs.Count);
log.WriteLine();
log.WriteLine("Estimating size of each directory...");
int foundShards = 0;
n = 0;
int max = 0;
IFdbDirectory bigBad = null;
foreach (var dir in dirs)
{
log.Write("\r> {0}{1}", dir.Name, dir.Name.Length > n ? String.Empty : new string(' ', n - dir.Name.Length));
n = dir.Name.Length;
var p = dir.Path.ToArray();
var key = ((FdbSubspace)dir).Key;
// verify that the subspace has at least one key inside
var bounds = await db.ReadAsync(async (tr) =>
{
var kvs = await Task.WhenAll(
tr.GetRange(FdbKeyRange.StartsWith(key)).FirstOrDefaultAsync(),
tr.GetRange(FdbKeyRange.StartsWith(key)).LastOrDefaultAsync()
);
return(new { Min = kvs[0].Key, Max = kvs[1].Key });
}, ct);
if (bounds.Min.HasValue)
{ // folder is not empty
shards = await Fdb.System.GetChunksAsync(db, FdbKeyRange.StartsWith(key), ct);
//TODO: we still need to check if the first and last shard really intersect the subspace
// we need to check if the shards actually contain data
//Console.WriteLine("/{0} under {1} with {2} shard(s)", string.Join("/", p), FdbKey.Dump(key), shards.Count);
foundShards += shards.Count;
account(p, shards.Count);
if (shards.Count > max)
{
max = shards.Count; bigBad = dir;
}
}
else
{
account(p, 0);
}
}
log.Write("\r" + new string(' ', n + 2));
log.WriteLine("\rFound a total of {0} shard(s) in {1} folder(s)", foundShards, dirs.Count);
log.WriteLine();
log.WriteLine("Shards %Total Path");
foreach (var kvp in map.OrderBy(x => x.Key))
{
log.WriteLine("{0,6} {1,-20} {2}", kvp.Value, RobustHistogram.FormatHistoBar((double)kvp.Value / foundShards, 20), kvp.Key);
}
log.WriteLine();
if (bigBad != null)
{
log.WriteLine("Biggest folder is /{0} with {1} shards ({2:N1}% total, {3:N1}% subtree)", bigBad.FullName, max, 100.0 * max / totalShards, 100.0 * max / foundShards);
log.WriteLine();
}
}
/// <summary>Estimate the number of keys in the specified range.</summary>
/// <param name="db">Database used for the operation</param>
/// <param name="range">Range defining the keys to count</param>
/// <param name="onProgress">Optional callback called everytime the count is updated. The first argument is the current count, and the second argument is the last key that was found.</param>
/// <param name="cancellationToken">Token used to cancel the operation</param>
/// <returns>Number of keys k such that range.Begin <= k > range.End</returns>
/// <remarks>If the range contains a large of number keys, the operation may need more than one transaction to complete, meaning that the number will not be transactionally accurate.</remarks>
public static Task<long> EstimateCountAsync([NotNull] IFdbDatabase db, FdbKeyRange range, IProgress<FdbTuple<long, Slice>> onProgress, CancellationToken cancellationToken)
{
return EstimateCountAsync(db, range.Begin, range.End, onProgress, cancellationToken);
}
/// <summary>Estimate the number of keys in the specified range.</summary>
/// <param name="db">Database used for the operation</param>
/// <param name="range">Range defining the keys to count</param>
/// <param name="cancellationToken">Token used to cancel the operation</param>
/// <returns>Number of keys k such that range.Begin <= k > range.End</returns>
/// <remarks>If the range contains a large of number keys, the operation may need more than one transaction to complete, meaning that the number will not be transactionally accurate.</remarks>
public static Task<long> EstimateCountAsync([NotNull] IFdbDatabase db, FdbKeyRange range, CancellationToken cancellationToken)
{
return EstimateCountAsync(db, range.Begin, range.End, null, cancellationToken);
}
//REVIEW: should we call this chunks? shard? fragments? contigous ranges?
/// <summary>Split a range of keys into smaller chunks where each chunk represents a contiguous range stored on a single server</summary>
/// <param name="db">Database to use for the operation</param>
/// <param name="range">Range of keys to split up into smaller chunks</param>
/// <param name="cancellationToken">Token used to cancel the operation</param>
/// <returns>List of one or more chunks that constitutes the range, where each chunk represents a contiguous range stored on a single server. If the list contains a single range, that means that the range is small enough to fit inside a single chunk.</returns>
/// <remarks>This method is not transactional. It will return an answer no older than the Database object it is passed, but the returned ranges are an estimate and may not represent the exact boundary locations at any database version.</remarks>
public static Task<List<FdbKeyRange>> GetChunksAsync([NotNull] IFdbDatabase db, FdbKeyRange range, CancellationToken cancellationToken)
{
//REVIEW: maybe rename this to SplitIntoChunksAsync or SplitIntoShardsAsync or GetFragmentsAsync ?
return GetChunksAsync(db, range.Begin, range.End, cancellationToken);
}
