private static void DumpIndex <TKey, TVal>(string label, MemoryIndex <TKey> index, Func <TKey, int, TVal> orderBy, IComparer <TVal> comparer = null, bool heatMaps = false)
{
comparer = comparer ?? Comparer <TVal> .Default;
int total = index.Statistics.Values.Sum();
long totalLegacy = 0;
int[] map = new int[100];
double r = (double)(map.Length - 1) / total;
Console.WriteLine("__{0}__", label);
Console.WriteLine("| Indexed Value | Count | Total % | Words | Lit% | 1-Bits | Word% | Bitmap | ratio % | Legacy | ratio % |" + (heatMaps ? " HeatMap |" : ""));
Console.WriteLine("|:------------------------|-------:|--------:|------:|-------:|-------:|-------:|---------:|--------:|----------:|--------:|" + (heatMaps ? ":-----------------------------------------------------------------------|" : ""));
foreach (var kv in index.Values.OrderBy((kv) => orderBy(kv.Key, index.Count(kv.Key)), comparer))
{
var t = FdbTuple.Create(kv.Key);
var tk = t.ToSlice();
int bits, words, literals, fillers;
double ratio;
kv.Value.GetStatistics(out bits, out words, out literals, out fillers, out ratio);
long legacyIndexSize = 0; // size estimate of a regular FDB index (..., "Value", GUID) = ""
Array.Clear(map, 0, map.Length);
foreach (var p in kv.Value.GetView())
{
map[(int)(r * p)]++;
legacyIndexSize += 3 + tk.Count + 17;
}
totalLegacy += legacyIndexSize;
int bytes = kv.Value.ToSlice().Count;
Console.WriteLine(string.Format(
CultureInfo.InvariantCulture,
"| {0,-24}| {1,6:N0} | {2,6:N2}% | {3,5:N0} | {4,5:N1}% | {5,6:N0} | {6,6:N2} | {7,8:N0} | {8,6:N2}% | {9,9:N0} | {10,6:N2}% |" + (heatMaps ? " `{11}` |" : ""),
/*0*/ t,
/*1*/ index.Count(kv.Key),
/*2*/ 100.0 * index.Frequency(kv.Key),
/*3*/ words,
/*4*/ (100.0 * literals) / words,
/*5*/ bits,
/*6*/ 1.0 * bits / words,
/*7*/ bytes,
/*8*/ 100.0 * ratio,
/*9*/ legacyIndexSize,
/*A*/ (100.0 * bytes) / legacyIndexSize,
/*B*/ heatMaps ? MakeHeatMap(map) : ""
));
}
Console.WriteLine(string.Format(
CultureInfo.InvariantCulture,
"> {0:N0} distinct value(s), {1:N0} document(s), {2:N0} bitmap bytes, {3:N0} legacy bytes",
index.Values.Count,
total,
index.Values.Values.Sum(x => x.ToSlice().Count),
totalLegacy
));
}
public async Task Test_Can_Batch_ForEach_AsyncWithContextAndState()
{
const int N = 50 * 1000;
using (var db = await OpenTestPartitionAsync())
{
Log("Bulk inserting {0:N0} items...", N);
var location = await GetCleanDirectory(db, "Bulk", "ForEach");
Log("Preparing...");
await Fdb.Bulk.WriteAsync(
db,
Enumerable.Range(1, N).Select((x) => new KeyValuePair <Slice, Slice>(location.Pack(x), Slice.FromInt32(x))),
this.Cancellation
);
Log("Reading...");
long total = 0;
long count = 0;
int chunks = 0;
var sw = Stopwatch.StartNew();
await Fdb.Bulk.ForEachAsync(
db,
Enumerable.Range(1, N).Select(x => location.Pack(x)),
() => FdbTuple.Create(0L, 0L),
async (xs, ctx, state) =>
{
Interlocked.Increment(ref chunks);
Log("> Called with batch of {0:N0} items at offset {1:N0} of gen #{2} with step {3:N0} and cooldown {4} (generation = {5:N3} sec, total = {6:N3} sec)", xs.Length, ctx.Position, ctx.Generation, ctx.Step, ctx.Cooldown, ctx.ElapsedGeneration.TotalSeconds, ctx.ElapsedTotal.TotalSeconds);
var throttle = Task.Delay(TimeSpan.FromMilliseconds(10 + (xs.Length / 25) * 5)); // magic numbers to try to last longer than 5 sec
var results = await ctx.Transaction.GetValuesAsync(xs);
await throttle;
long sum = 0;
for (int i = 0; i < results.Length; i++)
{
sum += results[i].ToInt32();
}
return(FdbTuple.Create(state.Item1 + sum, state.Item2 + results.Length));
},
(state) =>
{
Interlocked.Add(ref total, state.Item1);
Interlocked.Add(ref count, state.Item2);
},
this.Cancellation
);
sw.Stop();
Log("Done in {0:N3} sec and {1} chunks", sw.Elapsed.TotalSeconds, chunks);
Log("Sum of integers 1 to {0:N0} is {1:N0}", count, total);
// cleanup because this test can produce a lot of data
await location.RemoveAsync(db, this.Cancellation);
}
}
public void Test_Subspace_Partitioning_With_Tuple_Suffix()
{
// start from a parent subspace
var parent = new FdbSubspace(Slice.Create(new byte[] { 254 }));
Assert.That(parent.Key.ToString(), Is.EqualTo("<FE>"));
// create a child subspace using a tuple
var child = parent.Partition(FdbTuple.Create("hca"));
Assert.That(child, Is.Not.Null);
Assert.That(child.Key.ToString(), Is.EqualTo("<FE><02>hca<00>"));
// create a tuple from this child subspace
var tuple = child.Append(123);
Assert.That(tuple, Is.Not.Null);
Assert.That(tuple.ToSlice().ToString(), Is.EqualTo("<FE><02>hca<00><15>{"));
// derive another tuple from this one
var t1 = tuple.Append(false);
Assert.That(t1.ToSlice().ToString(), Is.EqualTo("<FE><02>hca<00><15>{<14>"));
// check that we could also create the same tuple starting from the parent subspace
var t2 = parent.Append("hca", 123, false);
Assert.That(t2.ToSlice(), Is.EqualTo(t1.ToSlice()));
// cornercase
Assert.That(child[FdbTuple.Empty].Key, Is.EqualTo(child.Key));
}
public void Test_FdbKeyRange_Test()
{
const int BEFORE = -1, INSIDE = 0, AFTER = +1;
FdbKeyRange range;
// range: [ "A", "Z" )
range = FdbKeyRange.Create(Slice.FromAscii("A"), Slice.FromAscii("Z"));
// Excluding the end: < "Z"
Assert.That(range.Test(Slice.FromAscii("\x00"), endIncluded: false), Is.EqualTo(BEFORE));
Assert.That(range.Test(Slice.FromAscii("@"), endIncluded: false), Is.EqualTo(BEFORE));
Assert.That(range.Test(Slice.FromAscii("A"), endIncluded: false), Is.EqualTo(INSIDE));
Assert.That(range.Test(Slice.FromAscii("Z"), endIncluded: false), Is.EqualTo(AFTER));
Assert.That(range.Test(Slice.FromAscii("Z\x00"), endIncluded: false), Is.EqualTo(AFTER));
Assert.That(range.Test(Slice.FromAscii("\xFF"), endIncluded: false), Is.EqualTo(AFTER));
// Including the end: <= "Z"
Assert.That(range.Test(Slice.FromAscii("\x00"), endIncluded: true), Is.EqualTo(BEFORE));
Assert.That(range.Test(Slice.FromAscii("@"), endIncluded: true), Is.EqualTo(BEFORE));
Assert.That(range.Test(Slice.FromAscii("A"), endIncluded: true), Is.EqualTo(INSIDE));
Assert.That(range.Test(Slice.FromAscii("Z"), endIncluded: true), Is.EqualTo(INSIDE));
Assert.That(range.Test(Slice.FromAscii("Z\x00"), endIncluded: true), Is.EqualTo(AFTER));
Assert.That(range.Test(Slice.FromAscii("\xFF"), endIncluded: true), Is.EqualTo(AFTER));
range = FdbKeyRange.Create(FdbTuple.Pack("A"), FdbTuple.Pack("Z"));
Assert.That(range.Test(FdbTuple.Create("@")), Is.EqualTo((BEFORE)));
Assert.That(range.Test(FdbTuple.Create("A")), Is.EqualTo((INSIDE)));
Assert.That(range.Test(FdbTuple.Create("Z")), Is.EqualTo((AFTER)));
Assert.That(range.Test(FdbTuple.Create("Z"), endIncluded: true), Is.EqualTo(INSIDE));
}
/// <summary>
/// Returns the list of names of all existing classes
/// </summary>
public Task <List <string> > AvailableClasses(IFdbReadOnlyTransaction tr)
{
return(tr.GetRange(this.Subspace.ToRange(FdbTuple.Create("class")))
.Where(kvp => { int _; return Int32.TryParse(kvp.Value.ToAscii(), out _); }) // (step 3)
.Select(kvp => this.Subspace.UnpackSingle <string>(kvp.Key))
.ToListAsync());
}
/// <summary>Remove an entity from the index</summary>
/// <param name="trans">Transaction to use</param>
/// <param name="id">Id of the entity that has been deleted</param>
/// <param name="value">Previous value of the entity in the index</param>
public void Remove([NotNull] IFdbTransaction trans, TId id, TValue value)
{
if (trans == null)
{
throw new ArgumentNullException("trans");
}
this.Location.Clear(trans, FdbTuple.Create(value, id));
}
public void Test_FdbKey_PrettyPrint()
{
// verify that the pretty printing of keys produce a user friendly output
Assert.That(FdbKey.Dump(Slice.Nil), Is.EqualTo("<null>"));
Assert.That(FdbKey.Dump(Slice.Empty), Is.EqualTo("<empty>"));
Assert.That(FdbKey.Dump(Slice.FromByte(0)), Is.EqualTo("<00>"));
Assert.That(FdbKey.Dump(Slice.FromByte(255)), Is.EqualTo("<FF>"));
Assert.That(FdbKey.Dump(Slice.Create(new byte[] { 0, 1, 2, 3, 4, 5, 6, 7 })), Is.EqualTo("<00><01><02><03><04><05><06><07>"));
Assert.That(FdbKey.Dump(Slice.Create(new byte[] { 255, 254, 253, 252, 251, 250, 249, 248 })), Is.EqualTo("<FF><FE><FD><FC><FB><FA><F9><F8>"));
Assert.That(FdbKey.Dump(Slice.FromString("hello")), Is.EqualTo("hello"));
Assert.That(FdbKey.Dump(Slice.FromString("héllø")), Is.EqualTo("h<C3><A9>ll<C3><B8>"));
// tuples should be decoded properly
Assert.That(FdbKey.Dump(FdbTuple.Pack(123)), Is.EqualTo("(123,)"), "Singleton tuples should end with a ','");
Assert.That(FdbKey.Dump(FdbTuple.Pack(Slice.FromAscii("hello"))), Is.EqualTo("('hello',)"), "ASCII strings should use single quotes");
Assert.That(FdbKey.Dump(FdbTuple.Pack("héllø")), Is.EqualTo("(\"héllø\",)"), "Unicode strings should use double quotes");
Assert.That(FdbKey.Dump(FdbTuple.Pack(Slice.Create(new byte[] { 1, 2, 3 }))), Is.EqualTo("(<01 02 03>,)"));
Assert.That(FdbKey.Dump(FdbTuple.Pack(123, 456)), Is.EqualTo("(123, 456)"), "Elements should be separated with a space, and not end up with ','");
Assert.That(FdbKey.Dump(FdbTuple.Pack(true, false, default(object))), Is.EqualTo("(1, 0, null)"), "Booleans should be displayed as numbers, and null should be in lowercase"); //note: even though it's tempting to using Python's "Nil", it's not very ".NETty"
Assert.That(FdbKey.Dump(FdbTuple.Pack(1.0d, Math.PI, Math.E)), Is.EqualTo("(1, 3.1415926535897931, 2.7182818284590451)"), "Doubles should used dot and have full precision (17 digits)");
Assert.That(FdbKey.Dump(FdbTuple.Pack(1.0f, (float)Math.PI, (float)Math.E)), Is.EqualTo("(1, 3.14159274, 2.71828175)"), "Singles should used dot and have full precision (10 digits)");
var guid = Guid.NewGuid();
Assert.That(FdbKey.Dump(FdbTuple.Pack(guid)), Is.EqualTo(String.Format("({0},)", guid.ToString("D"))), "GUIDs should be displayed as a string literal, without quotes");
var uuid128 = Uuid128.NewUuid();
Assert.That(FdbKey.Dump(FdbTuple.Pack(uuid128)), Is.EqualTo(String.Format("({0},)", uuid128.ToString("D"))), "Uuid128s should be displayed as a string literal, without quotes");
var uuid64 = Uuid64.NewUuid();
Assert.That(FdbKey.Dump(FdbTuple.Pack(uuid64)), Is.EqualTo(String.Format("({0},)", uuid64.ToString("D"))), "Uuid64s should be displayed as a string literal, without quotes");
// ranges should be decoded when possible
var key = FdbTuple.ToRange(FdbTuple.Create("hello"));
// "<02>hello<00><00>" .. "<02>hello<00><FF>"
Assert.That(FdbKey.PrettyPrint(key.Begin, FdbKey.PrettyPrintMode.Begin), Is.EqualTo("(\"hello\",).<00>"));
Assert.That(FdbKey.PrettyPrint(key.End, FdbKey.PrettyPrintMode.End), Is.EqualTo("(\"hello\",).<FF>"));
key = FdbKeyRange.StartsWith(FdbTuple.Pack("hello"));
// "<02>hello<00>" .. "<02>hello<01>"
Assert.That(FdbKey.PrettyPrint(key.Begin, FdbKey.PrettyPrintMode.Begin), Is.EqualTo("(\"hello\",)"));
Assert.That(FdbKey.PrettyPrint(key.End, FdbKey.PrettyPrintMode.End), Is.EqualTo("(\"hello\",) + 1"));
var t = FdbTuple.Pack(123);
Assert.That(FdbKey.PrettyPrint(t, FdbKey.PrettyPrintMode.Single), Is.EqualTo("(123,)"));
Assert.That(FdbKey.PrettyPrint(FdbTuple.ToRange(t).Begin, FdbKey.PrettyPrintMode.Begin), Is.EqualTo("(123,).<00>"));
Assert.That(FdbKey.PrettyPrint(FdbTuple.ToRange(t).End, FdbKey.PrettyPrintMode.End), Is.EqualTo("(123,).<FF>"));
}
public long GetCounter()
{
using (var tx = _le.BeginTransaction(TransactionBeginFlags.ReadOnly)) {
var key = FdbTuple.Create((byte)Tables.SysCounter);
var val = tx.Get(_ld, key.GetBytes());
if (val == null)
{
return(0);
}
return(BitConverter.ToInt64(val, 0));
}
}
public decimal GetItemQuantity(long id)
{
using (var tx = _le.BeginTransaction(TransactionBeginFlags.ReadOnly)) {
var key = FdbTuple.Create((byte)Tables.Quantity, id);
var val = tx.Get(_ld, key.GetBytes());
if (val == null)
{
return(0);
}
return(ToDecimal(val));
}
}
/// <summary>Insert a new document in the collection</summary>
public void Insert(IFdbTransaction trans, TDocument document)
{
if (trans == null)
{
throw new ArgumentNullException("trans");
}
if (document == null)
{
throw new ArgumentNullException("document");
}
var id = this.IdSelector(document);
if (id == null)
{
throw new InvalidOperationException("Cannot insert a document with a null identifier");
}
// encode the document
var packed = this.ValueEncoder.EncodeValue(document);
// Key Prefix = ...(id,)
var key = this.Location.Partial.EncodeKey(id);
// clear previous value
trans.ClearRange(FdbKeyRange.StartsWith(key));
int remaining = packed.Count;
if (remaining <= this.ChunkSize)
{ // stored as a single element
// Key = ...(id,)
trans.Set(key, packed);
}
else
{ // splits in as many chunks as necessary
// Key = ...(id, N) where N is the chunk index (0-based)
int p = 0;
int index = 0;
while (remaining > 0)
{
int sz = Math.Max(remaining, this.ChunkSize);
this.Location.Set(trans, FdbTuple.Create(id, index), packed.Substring(p, sz));
++index;
p += sz;
remaining -= sz;
}
}
}
public List <KeyValuePair <string, IFdbTuple> > Pack(TDictionary document)
{
var dic = new List <KeyValuePair <string, IFdbTuple> >(document.Count);
// convert everything, except the Id
foreach (var kvp in document)
{
if (!m_keyComparer.Equals(kvp.Key, this.IdName))
{
dic.Add(new KeyValuePair <string, IFdbTuple>(kvp.Key, FdbTuple.Create(kvp.Key)));
}
}
return(dic);
}
public void SetCounter(long id)
{
using (var tx = _le.BeginTransaction()) {
var key = FdbTuple.Create((byte)Tables.SysCounter);
if (id == 0)
{
tx.Delete(_ld, key.GetBytes());
}
else
{
tx.Put(_ld, key.GetBytes(), BitConverter.GetBytes(id));
}
tx.Commit();
}
}
public override FdbTuple <T1, T2> DecodeComposite(Slice encoded, int items)
{
if (items < 1 || items > 2)
{
throw new ArgumentOutOfRangeException("items", items, "Item count must be either 1 or 2");
}
var t = FdbTuple.Unpack(encoded).OfSize(items);
Contract.Assert(t != null);
return(FdbTuple.Create <T1, T2>(
t.Get <T1>(0),
items >= 2 ? t.Get <T2>(1) : default(T2)
));
}
public void SetItemQuantity(long id, decimal amount)
{
var key = FdbTuple.Create((byte)Tables.Quantity, id);
using (var tx = _le.BeginTransaction()) {
if (amount == 0)
{
tx.TryDelete(_ld, key.GetBytes());
}
else
{
tx.Put(_ld, key.GetBytes(), GetBytes(amount));
}
tx.Commit();
}
}
public decimal Count()
{
decimal total = 0M;
using (var tx = _le.BeginTransaction(TransactionBeginFlags.ReadOnly)) {
var prefix = FdbTuple.Create((byte)Tables.Quantity).ToSlice();
var range = FdbKeyRange.StartsWith(prefix);
var nums = InternalScan(tx, range, (slice, bytes) => ToDecimal(bytes));
foreach (var num in nums)
{
total += num;
}
}
return(total);
}
public KeyValuePair <IFdbTuple, Slice>[] Split(List <KeyValuePair <string, IFdbTuple> > document)
{
if (document == null)
{
throw new ArgumentNullException("document");
}
return(document
// don't include the id
.Where(kvp => !m_keyComparer.Equals(kvp.Key, this.IdName))
// convert into tuples
.Select(kvp => new KeyValuePair <IFdbTuple, Slice>(
FdbTuple.Create(kvp.Key),
FdbTuple.Create(kvp.Value).ToSlice()
))
.ToArray());
}
public void Test_FdbQueryRangeExpression()
{
var expr = FdbQueryExpressions.Range(
FdbTuple.Create("Foo").ToSelectorPair()
);
Console.WriteLine(expr);
Assert.That(expr, Is.Not.Null);
Assert.That(expr.Range.Begin.Key.ToString(), Is.EqualTo("<02>Foo<00>"));
Assert.That(expr.Range.End.Key.ToString(), Is.EqualTo("<02>Foo<01>"));
Assert.That(expr.Type, Is.EqualTo(typeof(IFdbAsyncEnumerable <KeyValuePair <Slice, Slice> >)));
Assert.That(expr.ElementType, Is.EqualTo(typeof(KeyValuePair <Slice, Slice>)));
Console.WriteLine(FdbQueryExpressions.ExplainSequence(expr));
}
public void Test_FdbQueryFilterExpression()
{
var expr = FdbQueryExpressions.Filter(
FdbQueryExpressions.RangeStartsWith(FdbTuple.Create("Hello", "World")),
(kvp) => kvp.Value.ToInt32() % 2 == 0
);
Console.WriteLine(expr);
Assert.That(expr, Is.Not.Null);
Assert.That(expr.Source, Is.Not.Null.And.InstanceOf <FdbQueryRangeExpression>());
Assert.That(expr.Filter, Is.Not.Null);
Assert.That(expr.Type, Is.EqualTo(typeof(IFdbAsyncEnumerable <KeyValuePair <Slice, Slice> >)));
Assert.That(expr.ElementType, Is.EqualTo(typeof(KeyValuePair <Slice, Slice>)));
Console.WriteLine(FdbQueryExpressions.ExplainSequence(expr));
}
public void Test_FdbQueryTransformExpression()
{
var expr = FdbQueryExpressions.Transform(
FdbQueryExpressions.RangeStartsWith(FdbTuple.Create("Hello", "World")),
(kvp) => kvp.Value.ToUnicode()
);
Console.WriteLine(expr);
Assert.That(expr, Is.Not.Null);
Assert.That(expr.Source, Is.Not.Null.And.InstanceOf <FdbQueryRangeExpression>());
Assert.That(expr.Transform, Is.Not.Null);
Assert.That(expr.Type, Is.EqualTo(typeof(IFdbAsyncEnumerable <string>)));
Assert.That(expr.ElementType, Is.EqualTo(typeof(string)));
Console.WriteLine(FdbQueryExpressions.ExplainSequence(expr));
}
public override FdbTuple <T1, T2> DecodeComposite(Slice encoded, int items)
{
if (items < 1 || items > 2)
{
throw new ArgumentOutOfRangeException("items", items, "Item count must be either 1 or 2");
}
var t = FdbTuple.Unpack(encoded);
Contract.Assert(t != null);
if (t.Count != items)
{
throw new ArgumentException(String.Format("Was expected {0} items, but decoded tuple only has {1}", items, t.Count));
}
return(FdbTuple.Create <T1, T2>(
t.Get <T1>(0),
items >= 2 ? t.Get <T2>(1) : default(T2)
));
}
/// <summary>Update the indexed values of an entity</summary>
/// <param name="trans">Transaction to use</param>
/// <param name="id">Id of the entity that has changed</param>
/// <param name="newValue">Previous value of this entity in the index</param>
/// <param name="previousValue">New value of this entity in the index</param>
/// <returns>True if a change was performed in the index; otherwise false (if <paramref name="previousValue"/> and <paramref name="newValue"/>)</returns>
/// <remarks>If <paramref name="newValue"/> and <paramref name="previousValue"/> are identical, then nothing will be done. Otherwise, the old index value will be deleted and the new value will be added</remarks>
public bool Update([NotNull] IFdbTransaction trans, TId id, TValue newValue, TValue previousValue)
{
if (!this.ValueComparer.Equals(newValue, previousValue))
{
// remove previous value
if (this.IndexNullValues || previousValue != null)
{
this.Location.Clear(trans, FdbTuple.Create(previousValue, id));
}
// add new value
if (this.IndexNullValues || newValue != null)
{
this.Location.Set(trans, FdbTuple.Create(newValue, id), Slice.Empty);
}
// cannot be both null, so we did at least something)
return(true);
}
return(false);
}
public void Test_Subspace_With_Binary_Prefix()
{
var subspace = new FdbSubspace(Slice.Create(new byte[] { 42, 255, 0, 127 }));
Assert.That(subspace.Key.ToString(), Is.EqualTo("*<FF><00><7F>"));
Assert.That(subspace.Copy(), Is.Not.SameAs(subspace));
Assert.That(subspace.Copy().Key, Is.EqualTo(subspace.Key));
// concat(Slice) should append the slice to the binary prefix directly
Assert.That(subspace.Concat(Slice.FromInt32(0x01020304)).ToString(), Is.EqualTo("*<FF><00><7F><04><03><02><01>"));
Assert.That(subspace.Concat(Slice.FromAscii("hello")).ToString(), Is.EqualTo("*<FF><00><7F>hello"));
// pack(...) should use tuple serialization
Assert.That(subspace.Pack(123).ToString(), Is.EqualTo("*<FF><00><7F><15>{"));
Assert.That(subspace.Pack("hello").ToString(), Is.EqualTo("*<FF><00><7F><02>hello<00>"));
Assert.That(subspace.Pack(Slice.FromAscii("world")).ToString(), Is.EqualTo("*<FF><00><7F><01>world<00>"));
Assert.That(subspace.Pack(FdbTuple.Create("hello", 123)).ToString(), Is.EqualTo("*<FF><00><7F><02>hello<00><15>{"));
// if we derive a tuple from this subspace, it should keep the binary prefix when converted to a key
var t = subspace.Append("world", 123, false);
Assert.That(t, Is.Not.Null);
Assert.That(t.Count, Is.EqualTo(3));
Assert.That(t.Get <string>(0), Is.EqualTo("world"));
Assert.That(t.Get <int>(1), Is.EqualTo(123));
Assert.That(t.Get <bool>(2), Is.False);
var k = t.ToSlice();
Assert.That(k.ToString(), Is.EqualTo("*<FF><00><7F><02>world<00><15>{<14>"));
// if we unpack the key with the binary prefix, we should get a valid tuple
var t2 = subspace.Unpack(k);
Assert.That(t2, Is.Not.Null);
Assert.That(t2.Count, Is.EqualTo(3));
Assert.That(t2.Get <string>(0), Is.EqualTo("world"));
Assert.That(t2.Get <int>(1), Is.EqualTo(123));
Assert.That(t2.Get <bool>(2), Is.False);
}
public override FdbTuple <T1, T2> DecodeComposite(Slice encoded, int count)
{
Contract.Requires(count > 0);
var reader = new SliceReader(encoded);
T1 key1 = default(T1);
T2 key2 = default(T2);
if (count >= 1)
{
key1 = m_codec1.DecodeOrderedSelfTerm(ref reader);
}
if (count >= 2)
{
key2 = m_codec2.DecodeOrderedSelfTerm(ref reader);
}
if (reader.HasMore)
{
throw new InvalidOperationException(String.Format("Unexpected data at the end of composite key after {0} items", count));
}
return(FdbTuple.Create <T1, T2>(key1, key2));
}
public void Test_Subspace_With_Tuple_Prefix()
{
var subspace = new FdbSubspace(FdbTuple.Create("hello"));
Assert.That(subspace.Key.ToString(), Is.EqualTo("<02>hello<00>"));
Assert.That(subspace.Copy(), Is.Not.SameAs(subspace));
Assert.That(subspace.Copy().Key, Is.EqualTo(subspace.Key));
// concat(Slice) should append the slice to the tuple prefix directly
Assert.That(subspace.Concat(Slice.FromInt32(0x01020304)).ToString(), Is.EqualTo("<02>hello<00><04><03><02><01>"));
Assert.That(subspace.Concat(Slice.FromAscii("world")).ToString(), Is.EqualTo("<02>hello<00>world"));
// pack(...) should use tuple serialization
Assert.That(subspace.Pack(123).ToString(), Is.EqualTo("<02>hello<00><15>{"));
Assert.That(subspace.Pack("world").ToString(), Is.EqualTo("<02>hello<00><02>world<00>"));
// even though the subspace prefix is a tuple, appending to it will only return the new items
var t = subspace.Append("world", 123, false);
Assert.That(t, Is.Not.Null);
Assert.That(t.Count, Is.EqualTo(3));
Assert.That(t.Get <string>(0), Is.EqualTo("world"));
Assert.That(t.Get <int>(1), Is.EqualTo(123));
Assert.That(t.Get <bool>(2), Is.False);
// but ToSlice() should include the prefix
var k = t.ToSlice();
Assert.That(k.ToString(), Is.EqualTo("<02>hello<00><02>world<00><15>{<14>"));
// if we unpack the key with the binary prefix, we should get a valid tuple
var t2 = subspace.Unpack(k);
Assert.That(t2, Is.Not.Null);
Assert.That(t2.Count, Is.EqualTo(3));
Assert.That(t2.Get <string>(0), Is.EqualTo("world"));
Assert.That(t2.Get <int>(1), Is.EqualTo(123));
Assert.That(t2.Get <bool>(2), Is.False);
}
public async Task Test_Can_Open_Database_With_Non_Empty_GlobalSpace()
{
// using a tuple prefix
using (var db = await Fdb.OpenAsync(null, "DB", new FdbSubspace(FdbTuple.Create("test")), false, this.Cancellation))
{
Assert.That(db, Is.Not.Null);
Assert.That(db.GlobalSpace, Is.Not.Null);
Assert.That(db.GlobalSpace.Key.ToString(), Is.EqualTo("<02>test<00>"));
var subspace = db.Partition("hello");
Assert.That(subspace.Key.ToString(), Is.EqualTo("<02>test<00><02>hello<00>"));
// keys inside the global space are invlaid
Assert.That(db.IsKeyValid(FdbTuple.Pack("test", 123)), Is.True);
// keys outside the global space are invlaid
Assert.That(db.IsKeyValid(Slice.Create(new byte[] { 42 })), Is.False);
}
// using a random binary prefix
using (var db = await Fdb.OpenAsync(null, "DB", new FdbSubspace(Slice.Create(new byte[] { 42, 255, 0, 90 })), false, this.Cancellation))
{
Assert.That(db, Is.Not.Null);
Assert.That(db.GlobalSpace, Is.Not.Null);
Assert.That(db.GlobalSpace.Key.ToString(), Is.EqualTo("*<FF><00>Z"));
var subspace = db.Partition("hello");
Assert.That(subspace.Key.ToString(), Is.EqualTo("*<FF><00>Z<02>hello<00>"));
// keys inside the global space are invlaid
Assert.That(db.IsKeyValid(Slice.Unescape("*<FF><00>Z123")), Is.True);
// keys outside the global space are invlaid
Assert.That(db.IsKeyValid(Slice.Create(new byte[] { 123 })), Is.False);
Assert.That(db.IsKeyValid(Slice.Unescape("*<FF>")), Is.False);
}
}
public static byte[] CreateKey(byte table)
{
return(FdbTuple.Create(table).ToSlice().GetBytes());
}
public static byte[] CreateKey <T1, T2, T3, T4, T5, T6>(byte table, T1 t1, T2 t2, T3 t3, T4 t4,
T5 t5, T6 t6)
{
return(FdbTuple.Create(table, t1, t2, t3, t4, t5, t6).ToSlice().GetBytes());
}
public static byte[] CreateKey <T1, T2, T3>(byte table, T1 t1, T2 t2, T3 t3)
{
return(FdbTuple.Create(table, t1, t2, t3).ToSlice().GetBytes());
}
public static byte[] CreateKey <T1>(byte table, T1 t1)
{
return(FdbTuple.Create(table, t1).ToSlice().GetBytes());
}
private static async Task MainAsync(string[] args, CancellationToken cancel)
{
#region Options Parsing...
string clusterFile = null;
var dbName = "DB";
var partition = new string[0];
bool showHelp = false;
int timeout = 30;
int maxRetries = 10;
string execCommand = null;
var opts = new OptionSet()
{
{
"c|connfile=",
"The path of a file containing the connection string for the FoundationDB cluster.",
v => clusterFile = v
},
{
"p|partition=",
"The name of the database partition to open.",
v => partition = v.Trim().Split('/')
},
{
"t|timeout=",
"Default timeout (in seconds) for failed transactions.",
(int v) => timeout = v
},
{
"r|retries=",
"Default max retry count for failed transactions.",
(int v) => maxRetries = v
},
{
"exec=",
"Execute this command, and exits immediately.",
v => execCommand = v
},
{
"h|help",
"Show this help and exit.",
v => showHelp = v != null
}
};
var extra = opts.Parse(args);
if (showHelp)
{
//TODO!
opts.WriteOptionDescriptions(Console.Out);
return;
}
string startCommand = null;
if (!string.IsNullOrEmpty(execCommand))
{
startCommand = execCommand;
}
else if (extra.Count > 0)
{ // the remainder of the command line will be the first command to execute
startCommand = String.Join(" ", extra);
}
#endregion
bool stop = false;
Db = null;
try
{
Db = await ChangeDatabase(clusterFile, dbName, partition, cancel);
Db.DefaultTimeout = Math.Max(0, timeout) * 1000;
Db.DefaultRetryLimit = Math.Max(0, maxRetries);
Console.WriteLine("Using API v" + Fdb.ApiVersion + " (max " + Fdb.GetMaxApiVersion() + ")");
Console.WriteLine("Cluster file: " + (clusterFile ?? "<default>"));
Console.WriteLine();
Console.WriteLine("FoundationDB Shell menu:");
Console.WriteLine("\tdir\tShow the content of the current directory");
Console.WriteLine("\ttree\tShow all the directories under the current directory");
Console.WriteLine("\tsampling\tDisplay statistics on random shards from the database");
Console.WriteLine("\tcoordinators\tShow the current coordinators for the cluster");
Console.WriteLine("\tmem\tShow memory usage statistics");
Console.WriteLine("\tgc\tTrigger garbage collection");
Console.WriteLine("\tquit\tQuit");
Console.WriteLine("Ready...");
var le = new LineEditor("FDBShell");
string[] cmds = new string[]
{
"cd",
"coordinators",
"count",
"dir",
"exit",
"gc",
"help",
"layer",
"map",
"mem",
"mkdir",
"mv",
"partition",
"pwd",
"quit",
"ren",
"rmdir",
"sampling",
"shards",
"show",
"status",
"topology",
"tree",
"version",
"wide",
};
le.AutoCompleteEvent = (txt, pos) =>
{
string[] res;
int p = txt.IndexOf(' ');
if (p > 0)
{
string cmd = txt.Substring(0, p);
string arg = txt.Substring(p + 1);
if (cmd == "cd")
{ // handle completion for directories
// txt: "cd foo" => prefix = "foo"
// txt: "cd foobar/b" => prefix = "b"
string path = CurrentDirectoryPath;
string prefix = "";
string search = arg;
p = arg.LastIndexOf('/');
if (p > 0)
{
path = Path.Combine(path, arg.Substring(0, p));
search = arg.Substring(p + 1);
prefix = arg.Substring(0, p + 1);
}
var subdirs = RunAsyncCommand((db, log, ct) => AutoCompleteDirectories(path, db, log, ct), cancel).GetAwaiter().GetResult();
if (!subdirs.HasValue || subdirs.Value == null)
{
return(new LineEditor.Completion(txt, null));
}
res = subdirs.Value
.Where(s => s.StartsWith(search, StringComparison.Ordinal))
.Select(s => (cmd + " " + prefix + s).Substring(txt.Length))
.ToArray();
return(new LineEditor.Completion(txt, res));
}
// unknown command
return(new LineEditor.Completion(txt, null));
}
// list of commands
res = cmds
.Where(cmd => cmd.StartsWith(txt, StringComparison.OrdinalIgnoreCase))
.Select(cmd => cmd.Substring(txt.Length))
.ToArray();
return(new LineEditor.Completion(txt, res));
};
le.TabAtStartCompletes = true;
string prompt = null;
Action <string> updatePrompt = (path) => { prompt = String.Format("fdb:{0}> ", path); };
updatePrompt(CurrentDirectoryPath);
while (!stop)
{
string s = startCommand != null ? startCommand : le.Edit(prompt, "");
startCommand = null;
if (s == null)
{
break;
}
var tokens = s.Trim().Split(new [] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
string cmd = tokens.Length > 0 ? tokens[0] : String.Empty;
string prm = tokens.Length > 1 ? tokens[1] : String.Empty;
var extras = tokens.Length > 2 ? FdbTuple.CreateRange <string>(tokens.Skip(2)) : FdbTuple.Empty;
var trimmedCommand = cmd.Trim().ToLowerInvariant();
switch (trimmedCommand)
{
case "":
{
continue;
}
case "log":
{
LogCommand(prm, Console.Out);
break;
}
case "version":
{
await VersionCommand(prm, clusterFile, Console.Out, cancel);
break;
}
case "tree":
{
var path = ParsePath(CombinePath(CurrentDirectoryPath, prm));
await RunAsyncCommand((db, log, ct) => BasicCommands.Tree(path, extras, db, log, ct), cancel);
break;
}
case "map":
{
var path = ParsePath(CombinePath(CurrentDirectoryPath, prm));
await RunAsyncCommand((db, log, ct) => BasicCommands.Map(path, extras, db, log, ct), cancel);
break;
}
case "dir":
case "ls":
{
var path = ParsePath(CombinePath(CurrentDirectoryPath, prm));
await RunAsyncCommand((db, log, ct) => BasicCommands.Dir(path, extras, BasicCommands.DirectoryBrowseOptions.Default, db, log, ct), cancel);
break;
}
case "ll":
{
var path = ParsePath(CombinePath(CurrentDirectoryPath, prm));
await RunAsyncCommand((db, log, ct) => BasicCommands.Dir(path, extras, BasicCommands.DirectoryBrowseOptions.ShowCount, db, log, ct), cancel);
break;
}
case "count":
{
var path = ParsePath(CombinePath(CurrentDirectoryPath, prm));
await RunAsyncCommand((db, log, ct) => BasicCommands.Count(path, extras, db, log, ct), cancel);
break;
}
case "show":
case "top":
{
var path = ParsePath(CurrentDirectoryPath);
await RunAsyncCommand((db, log, ct) => BasicCommands.Show(path, extras, false, db, log, ct), cancel);
break;
}
case "last":
{
var path = ParsePath(CurrentDirectoryPath);
await RunAsyncCommand((db, log, ct) => BasicCommands.Show(path, extras, true, db, log, ct), cancel);
break;
}
case "cd":
case "pwd":
{
if (!string.IsNullOrEmpty(prm))
{
var newPath = CombinePath(CurrentDirectoryPath, prm);
var res = await RunAsyncCommand((db, log, ct) => BasicCommands.TryOpenCurrentDirectoryAsync(ParsePath(newPath), db, ct), cancel);
if (res == null)
{
Console.WriteLine("# Directory {0} does not exist!", newPath);
Console.Beep();
}
else
{
CurrentDirectoryPath = newPath;
updatePrompt(CurrentDirectoryPath);
}
}
else
{
var res = await RunAsyncCommand((db, log, ct) => BasicCommands.TryOpenCurrentDirectoryAsync(ParsePath(CurrentDirectoryPath), db, ct), cancel);
if (res.GetValueOrDefault() == null)
{
Console.WriteLine("# Directory {0} does not exist anymore", CurrentDirectoryPath);
}
else
{
Console.WriteLine("# {0}", res);
}
}
break;
}
case "mkdir":
case "md":
{ // "mkdir DIRECTORYNAME"
if (!string.IsNullOrEmpty(prm))
{
var path = ParsePath(CombinePath(CurrentDirectoryPath, prm));
await RunAsyncCommand((db, log, ct) => BasicCommands.CreateDirectory(path, extras, db, log, ct), cancel);
}
break;
}
case "rmdir":
{ // "rmdir DIRECTORYNAME"
if (!string.IsNullOrEmpty(prm))
{
var path = ParsePath(CombinePath(CurrentDirectoryPath, prm));
await RunAsyncCommand((db, log, ct) => BasicCommands.RemoveDirectory(path, extras, db, log, ct), cancel);
}
break;
}
case "mv":
case "ren":
{ // "mv SOURCE DESTINATION"
var srcPath = ParsePath(CombinePath(CurrentDirectoryPath, prm));
var dstPath = ParsePath(CombinePath(CurrentDirectoryPath, extras.Get <string>(0)));
await RunAsyncCommand((db, log, ct) => BasicCommands.MoveDirectory(srcPath, dstPath, extras.Substring(1), db, log, ct), cancel);
break;
}
case "layer":
{
if (string.IsNullOrEmpty(prm))
{ // displays the layer id of the current folder
var path = ParsePath(CurrentDirectoryPath);
await RunAsyncCommand((db, log, ct) => BasicCommands.ShowDirectoryLayer(path, extras, db, log, ct), cancel);
}
else
{ // change the layer id of the current folder
prm = prm.Trim();
// double or single quotes can be used to escape the value
if (prm.Length >= 2 && (prm.StartsWith("'") && prm.EndsWith("'")) || (prm.StartsWith("\"") && prm.EndsWith("\"")))
{
prm = prm.Substring(1, prm.Length - 2);
}
var path = ParsePath(CurrentDirectoryPath);
await RunAsyncCommand((db, log, ct) => BasicCommands.ChangeDirectoryLayer(path, prm, extras, db, log, ct), cancel);
}
break;
}
case "mkpart":
{ // "mkpart PARTITIONNAME"
if (!string.IsNullOrEmpty(prm))
{
var path = ParsePath(CombinePath(CurrentDirectoryPath, prm));
await RunAsyncCommand((db, log, ct) => BasicCommands.CreateDirectory(path, FdbTuple.Create(FdbDirectoryPartition.LayerId).Concat(extras), db, log, ct), cancel);
}
break;
}
case "topology":
{
await RunAsyncCommand((db, log, ct) => BasicCommands.Topology(null, extras, db, log, ct), cancel);
break;
}
case "shards":
{
var path = ParsePath(CombinePath(CurrentDirectoryPath, prm));
await RunAsyncCommand((db, log, ct) => BasicCommands.Shards(path, extras, db, log, ct), cancel);
break;
}
case "sampling":
{
var path = ParsePath(CombinePath(CurrentDirectoryPath, prm));
await RunAsyncCommand((db, log, ct) => BasicCommands.Sampling(path, extras, db, log, ct), cancel);
break;
}
case "coordinators":
{
await RunAsyncCommand((db, log, ct) => CoordinatorsCommand(db, log, ct), cancel);
break;
}
case "partition":
{
if (string.IsNullOrEmpty(prm))
{
Console.WriteLine("# Current partition is {0}", String.Join("/", partition));
//TODO: browse existing partitions ?
break;
}
var newPartition = prm.Split(new[] { '/' }, StringSplitOptions.RemoveEmptyEntries);
IFdbDatabase newDb = null;
try
{
newDb = await ChangeDatabase(clusterFile, dbName, newPartition, cancel);
}
catch (Exception)
{
if (newDb != null)
{
newDb.Dispose();
}
newDb = null;
throw;
}
finally
{
if (newDb != null)
{
if (Db != null)
{
Db.Dispose(); Db = null;
}
Db = newDb;
partition = newPartition;
Console.WriteLine("# Changed partition to {0}", partition);
}
}
break;
}
case "q":
case "x":
case "quit":
case "exit":
case "bye":
{
stop = true;
break;
}
case "gc":
{
long before = GC.GetTotalMemory(false);
Console.Write("Collecting garbage...");
GC.Collect();
GC.WaitForPendingFinalizers();
GC.Collect();
Console.WriteLine(" Done");
long after = GC.GetTotalMemory(false);
Console.WriteLine("- before = " + before.ToString("N0"));
Console.WriteLine("- after = " + after.ToString("N0"));
Console.WriteLine("- delta = " + (before - after).ToString("N0"));
break;
}
case "mem":
{
Console.WriteLine("Memory usage:");
Console.WriteLine("- Working Set : " + PerfCounters.WorkingSet.NextValue().ToString("N0") + " (peak " + PerfCounters.WorkingSetPeak.NextValue().ToString("N0") + ")");
Console.WriteLine("- Virtual Bytes: " + PerfCounters.VirtualBytes.NextValue().ToString("N0") + " (peak " + PerfCounters.VirtualBytesPeak.NextValue().ToString("N0") + ")");
Console.WriteLine("- Private Bytes: " + PerfCounters.PrivateBytes.NextValue().ToString("N0"));
Console.WriteLine("- Managed Mem : " + GC.GetTotalMemory(false).ToString("N0"));
Console.WriteLine("- BytesInAlHeap: " + PerfCounters.ClrBytesInAllHeaps.NextValue().ToString("N0"));
break;
}
case "wide":
{
Console.WindowWidth = 160;
break;
}
case "status":
case "wtf":
{
var result = await RunAsyncCommand((_, log, ct) => FdbCliCommands.RunFdbCliCommand("status details", null, clusterFile, log, ct), cancel);
if (result.HasFailed)
{
break;
}
if (result.Value.ExitCode != 0)
{
Console.WriteLine("# fdbcli exited with code {0}", result.Value.ExitCode);
Console.WriteLine("> StdErr:");
Console.WriteLine(result.Value.StdErr);
Console.WriteLine("> StdOut:");
}
Console.WriteLine(result.Value.StdOut);
break;
}
default:
{
Console.WriteLine(string.Format("Unknown command : '{0}'", trimmedCommand));
break;
}
}
if (!string.IsNullOrEmpty(execCommand))
{ // only run one command, and then exit
break;
}
}
}
finally
{
if (Db != null)
{
Db.Dispose();
}
}
}