public static async Task Shards(string[] path, IFdbTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct) { var ranges = await Fdb.System.GetChunksAsync(db, FdbKey.MinValue, FdbKey.MaxValue, ct); Console.WriteLine("Found {0} shards in the whole cluster", ranges.Count); // look if there is something under there var folder = (await TryOpenCurrentDirectoryAsync(path, db, ct)) as FdbDirectorySubspace; if (folder != null) { var r = FdbKeyRange.StartsWith(folder.Copy().Key); Console.WriteLine("Searching for shards that intersect with /{0} ...", String.Join("/", path)); ranges = await Fdb.System.GetChunksAsync(db, r, ct); Console.WriteLine("Found {0} ranges intersecting {1}:", ranges.Count, r); var last = Slice.Empty; foreach (var range in ranges) { Console.Write("> " + FdbKey.Dump(range.Begin) + " ..."); long count = await Fdb.System.EstimateCountAsync(db, range, ct); Console.WriteLine(" {0:N0}", count); last = range.End; //TODO: we can probably get more details on this shard looking in the system keyspace (where it is, how many replicas, ...) } Console.WriteLine("> ... " + FdbKey.Dump(last)); } //Console.WriteLine("Found " + ranges.Count + " shards in the cluster"); //TODO: shards that intersect the current directory }
/// <summary>Return all fields of an hashset</summary> /// <param name="trans">Transaction that will be used for this request</param> /// <param name="id">Unique identifier of the hashset</param> /// <returns>Dictionary containing, for all fields, their associated values</returns> public async Task <IDictionary <string, Slice> > GetAsync(IFdbReadOnlyTransaction trans, IFdbTuple id) { if (trans == null) { throw new ArgumentNullException("trans"); } if (id == null) { throw new ArgumentNullException("id"); } var prefix = GetKey(id); var results = new Dictionary <string, Slice>(StringComparer.OrdinalIgnoreCase); await trans .GetRange(FdbKeyRange.StartsWith(prefix)) .ForEachAsync((kvp) => { string field = this.Subspace.UnpackLast <string>(kvp.Key); results[field] = kvp.Value; }) .ConfigureAwait(false); return(results); }
/// <summary>Remove all the values for a specific key</summary> /// <param name="trans"></param> /// <param name="key"></param> /// <returns></returns> public void Remove([NotNull] IFdbTransaction trans, TKey key) { if (trans == null) { throw new ArgumentNullException("trans"); } trans.ClearRange(FdbKeyRange.StartsWith(this.Location.Partial.Keys.Encode(key))); }
protected virtual Task <List <Slice> > LoadPartsAsync(IFdbReadOnlyTransaction trans, TId id) { var key = this.Location.Partial.EncodeKey(id); return(trans .GetRange(FdbKeyRange.StartsWith(key)) //TODO: options ? .Select(kvp => kvp.Value) .ToListAsync()); }
private void ClearTask(IFdbTransaction tr, Slice taskId) { tr.Annotate("Deleting task {0}", taskId.ToAsciiOrHexaString()); // clear all metadata about the task tr.ClearRange(FdbKeyRange.StartsWith(this.TaskStore.Pack(taskId))); // decrement pending number of tasks this.Counters.Decrement(tr, COUNTER_PENDING_TASKS); }
public FdbRangeQuery <TId> Lookup(IFdbReadOnlyTransaction trans, TValue value, bool reverse = false) { var prefix = this.Location.Partial.Keys.Encode(value); return(trans .GetRange(FdbKeyRange.StartsWith(prefix), new FdbRangeOptions { Reverse = reverse }) .Select((kvp) => this.Location.Keys.Decode(kvp.Key).Item2)); }
public void Test_FdbKey_PrettyPrint() { // verify that the pretty printing of keys produce a user friendly output Assert.That(FdbKey.Dump(Slice.Nil), Is.EqualTo("<null>")); Assert.That(FdbKey.Dump(Slice.Empty), Is.EqualTo("<empty>")); Assert.That(FdbKey.Dump(Slice.FromByte(0)), Is.EqualTo("<00>")); Assert.That(FdbKey.Dump(Slice.FromByte(255)), Is.EqualTo("<FF>")); Assert.That(FdbKey.Dump(Slice.Create(new byte[] { 0, 1, 2, 3, 4, 5, 6, 7 })), Is.EqualTo("<00><01><02><03><04><05><06><07>")); Assert.That(FdbKey.Dump(Slice.Create(new byte[] { 255, 254, 253, 252, 251, 250, 249, 248 })), Is.EqualTo("<FF><FE><FD><FC><FB><FA><F9><F8>")); Assert.That(FdbKey.Dump(Slice.FromString("hello")), Is.EqualTo("hello")); Assert.That(FdbKey.Dump(Slice.FromString("héllø")), Is.EqualTo("h<C3><A9>ll<C3><B8>")); // tuples should be decoded properly Assert.That(FdbKey.Dump(FdbTuple.Pack(123)), Is.EqualTo("(123,)"), "Singleton tuples should end with a ','"); Assert.That(FdbKey.Dump(FdbTuple.Pack(Slice.FromAscii("hello"))), Is.EqualTo("('hello',)"), "ASCII strings should use single quotes"); Assert.That(FdbKey.Dump(FdbTuple.Pack("héllø")), Is.EqualTo("(\"héllø\",)"), "Unicode strings should use double quotes"); Assert.That(FdbKey.Dump(FdbTuple.Pack(Slice.Create(new byte[] { 1, 2, 3 }))), Is.EqualTo("(<01 02 03>,)")); Assert.That(FdbKey.Dump(FdbTuple.Pack(123, 456)), Is.EqualTo("(123, 456)"), "Elements should be separated with a space, and not end up with ','"); Assert.That(FdbKey.Dump(FdbTuple.Pack(true, false, default(object))), Is.EqualTo("(1, 0, null)"), "Booleans should be displayed as numbers, and null should be in lowercase"); //note: even though it's tempting to using Python's "Nil", it's not very ".NETty" Assert.That(FdbKey.Dump(FdbTuple.Pack(1.0d, Math.PI, Math.E)), Is.EqualTo("(1, 3.1415926535897931, 2.7182818284590451)"), "Doubles should used dot and have full precision (17 digits)"); Assert.That(FdbKey.Dump(FdbTuple.Pack(1.0f, (float)Math.PI, (float)Math.E)), Is.EqualTo("(1, 3.14159274, 2.71828175)"), "Singles should used dot and have full precision (10 digits)"); var guid = Guid.NewGuid(); Assert.That(FdbKey.Dump(FdbTuple.Pack(guid)), Is.EqualTo(String.Format("({0},)", guid.ToString("D"))), "GUIDs should be displayed as a string literal, without quotes"); var uuid128 = Uuid128.NewUuid(); Assert.That(FdbKey.Dump(FdbTuple.Pack(uuid128)), Is.EqualTo(String.Format("({0},)", uuid128.ToString("D"))), "Uuid128s should be displayed as a string literal, without quotes"); var uuid64 = Uuid64.NewUuid(); Assert.That(FdbKey.Dump(FdbTuple.Pack(uuid64)), Is.EqualTo(String.Format("({0},)", uuid64.ToString("D"))), "Uuid64s should be displayed as a string literal, without quotes"); // ranges should be decoded when possible var key = FdbTuple.ToRange(FdbTuple.Create("hello")); // "<02>hello<00><00>" .. "<02>hello<00><FF>" Assert.That(FdbKey.PrettyPrint(key.Begin, FdbKey.PrettyPrintMode.Begin), Is.EqualTo("(\"hello\",).<00>")); Assert.That(FdbKey.PrettyPrint(key.End, FdbKey.PrettyPrintMode.End), Is.EqualTo("(\"hello\",).<FF>")); key = FdbKeyRange.StartsWith(FdbTuple.Pack("hello")); // "<02>hello<00>" .. "<02>hello<01>" Assert.That(FdbKey.PrettyPrint(key.Begin, FdbKey.PrettyPrintMode.Begin), Is.EqualTo("(\"hello\",)")); Assert.That(FdbKey.PrettyPrint(key.End, FdbKey.PrettyPrintMode.End), Is.EqualTo("(\"hello\",) + 1")); var t = FdbTuple.Pack(123); Assert.That(FdbKey.PrettyPrint(t, FdbKey.PrettyPrintMode.Single), Is.EqualTo("(123,)")); Assert.That(FdbKey.PrettyPrint(FdbTuple.ToRange(t).Begin, FdbKey.PrettyPrintMode.Begin), Is.EqualTo("(123,).<00>")); Assert.That(FdbKey.PrettyPrint(FdbTuple.ToRange(t).End, FdbKey.PrettyPrintMode.End), Is.EqualTo("(123,).<FF>")); }
private Task Scenario2(IFdbTransaction tr) { var location = FdbSubspace.Create(Slice.FromAscii("TEST")); tr.ClearRange(FdbKeyRange.StartsWith(location.Key)); for (int i = 0; i < 10; i++) { tr.Set(location.Pack(i), Slice.FromString("value of " + i)); } return(Task.FromResult <object>(null)); }
/// <summary>Remove all fields of an hashset</summary> /// <param name="id"></param> public void Delete(IFdbTransaction trans, IFdbTuple id) { if (trans == null) { throw new ArgumentNullException("trans"); } if (id == null) { throw new ArgumentNullException("id"); } // remove all fields of the hash trans.ClearRange(FdbKeyRange.StartsWith(GetKey(id))); }
/// <summary>Insert a new document in the collection</summary> public void Insert(IFdbTransaction trans, TDocument document) { if (trans == null) { throw new ArgumentNullException("trans"); } if (document == null) { throw new ArgumentNullException("document"); } var id = this.IdSelector(document); if (id == null) { throw new InvalidOperationException("Cannot insert a document with a null identifier"); } // encode the document var packed = this.ValueEncoder.EncodeValue(document); // Key Prefix = ...(id,) var key = this.Location.Partial.EncodeKey(id); // clear previous value trans.ClearRange(FdbKeyRange.StartsWith(key)); int remaining = packed.Count; if (remaining <= this.ChunkSize) { // stored as a single element // Key = ...(id,) trans.Set(key, packed); } else { // splits in as many chunks as necessary // Key = ...(id, N) where N is the chunk index (0-based) int p = 0; int index = 0; while (remaining > 0) { int sz = Math.Max(remaining, this.ChunkSize); this.Location.Set(trans, FdbTuple.Create(id, index), packed.Substring(p, sz)); ++index; p += sz; remaining -= sz; } } }
/// <summary>Delete a document from the collection</summary> /// <param name="trans"></param> /// <param name="id"></param> public void Delete(IFdbTransaction trans, TId id) { if (trans == null) { throw new ArgumentNullException("trans"); } if (id == null) { throw new ArgumentNullException("id"); } var key = this.Location.Partial.EncodeKey(id); trans.ClearRange(FdbKeyRange.StartsWith(key)); }
/// <summary>Delete a document from the collection</summary> /// <param name="trans"></param> /// <param name="ids"></param> public void DeleteMultiple(IFdbTransaction trans, IEnumerable <TId> ids) { if (trans == null) { throw new ArgumentNullException("trans"); } if (ids == null) { throw new ArgumentNullException("ids"); } foreach (var key in this.Location.Partial.EncodeKeyRange(ids)) { trans.ClearRange(FdbKeyRange.StartsWith(key)); } }
public decimal Count() { decimal total = 0M; using (var tx = _le.BeginTransaction(TransactionBeginFlags.ReadOnly)) { var prefix = FdbTuple.Create((byte)Tables.Quantity).ToSlice(); var range = FdbKeyRange.StartsWith(prefix); var nums = InternalScan(tx, range, (slice, bytes) => ToDecimal(bytes)); foreach (var num in nums) { total += num; } } return(total); }
public IFdbAsyncEnumerable <KeyValuePair <TValue, long> > GetCounts([NotNull] IFdbReadOnlyTransaction trans, TKey key) { var range = FdbKeyRange.StartsWith(this.Location.Partial.Keys.Encode(key)); var query = trans .GetRange(range) .Select(kvp => new KeyValuePair <TValue, long>(this.Location.Keys.Decode(kvp.Key).Item2, kvp.Value.ToInt64())); if (this.AllowNegativeValues) { return(query); } else { return(query.Where(kvp => kvp.Value > 0)); } }
public void Test_FdbKeyRange_StartsWith() { FdbKeyRange range; // "abc" => [ "abc", "abd" ) range = FdbKeyRange.StartsWith(Slice.FromAscii("abc")); Assert.That(range.Begin, Is.EqualTo(Slice.FromAscii("abc"))); Assert.That(range.End, Is.EqualTo(Slice.FromAscii("abd"))); // "" => ArgumentException Assert.That(() => FdbKeyRange.PrefixedBy(Slice.Empty), Throws.InstanceOf <ArgumentException>()); // "\xFF" => ArgumentException Assert.That(() => FdbKeyRange.PrefixedBy(Slice.FromAscii("\xFF")), Throws.InstanceOf <ArgumentException>()); // null => ArgumentException Assert.That(() => FdbKeyRange.PrefixedBy(Slice.Nil), Throws.InstanceOf <ArgumentException>()); }
static IEnumerable <TOut> InternalScan <TOut>(Tx tx, byte[] key, Func <Slice, byte[], TOut> convert, string traceName, int skip = 0) { tx.TraceStart(traceName); ushort x = 0; var slice = Slice.Create(key); var range = FdbKeyRange.StartsWith(slice); using (var c = tx.CreateCursor()) { if (!c.MoveToFirstAfter(key)) { tx.TraceStop(traceName, x); yield break; } var pair = c.Current; for (var i = 0; i < int.MaxValue; i++) { var current = Slice.Create(pair.Key); if (!range.Contains(current)) { tx.TraceStop(traceName, x); break; } if (i >= skip) { x += 1; yield return(convert(current, pair.Value)); } if (!c.MoveNext()) { tx.TraceStop(traceName, x); break; } pair = c.Current; } } }
public static void DeleteRange(Tx tx, byte[] key, string traceName) { tx.TraceStart(traceName); ushort x = 0; var slice = Slice.Create(key); var range = FdbKeyRange.StartsWith(slice); using (var c = tx.CreateCursor()) { if (!c.MoveToFirstAfter(key)) { tx.TraceStop(traceName, x); return; } var pair = c.Current; for (var i = 0; i < int.MaxValue; i++) { var current = Slice.Create(pair.Key); if (!range.Contains(current)) { tx.TraceStop(traceName, x); break; } x += 1; tx.Delete(pair.Key); if (!c.MoveNext()) { tx.TraceStop(traceName, x); break; } pair = c.Current; } } tx.TraceStop(traceName); }
/// <summary>Return the list the names of all fields of an hashset</summary> /// <param name="trans">Transaction that will be used for this request</param> /// <param name="id">Unique identifier of the hashset</param> /// <returns>List of all fields. If the list is empty, the hashset does not exist</returns> public Task <List <string> > GetKeys(IFdbReadOnlyTransaction trans, IFdbTuple id, CancellationToken cancellationToken = default(CancellationToken)) { //note: As of Beta2, FDB does not have a fdb_get_range that only return the keys. That means that we will have to also read the values from the db, in order to just get the names of the fields :( //TODO: find a way to optimize this ? if (trans == null) { throw new ArgumentNullException("trans"); } if (id == null) { throw new ArgumentNullException("id"); } var prefix = GetKey(id); var results = new Dictionary <string, Slice>(StringComparer.OrdinalIgnoreCase); return(trans .GetRange(FdbKeyRange.StartsWith(prefix)) .Select((kvp) => ParseFieldKey(FdbTuple.Unpack(kvp.Key))) .ToListAsync(cancellationToken)); }
public IFdbAsyncEnumerable <TValue> Get([NotNull] IFdbReadOnlyTransaction trans, TKey key) { if (trans == null) { throw new ArgumentNullException("trans"); } var range = FdbKeyRange.StartsWith(this.Location.Partial.Keys.Encode(key)); if (this.AllowNegativeValues) { return(trans .GetRange(range) .Select(kvp => this.Location.Keys.Decode(kvp.Key).Item2)); } else { return(trans .GetRange(range) .Where(kvp => kvp.Value.ToInt64() > 0) // we need to filter out zero or negative values (possible artefacts) .Select(kvp => this.Location.Keys.Decode(kvp.Key).Item2)); } }
public static async Task DumpSubspace([NotNull] IFdbReadOnlyTransaction tr, [NotNull] FdbSubspace subspace) { Assert.That(tr, Is.Not.Null); Console.WriteLine("Dumping content of subspace " + subspace.ToString() + " :"); int count = 0; await tr .GetRange(FdbKeyRange.StartsWith(subspace.Key)) .ForEachAsync((kvp) => { var key = subspace.Extract(kvp.Key); ++count; string keyDump = null; try { // attemps decoding it as a tuple keyDump = key.ToTuple().ToString(); } catch (Exception) { // not a tuple, dump as bytes keyDump = "'" + key.ToString() + "'"; } Console.WriteLine("- " + keyDump + " = " + kvp.Value.ToString()); }); if (count == 0) { Console.WriteLine("> empty !"); } else { Console.WriteLine("> Found " + count + " values"); } }
public async Task Test_Can_Log_A_Transaction() { const int N = 10; using (var db = await OpenTestPartitionAsync()) { var location = await GetCleanDirectory(db, "Logging"); // note: ensure that all methods are JITed await db.ReadWriteAsync(async (tr) => { await tr.GetReadVersionAsync(); tr.Set(location.Pack("Warmup", 0), Slice.FromInt32(1)); tr.Clear(location.Pack("Warmup", 1)); await tr.GetAsync(location.Pack("Warmup", 2)); await tr.GetRange(FdbKeyRange.StartsWith(location.Pack("Warmup", 3))).ToListAsync(); tr.ClearRange(location.Pack("Warmup", 4), location.Pack("Warmup", 5)); }, this.Cancellation); await db.WriteAsync((tr) => { var rnd = new Random(); tr.Set(location.Pack("One"), Slice.FromString("111111")); tr.Set(location.Pack("Two"), Slice.FromString("222222")); for (int j = 0; j < 4; j++) { for (int i = 0; i < 100; i++) { tr.Set(location.Pack("Range", j, rnd.Next(1000)), Slice.Empty); } } for (int j = 0; j < N; j++) { tr.Set(location.Pack("X", j), Slice.FromInt32(j)); tr.Set(location.Pack("Y", j), Slice.FromInt32(j)); tr.Set(location.Pack("Z", j), Slice.FromInt32(j)); tr.Set(location.Pack("W", j), Slice.FromInt32(j)); } }, this.Cancellation); bool first = true; Action <FdbLoggedTransaction> logHandler = (tr) => { if (first) { Console.WriteLine(tr.Log.GetCommandsReport()); first = false; } Console.WriteLine(tr.Log.GetTimingsReport(true)); }; // create a logged version of the database var logged = new FdbLoggedDatabase(db, false, false, logHandler); for (int k = 0; k < N; k++) { Console.WriteLine("==== " + k + " ==== "); Console.WriteLine(); await logged.ReadWriteAsync(async (tr) => { Assert.That(tr, Is.InstanceOf <FdbLoggedTransaction>()); //tr.SetOption(FdbTransactionOption.CausalReadRisky); long ver = await tr.GetReadVersionAsync().ConfigureAwait(false); await tr.GetAsync(location.Pack("One")).ConfigureAwait(false); await tr.GetAsync(location.Pack("NotFound")).ConfigureAwait(false); tr.Set(location.Pack("Write"), Slice.FromString("abcdef" + k.ToString())); //tr.Annotate("BEFORE"); //await Task.Delay(TimeSpan.FromMilliseconds(10)); //tr.Annotate("AFTER"); //await tr.Snapshot.GetAsync(location.Pack("Snap")).ConfigureAwait(false); tr.Annotate("This is a comment"); //await tr.GetRangeAsync(FdbKeySelector.LastLessOrEqual(location.Pack("A")), FdbKeySelector.FirstGreaterThan(location.Pack("Z"))).ConfigureAwait(false); await Task.WhenAll( tr.GetRange(FdbKeyRange.StartsWith(location.Pack("Range", 0))).ToListAsync(), tr.GetRange(location.Pack("Range", 1, 0), location.Pack("Range", 1, 200)).ToListAsync(), tr.GetRange(location.Pack("Range", 2, 400), location.Pack("Range", 2, 600)).ToListAsync(), tr.GetRange(location.Pack("Range", 3, 800), location.Pack("Range", 3, 1000)).ToListAsync() ).ConfigureAwait(false); await tr.GetAsync(location.Pack("Two")).ConfigureAwait(false); await tr.GetValuesAsync(Enumerable.Range(0, N).Select(x => location.Pack("X", x))).ConfigureAwait(false); for (int i = 0; i < N; i++) { await tr.GetAsync(location.Pack("Z", i)).ConfigureAwait(false); } await Task.WhenAll(Enumerable.Range(0, N / 2).Select(x => tr.GetAsync(location.Pack("Y", x)))).ConfigureAwait(false); await Task.WhenAll(Enumerable.Range(N / 2, N / 2).Select(x => tr.GetAsync(location.Pack("Y", x)))).ConfigureAwait(false); await Task.WhenAll( tr.GetAsync(location.Pack("W", 1)), tr.GetAsync(location.Pack("W", 2)), tr.GetAsync(location.Pack("W", 3)) ).ConfigureAwait(false); tr.Set(location.Pack("Write2"), Slice.FromString("ghijkl" + k.ToString())); tr.Clear(location.Pack("Clear", "0")); tr.ClearRange(location.Pack("Clear", "A"), location.Pack("Clear", "Z")); if (tr.Context.Retries == 0) { // make it fail //throw new FdbException(FdbError.PastVersion, "fake timeout"); } }, this.Cancellation); } } }
public async Task Run(IFdbDatabase db, TextWriter log, CancellationToken ct) { // estimate the number of machines... Console.WriteLine("# Detecting cluster topology..."); var servers = await db.QueryAsync(tr => tr .WithReadAccessToSystemKeys() .GetRange(FdbKeyRange.StartsWith(Fdb.System.ServerList)) .Select(kvp => new { Node = kvp.Value.Substring(8, 16).ToHexaString(), Machine = kvp.Value.Substring(24, 16).ToHexaString(), DataCenter = kvp.Value.Substring(40, 16).ToHexaString() }), ct ); var numNodes = servers.Select(s => s.Node).Distinct().Count(); var numMachines = servers.Select(s => s.Machine).Distinct().Count(); var numDCs = servers.Select(s => s.DataCenter).Distinct().Count(); Console.WriteLine("# > Found " + numNodes + " process(es) on " + numMachines + " machine(s) in " + numDCs + " datacenter(s)"); Console.WriteLine("# Reading list of shards..."); // dump keyServers var ranges = await Fdb.System.GetChunksAsync(db, FdbKey.MinValue, FdbKey.MaxValue, ct); Console.WriteLine("# > Found " + ranges.Count + " shards:"); // take a sample var rnd = new Random(1234); int sz = Math.Max((int)Math.Ceiling(this.Ratio * ranges.Count), 1); if (sz > 500) { sz = 500; //SAFETY } if (sz < 50) { sz = Math.Max(sz, Math.Min(50, ranges.Count)); } var samples = new List <FdbKeyRange>(); for (int i = 0; i < sz; i++) { int p = rnd.Next(ranges.Count); samples.Add(ranges[p]); ranges.RemoveAt(p); } Console.WriteLine("# Sampling " + sz + " out of " + ranges.Count + " shards (" + (100.0 * sz / ranges.Count).ToString("N1") + "%) ..."); Console.WriteLine("{0,9}{1,10}{2,10}{3,10} : K+V size distribution", "Count", "Keys", "Values", "Total"); var rangeOptions = new FdbRangeOptions { Mode = FdbStreamingMode.WantAll }; samples = samples.OrderBy(x => x.Begin).ToList(); long total = 0; int workers = Math.Min(numMachines, 8); var sw = Stopwatch.StartNew(); var tasks = new List <Task>(); while (samples.Count > 0) { while (tasks.Count < workers && samples.Count > 0) { var range = samples[0]; samples.RemoveAt(0); tasks.Add(Task.Run(async() => { var hh = new RobustHistogram(RobustHistogram.TimeScale.Ticks); #region Method 1: get_range everything... using (var tr = db.BeginTransaction(ct)) { long keySize = 0; long valueSize = 0; long count = 0; int iter = 0; var beginSelector = FdbKeySelector.FirstGreaterOrEqual(range.Begin); var endSelector = FdbKeySelector.FirstGreaterOrEqual(range.End); while (true) { FdbRangeChunk data = default(FdbRangeChunk); FdbException error = null; try { data = await tr.Snapshot.GetRangeAsync( beginSelector, endSelector, rangeOptions, iter ).ConfigureAwait(false); } catch (FdbException e) { error = e; } if (error != null) { await tr.OnErrorAsync(error.Code).ConfigureAwait(false); continue; } if (data.Count == 0) { break; } count += data.Count; foreach (var kvp in data.Chunk) { keySize += kvp.Key.Count; valueSize += kvp.Value.Count; hh.Add(TimeSpan.FromTicks(kvp.Key.Count + kvp.Value.Count)); } if (!data.HasMore) { break; } beginSelector = FdbKeySelector.FirstGreaterThan(data.Last.Key); ++iter; } long totalSize = keySize + valueSize; Interlocked.Add(ref total, totalSize); Console.WriteLine("{0,9}{1,10}{2,10}{3,10} : {4}", count.ToString("N0"), FormatSize(keySize), FormatSize(valueSize), FormatSize(totalSize), hh.GetDistribution(begin: 1, end: 10000, fold: 2)); } #endregion #region Method 2: estimate the count using key selectors... //long counter = await Fdb.System.EstimateCountAsync(db, range, ct); //Console.WriteLine("COUNT = " + counter.ToString("N0")); #endregion }, ct)); } var done = await Task.WhenAny(tasks); tasks.Remove(done); } await Task.WhenAll(tasks); sw.Stop(); Console.WriteLine("> Sampled " + FormatSize(total) + " (" + total.ToString("N0") + " bytes) in " + sw.Elapsed.TotalSeconds.ToString("N1") + " sec"); Console.WriteLine("> Estimated total size is " + FormatSize(total * ranges.Count / sz)); }
private async Task RunAsync(IFdbDatabase db, FdbSubspace location, CancellationToken ct, Action done, int N, int K, int W) { if (db == null) { throw new ArgumentNullException("db"); } StringBuilder sb = new StringBuilder(); db = new FdbLoggedDatabase(db, false, false, (log) => { sb.AppendLine(log.Log.GetTimingsReport(true)); //Console.WriteLine(log.Log.GetTimingsReport(true)); }); try { var workerPool = new FdbWorkerPool(location); Console.WriteLine("workerPool at " + location.Key.ToAsciiOrHexaString()); var workerSignal = new AsyncCancelableMutex(ct); var clientSignal = new AsyncCancelableMutex(ct); int taskCounter = 0; int msgSent = 0; int msgReceived = 0; Func <FdbWorkerMessage, CancellationToken, Task> handler = async(msg, _ct) => { Interlocked.Increment(ref msgReceived); //await Task.Delay(10 + Math.Abs(msg.Id.GetHashCode()) % 50); await Task.Delay(10).ConfigureAwait(false); }; Func <int, Task> worker = async(id) => { await workerSignal.Task.ConfigureAwait(false); Console.WriteLine("Worker #" + id + " is starting"); try { await workerPool.RunWorkerAsync(db, handler, ct).ConfigureAwait(false); } finally { Console.WriteLine("Worker #" + id + " has stopped"); } }; Func <int, Task> client = async(id) => { await clientSignal.Task.ConfigureAwait(false); await Task.Delay(10).ConfigureAwait(false); var rnd = new Random(id * 111); for (int i = 0; i < N; i++) { var taskId = Slice.FromString("T" + Interlocked.Increment(ref taskCounter)); var taskBody = Slice.FromString("Message " + (i + 1) + " of " + N + " from client #" + id); await workerPool.ScheduleTaskAsync(db, taskId, taskBody, ct).ConfigureAwait(false); Interlocked.Increment(ref msgSent); //if (i > 0 && i % 10 == 0) Console.WriteLine("@@@ Client#" + id + " pushed " + (i + 1) + " / " + N + " messages"); switch (rnd.Next(5)) { case 0: await Task.Delay(10).ConfigureAwait(false); break; case 1: await Task.Delay(100).ConfigureAwait(false); break; case 2: await Task.Delay(500).ConfigureAwait(false); break; } } Console.WriteLine("@@@ Client#" + id + " has finished!"); }; Func <string, Task> dump = async(label) => { Console.WriteLine("<dump label='" + label + "' key='" + location.Key.ToAsciiOrHexaString() + "'>"); using (var tr = db.BeginTransaction(ct)) { await tr.Snapshot .GetRange(FdbKeyRange.StartsWith(location.Key)) .ForEachAsync((kvp) => { Console.WriteLine(" - " + FdbTuple.Unpack(location.Extract(kvp.Key)) + " = " + kvp.Value.ToAsciiOrHexaString()); }).ConfigureAwait(false); } Console.WriteLine("</dump>"); }; var workers = Enumerable.Range(0, W).Select((i) => worker(i)).ToArray(); var clients = Enumerable.Range(0, K).Select((i) => client(i)).ToArray(); DateTime start = DateTime.Now; DateTime last = start; int lastHandled = -1; using (var timer = new Timer((_) => { var now = DateTime.Now; Console.WriteLine("@@@ T=" + now.Subtract(start) + ", sent: " + msgSent.ToString("N0") + ", recv: " + msgReceived.ToString("N0")); Console.WriteLine("### Workers: " + workerPool.IdleWorkers + " / " + workerPool.ActiveWorkers + " (" + new string('#', workerPool.IdleWorkers) + new string('.', workerPool.ActiveWorkers - workerPool.IdleWorkers) + "), sent: " + workerPool.MessageScheduled.ToString("N0") + ", recv: " + workerPool.MessageReceived.ToString("N0") + ", delta: " + (workerPool.MessageScheduled - workerPool.MessageReceived).ToString("N0") + ", busy: " + workerPool.WorkerBusyTime + " (avg " + workerPool.WorkerAverageBusyDuration.TotalMilliseconds.ToString("N3") + " ms)"); if (now.Subtract(last).TotalSeconds >= 10) { //dump("timer").GetAwaiter().GetResult(); last = now; if (lastHandled == msgReceived) { // STALL ? Console.WriteLine("STALL! "); done(); } lastHandled = msgReceived; } if (msgReceived >= K * N) { dump("complete").GetAwaiter().GetResult(); done(); } }, null, 1000, 1000)) { var sw = Stopwatch.StartNew(); // start the workers workerSignal.Set(async: true); await Task.Delay(500); await dump("workers started"); // start the clients clientSignal.Set(async: true); await Task.WhenAll(clients); Console.WriteLine("Clients completed after " + sw.Elapsed); await Task.WhenAll(workers); Console.WriteLine("Workers completed after " + sw.Elapsed); } } finally { Console.WriteLine("---------------------------------------------------------------------------"); Console.WriteLine("Transaction logs:"); Console.WriteLine(); Console.WriteLine(sb.ToString()); } }
public static async Task Sampling(string[] path, IFdbTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct) { double ratio = 0.1d; bool auto = true; if (extras.Count > 0) { double x = extras.Get <double>(0); if (x > 0 && x <= 1) { ratio = x; } auto = false; } var folder = await TryOpenCurrentDirectoryAsync(path, db, ct); FdbKeyRange span; if (folder is FdbDirectorySubspace) { span = FdbKeyRange.StartsWith((folder as FdbDirectorySubspace).Copy()); log.WriteLine("Reading list of shards for /{0} under {1} ...", String.Join("/", path), FdbKey.Dump(span.Begin)); } else { log.WriteLine("Reading list of shards for the whole cluster ..."); span = FdbKeyRange.All; } // dump keyServers var ranges = await Fdb.System.GetChunksAsync(db, span, ct); log.WriteLine("> Found {0:N0} shard(s)", ranges.Count); // take a sample var samples = new List <FdbKeyRange>(); if (ranges.Count <= 32) { // small enough to scan it all samples.AddRange(ranges); log.WriteLine("Sampling all {0:N0} shards ...", samples.Count); } else { // need to take a random subset var rnd = new Random(); int sz = Math.Max((int)Math.Ceiling(ratio * ranges.Count), 1); if (auto) { if (sz > 100) { sz = 100; //SAFETY } if (sz < 32) { sz = Math.Max(sz, Math.Min(32, ranges.Count)); } } var population = new List <FdbKeyRange>(ranges); for (int i = 0; i < sz; i++) { int p = rnd.Next(population.Count); samples.Add(population[p]); population.RemoveAt(p); } log.WriteLine("Sampling " + samples.Count + " out of " + ranges.Count + " shards (" + (100.0 * samples.Count / ranges.Count).ToString("N1") + "%) ..."); } log.WriteLine(); const string FORMAT_STRING = "{0,9} ║{1,10}{6,6} {2,-29} ║{3,10}{7,7} {4,-37} ║{5,10}"; const string SCALE_KEY = "....--------========########M"; const string SCALE_VAL = "....--------========########@@@@@@@@M"; log.WriteLine(FORMAT_STRING, "Count", "Keys", SCALE_KEY, "Values", SCALE_VAL, "Total", "med.", "med."); var rangeOptions = new FdbRangeOptions { Mode = FdbStreamingMode.WantAll }; samples = samples.OrderBy(x => x.Begin).ToList(); long globalSize = 0; long globalCount = 0; int workers = 8; // Math.Max(4, Environment.ProcessorCount); var sw = Stopwatch.StartNew(); var tasks = new List <Task>(); int n = samples.Count; while (samples.Count > 0) { while (tasks.Count < workers && samples.Count > 0) { var range = samples[0]; samples.RemoveAt(0); tasks.Add(Task.Run(async() => { var kk = new RobustHistogram(RobustHistogram.TimeScale.Ticks); var vv = new RobustHistogram(RobustHistogram.TimeScale.Ticks); #region Method 1: get_range everything... using (var tr = db.BeginTransaction(ct)) { long keySize = 0; long valueSize = 0; long count = 0; int iter = 0; var beginSelector = FdbKeySelector.FirstGreaterOrEqual(range.Begin); var endSelector = FdbKeySelector.FirstGreaterOrEqual(range.End); while (true) { FdbRangeChunk data = default(FdbRangeChunk); FdbException error = null; try { data = await tr.Snapshot.GetRangeAsync( beginSelector, endSelector, rangeOptions, iter ).ConfigureAwait(false); } catch (FdbException e) { error = e; } if (error != null) { await tr.OnErrorAsync(error.Code).ConfigureAwait(false); continue; } if (data.Count == 0) { break; } count += data.Count; foreach (var kvp in data.Chunk) { keySize += kvp.Key.Count; valueSize += kvp.Value.Count; kk.Add(TimeSpan.FromTicks(kvp.Key.Count)); vv.Add(TimeSpan.FromTicks(kvp.Value.Count)); } if (!data.HasMore) { break; } beginSelector = FdbKeySelector.FirstGreaterThan(data.Last.Key); ++iter; } long totalSize = keySize + valueSize; Interlocked.Add(ref globalSize, totalSize); Interlocked.Add(ref globalCount, count); lock (log) { log.WriteLine(FORMAT_STRING, count.ToString("N0"), FormatSize(keySize), kk.GetDistribution(begin: 1, end: 12000, fold: 2), FormatSize(valueSize), vv.GetDistribution(begin: 1, end: 120000, fold: 2), FormatSize(totalSize), FormatSize((int)Math.Ceiling(kk.Median)), FormatSize((int)Math.Ceiling(vv.Median))); } } #endregion #region Method 2: estimate the count using key selectors... //long counter = await Fdb.System.EstimateCountAsync(db, range, ct); //Console.WriteLine("COUNT = " + counter.ToString("N0")); #endregion }, ct)); } var done = await Task.WhenAny(tasks); tasks.Remove(done); } await Task.WhenAll(tasks); sw.Stop(); log.WriteLine(); if (n != ranges.Count) { log.WriteLine("Sampled " + FormatSize(globalSize) + " (" + globalSize.ToString("N0") + " bytes) and " + globalCount.ToString("N0") + " keys in " + sw.Elapsed.TotalSeconds.ToString("N1") + " sec"); log.WriteLine("> Estimated total size is " + FormatSize(globalSize * ranges.Count / n)); } else { log.WriteLine("Found " + FormatSize(globalSize) + " (" + globalSize.ToString("N0") + " bytes) and " + globalCount.ToString("N0") + " keys in " + sw.Elapsed.TotalSeconds.ToString("N1") + " sec"); // compare to the whole cluster ranges = await Fdb.System.GetChunksAsync(db, FdbKey.MinValue, FdbKey.MaxValue, ct); log.WriteLine("> This directory contains ~{0:N2}% of all data", (100.0 * n / ranges.Count)); } log.WriteLine(); }
public static async Task Map(string[] path, IFdbTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct) { // we want to merge the map of shards, with the map of directories from the Directory Layer, and count for each directory how many shards intersect var folder = await TryOpenCurrentDirectoryAsync(path, db, ct); if (folder == null) { log.WriteLine("# Directory not found"); return; } var span = folder.DirectoryLayer.ContentSubspace.ToRange(); // note: this may break in future versions of the DL! Maybe we need a custom API to get a flat list of all directories in a DL that span a specific range ? var shards = await Fdb.System.GetChunksAsync(db, span, ct); int totalShards = shards.Count; log.WriteLine("Found {0} shard(s) in partition /{1}", totalShards, folder.DirectoryLayer.FullName); log.WriteLine("Listing all directories..."); var map = new Dictionary <string, int>(StringComparer.Ordinal); Action <string[], int> account = (p, c) => { for (int i = 1; i <= p.Length; i++) { var s = "/" + String.Join("/", p, 0, i); int x; map[s] = map.TryGetValue(s, out x) ? (x + c) : c; } }; var work = new Stack <IFdbDirectory>(); work.Push(folder); var dirs = new List <IFdbDirectory>(); int n = 0; while (work.Count > 0) { var cur = work.Pop(); // skip sub partitions var names = await cur.ListAsync(db, ct); foreach (var name in names) { var sub = await cur.TryOpenAsync(db, name, ct); if (sub != null) { var p = sub.FullName; if (sub is FdbDirectoryPartition) { log.WriteLine("\r! Skipping partition {0} ", sub.Name); n = 0; continue; } log.Write("\r/{0}{1}", p, p.Length > n ? String.Empty : new string(' ', n - p.Length)); n = p.Length; work.Push(sub); dirs.Add(sub); } } } log.Write("\r" + new string(' ', n + 2)); log.WriteLine("\r> Found {0} sub-directories", dirs.Count); log.WriteLine(); log.WriteLine("Estimating size of each directory..."); int foundShards = 0; n = 0; int max = 0; IFdbDirectory bigBad = null; foreach (var dir in dirs) { log.Write("\r> {0}{1}", dir.Name, dir.Name.Length > n ? String.Empty : new string(' ', n - dir.Name.Length)); n = dir.Name.Length; var p = dir.Path.ToArray(); var key = ((FdbSubspace)dir).Key; // verify that the subspace has at least one key inside var bounds = await db.ReadAsync(async (tr) => { var kvs = await Task.WhenAll( tr.GetRange(FdbKeyRange.StartsWith(key)).FirstOrDefaultAsync(), tr.GetRange(FdbKeyRange.StartsWith(key)).LastOrDefaultAsync() ); return(new { Min = kvs[0].Key, Max = kvs[1].Key }); }, ct); if (bounds.Min.HasValue) { // folder is not empty shards = await Fdb.System.GetChunksAsync(db, FdbKeyRange.StartsWith(key), ct); //TODO: we still need to check if the first and last shard really intersect the subspace // we need to check if the shards actually contain data //Console.WriteLine("/{0} under {1} with {2} shard(s)", string.Join("/", p), FdbKey.Dump(key), shards.Count); foundShards += shards.Count; account(p, shards.Count); if (shards.Count > max) { max = shards.Count; bigBad = dir; } } else { account(p, 0); } } log.Write("\r" + new string(' ', n + 2)); log.WriteLine("\rFound a total of {0} shard(s) in {1} folder(s)", foundShards, dirs.Count); log.WriteLine(); log.WriteLine("Shards %Total Path"); foreach (var kvp in map.OrderBy(x => x.Key)) { log.WriteLine("{0,6} {1,-20} {2}", kvp.Value, RobustHistogram.FormatHistoBar((double)kvp.Value / foundShards, 20), kvp.Key); } log.WriteLine(); if (bigBad != null) { log.WriteLine("Biggest folder is /{0} with {1} shards ({2:N1}% total, {3:N1}% subtree)", bigBad.FullName, max, 100.0 * max / totalShards, 100.0 * max / foundShards); log.WriteLine(); } }
/// <summary>Find the DCs, machines and processes in the cluster</summary> public static async Task Topology(string[] path, IFdbTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct) { var coords = await Fdb.System.GetCoordinatorsAsync(db, ct); log.WriteLine("[Cluster] {0}", coords.Id); var servers = await db.QueryAsync(tr => tr .WithReadAccessToSystemKeys() .GetRange(FdbKeyRange.StartsWith(Fdb.System.ServerList)) .Select(kvp => new { // Offsets Size Type Name Description // 0 2 Word Version? 0100 (1.0 ?) // 2 4 DWord ??? 0x00 0x20 0xA2 0x00 // 6 2 Word FDBMagic 0xDB 0x0F "FDB" // 8 16 Guid NodeId Unique Process ID // 24 16 Guid Machine "machine_id" field in foundationdb.conf (ends with 8x0 if manually specified) // 40 16 Guid DataCenter "datacenter_id" field in foundationdb.conf (ends with 8x0 if manually specified) // 56 4 ??? ?? 4 x 0 // 60 12 x24 ARRAY[] ?? array of 12x the same 24-byte struct defined below // ...0 4 DWord IPAddress 01 00 00 7F => 127.0.0.1 // ...4 4 DWord Port 94 11 00 00 -> 4500 // ...8 4 DWord ?? randomish, changes every reboot // ..12 4 DWord ?? randomish, changes every reboot // ..16 4 DWord Size? small L-E integer, usually between 0x20 and 0x40... // ..20 4 DWord ?? randmoish, changes every reboot ProcessId = kvp.Value.Substring(8, 16).ToHexaString(), MachineId = kvp.Value.Substring(24, 16).ToHexaString(), DataCenterId = kvp.Value.Substring(40, 16).ToHexaString(), Parts = Enumerable.Range(0, 12).Select(i => { int p = 60 + 24 * i; return(new { Address = new IPAddress(kvp.Value.Substring(p, 4).GetBytes().Reverse().ToArray()), Port = kvp.Value.Substring(p + 4, 4).ToInt32(), Unknown1 = kvp.Value.Substring(p + 8, 4).ToInt32(), Unknown2 = kvp.Value.Substring(p + 12, 4).ToInt32(), Unknown3 = kvp.Value.Substring(p + 16, 4).ToInt32(), Unknown4 = kvp.Value.Substring(p + 20, 4).ToInt32(), }); }).ToList(), Raw = kvp.Value, }), ct ); var numNodes = servers.Select(s => s.ProcessId).Distinct().Count(); var numMachines = servers.Select(s => s.MachineId).Distinct().Count(); var numDCs = servers.Select(s => s.DataCenterId).Distinct().Count(); var dcs = servers.GroupBy(x => x.DataCenterId).ToArray(); for (int dcIndex = 0; dcIndex < dcs.Length; dcIndex++) { var dc = dcs[dcIndex]; bool lastDc = dcIndex == dcs.Length - 1; string dcId = dc.Key.EndsWith("0000000000000000") ? dc.Key.Substring(0, 16) : dc.Key; log.WriteLine((lastDc ? "`- " : "|- ") + "[DataCenter] {0} (#{1})", dcId, dcIndex); var machines = dc.GroupBy(x => x.MachineId).ToArray(); string dcPrefix = lastDc ? " " : "| "; for (int machineIndex = 0; machineIndex < machines.Length; machineIndex++) { var machine = machines[machineIndex]; var lastMachine = machineIndex == machines.Length - 1; string machineId = machine.Key.EndsWith("0000000000000000") ? machine.Key.Substring(0, 16) : machine.Key; log.WriteLine(dcPrefix + (lastMachine ? "`- " : "|- ") + "[Machine] {0}, {1}", machine.First().Parts[0].Address, machineId); var procs = machine.ToArray(); string machinePrefix = dcPrefix + (lastMachine ? " " : "| "); for (int procIndex = 0; procIndex < procs.Length; procIndex++) { var proc = procs[procIndex]; bool lastProc = procIndex == procs.Length - 1; log.WriteLine(machinePrefix + (lastProc ? "`- " : "|- ") + "[Process] {0}:{1}, {2}", proc.Parts[0].Address, proc.Parts[0].Port, proc.ProcessId); //foreach (var part in proc.Parts) //{ // log.WriteLine(machinePrefix + "| -> {0}, {1}, {2:X8}, {3:X8}, {4}, {5:X8}", part.Address, part.Port, part.Unknown1, part.Unknown2, part.Unknown3, part.Unknown4); //} } } } log.WriteLine(); log.WriteLine("Found {0} process(es) on {1} machine(s) in {2} datacenter(s)", numNodes, numMachines, numDCs); log.WriteLine(); }