public FdbDynamicSubspacePartition([NotNull] IFdbDynamicSubspace subspace, [NotNull] IDynamicKeyEncoder encoder) { Contract.NotNull(subspace, nameof(subspace)); Contract.NotNull(encoder, nameof(encoder)); this.Subspace = subspace; this.Encoder = encoder; }
/// <summary>Create an allocator operating under a specific location</summary> /// <param name="subspace"></param> public FdbHighContentionAllocator(IFdbDynamicSubspace subspace) { if (subspace == null) throw new ArgumentException("subspace"); this.Subspace = subspace; this.Counters = subspace.Partition.ByKey(COUNTERS); this.Recent = subspace.Partition.ByKey(RECENT); }
/// <summary>Create an allocator operating under a specific location</summary> /// <param name="subspace"></param> public FdbHighContentionAllocator(IFdbDynamicSubspace subspace) { if (subspace == null) { throw new ArgumentException("subspace"); } this.Subspace = subspace; this.Counters = subspace.Partition.ByKey(COUNTERS); this.Recent = subspace.Partition.ByKey(RECENT); }
/// <summary>Change the current global namespace.</summary> /// <remarks>Do NOT call this, unless you know exactly what you are doing !</remarks> internal void ChangeRoot(IFdbSubspace subspace, IFdbDirectory directory, bool readOnly) { //REVIEW: rename to "ChangeRootSubspace" ? subspace = subspace ?? FdbSubspace.Empty; lock (this) //TODO: don't use this for locking { m_readOnly = readOnly; m_globalSpace = FdbSubspace.CopyDynamic(subspace, TypeSystem.Tuples); m_globalSpaceCopy = FdbSubspace.CopyDynamic(subspace, TypeSystem.Tuples); // keep another copy m_directory = directory == null ? null : new FdbDatabasePartition(this, directory); } }
public FdbDynamicSubspacePartition([NotNull] IFdbDynamicSubspace subspace, [NotNull] IDynamicKeyEncoder encoder) { if (subspace == null) { throw new ArgumentNullException("subspace"); } if (encoder == null) { throw new ArgumentNullException("encoder"); } this.Subspace = subspace; this.Encoder = encoder; }
/// <summary> /// Setup the initial state of the database /// </summary> public async Task Init(IFdbDatabase db, CancellationToken ct) { // open the folder where we will store everything this.Subspace = await db.Directory.CreateOrOpenAsync(new [] { "Benchmarks", "LeakTest" }, cancellationToken : ct); // clear all previous values await db.ClearRangeAsync(this.Subspace, ct); // insert all the classes await db.WriteAsync((tr) => { tr.Set(this.Subspace.Key + FdbKey.MinValue, Slice.FromString("BEGIN")); tr.Set(this.Subspace.Key + FdbKey.MaxValue, Slice.FromString("END")); }, ct); }
private async Task PushQueueAsync(IFdbTransaction tr, IFdbDynamicSubspace queue, Slice taskId) { //TODO: use a high contention algo ? // - must support Push and Pop // - an empty queue must correspond to an empty subspace // get the current size of the queue var range = queue.Keys.ToRange(); var lastKey = await tr.Snapshot.GetKeyAsync(KeySelector.LastLessThan(range.End)).ConfigureAwait(false); int count = lastKey < range.Begin ? 0 : queue.Keys.DecodeFirst <int>(lastKey) + 1; // set the value tr.Set(queue.Keys.Encode(count, GetRandomId()), taskId); }
/// <summary> /// Setup the initial state of the database /// </summary> public async Task Init(IFdbDatabase db, CancellationToken ct) { // open the folder where we will store everything this.Subspace = await db.Directory.CreateOrOpenAsync(new [] { "Benchmarks", "LeakTest" }, cancellationToken: ct); // clear all previous values await db.ClearRangeAsync(this.Subspace, ct); // insert all the classes await db.WriteAsync((tr) => { tr.Set(this.Subspace.Key + FdbKey.MinValue, Slice.FromString("BEGIN")); tr.Set(this.Subspace.Key + FdbKey.MaxValue, Slice.FromString("END")); }, ct); }
/// <summary> /// Setup the initial state of the database /// </summary> public async Task Init(IFdbDatabase db, CancellationToken ct) { // open the folder where we will store everything this.Subspace = await db.Directory.CreateOrOpenAsync(new [] { "Tutorials", "ClassScheduling" }, cancellationToken : ct); // clear all previous values await db.ClearRangeAsync(this.Subspace, ct); // insert all the classes await db.WriteAsync((tr) => { foreach (var c in this.ClassNames) { tr.Set(ClassKey(c), Slice.FromAscii("100")); } }, ct); }
/// <summary> /// Setup the initial state of the database /// </summary> public async Task Init(IFdbDatabase db, CancellationToken ct) { // open the folder where we will store everything this.Subspace = await db.Directory.CreateOrOpenAsync(new [] { "Tutorials", "ClassScheduling" }, cancellationToken: ct); // clear all previous values await db.ClearRangeAsync(this.Subspace, ct); // insert all the classes await db.WriteAsync((tr) => { foreach (var c in this.ClassNames) { tr.Set(ClassKey(c), Slice.FromAscii("100")); } }, ct); }
private async Task <KeyValuePair <Slice, Slice> > FindRandomItem(IFdbTransaction tr, IFdbDynamicSubspace ring) { var range = ring.Keys.ToRange(); // start from a random position around the ring Slice key = ring.Keys.Encode(GetRandomId()); // We want to find the next item in the clockwise direction. If we reach the end of the ring, we "wrap around" by starting again from the start // => So we do find_next(key <= x < MAX) and if that does not produce any result, we do a find_next(MIN <= x < key) // When the ring only contains a few items (or is empty), there is more than 50% change that we wont find anything in the first read. // To reduce the latency for this case, we will issue both range reads at the same time, and discard the second one if the first returned something. // This should reduce the latency in half when the ring is empty, or when it contains only items before the random key. var candidate = await tr.GetRange(key, range.End).FirstOrDefaultAsync(); if (!candidate.Key.IsPresent) { candidate = await tr.GetRange(range.Begin, key).FirstOrDefaultAsync(); } return(candidate); }
private async Task <long> GetNextIndexAsync([NotNull] IFdbReadOnlyTransaction tr, IFdbDynamicSubspace subspace) { var range = subspace.Keys.ToRange(); var lastKey = await tr.GetKeyAsync(FdbKeySelector.LastLessThan(range.End)).ConfigureAwait(false); if (lastKey < range.Begin) { return(0); } return(subspace.Keys.DecodeFirst <long>(lastKey) + 1); }
public Node(IFdbDynamicSubspace subspace, IFdbTuple path, IFdbTuple targetPath, Slice layer) { this.Subspace = subspace; this.Path = path; this.TargetPath = targetPath; this.Layer = layer; }
private async Task RunAsync(IFdbDatabase db, IFdbDynamicSubspace location, CancellationToken ct, Action done, int N, int K, int W) { if (db == null) { throw new ArgumentNullException(nameof(db)); } StringBuilder sb = new StringBuilder(); db = new FdbLoggedDatabase(db, false, false, (log) => { sb.AppendLine(log.Log.GetTimingsReport(true)); //Console.WriteLine(log.Log.GetTimingsReport(true)); }); try { var workerPool = new FdbWorkerPool(location); Console.WriteLine("workerPool at " + location.Key.ToAsciiOrHexaString()); var workerSignal = new AsyncCancelableMutex(ct); var clientSignal = new AsyncCancelableMutex(ct); int taskCounter = 0; int msgSent = 0; int msgReceived = 0; Func <FdbWorkerMessage, CancellationToken, Task> handler = async(msg, _ct) => { Interlocked.Increment(ref msgReceived); //await Task.Delay(10 + Math.Abs(msg.Id.GetHashCode()) % 50); await Task.Delay(10).ConfigureAwait(false); }; Func <int, Task> worker = async(id) => { await workerSignal.Task.ConfigureAwait(false); Console.WriteLine("Worker #" + id + " is starting"); try { await workerPool.RunWorkerAsync(db, handler, ct).ConfigureAwait(false); } finally { Console.WriteLine("Worker #" + id + " has stopped"); } }; Func <int, Task> client = async(id) => { await clientSignal.Task.ConfigureAwait(false); await Task.Delay(10).ConfigureAwait(false); var rnd = new Random(id * 111); for (int i = 0; i < N; i++) { var taskId = Slice.FromString("T" + Interlocked.Increment(ref taskCounter)); var taskBody = Slice.FromString("Message " + (i + 1) + " of " + N + " from client #" + id); await workerPool.ScheduleTaskAsync(db, taskId, taskBody, ct).ConfigureAwait(false); Interlocked.Increment(ref msgSent); //if (i > 0 && i % 10 == 0) Console.WriteLine("@@@ Client#" + id + " pushed " + (i + 1) + " / " + N + " messages"); switch (rnd.Next(5)) { case 0: await Task.Delay(10).ConfigureAwait(false); break; case 1: await Task.Delay(100).ConfigureAwait(false); break; case 2: await Task.Delay(500).ConfigureAwait(false); break; } } Console.WriteLine("@@@ Client#" + id + " has finished!"); }; Func <string, Task> dump = async(label) => { Console.WriteLine("<dump label='" + label + "' key='" + location.Key.ToAsciiOrHexaString() + "'>"); using (var tr = db.BeginTransaction(ct)) { await tr.Snapshot .GetRange(KeyRange.StartsWith(location.Key)) .ForEachAsync((kvp) => { Console.WriteLine(" - " + location.Keys.Unpack(kvp.Key) + " = " + kvp.Value.ToAsciiOrHexaString()); }).ConfigureAwait(false); } Console.WriteLine("</dump>"); }; var workers = Enumerable.Range(0, W).Select((i) => worker(i)).ToArray(); var clients = Enumerable.Range(0, K).Select((i) => client(i)).ToArray(); DateTime start = DateTime.Now; DateTime last = start; int lastHandled = -1; using (var timer = new Timer((_) => { var now = DateTime.Now; Console.WriteLine("@@@ T=" + now.Subtract(start) + ", sent: " + msgSent.ToString("N0") + ", recv: " + msgReceived.ToString("N0")); Console.WriteLine("### Workers: " + workerPool.IdleWorkers + " / " + workerPool.ActiveWorkers + " (" + new string('#', workerPool.IdleWorkers) + new string('.', workerPool.ActiveWorkers - workerPool.IdleWorkers) + "), sent: " + workerPool.MessageScheduled.ToString("N0") + ", recv: " + workerPool.MessageReceived.ToString("N0") + ", delta: " + (workerPool.MessageScheduled - workerPool.MessageReceived).ToString("N0") + ", busy: " + workerPool.WorkerBusyTime + " (avg " + workerPool.WorkerAverageBusyDuration.TotalMilliseconds.ToString("N3") + " ms)"); if (now.Subtract(last).TotalSeconds >= 10) { //dump("timer").GetAwaiter().GetResult(); last = now; if (lastHandled == msgReceived) { // STALL ? Console.WriteLine("STALL! "); done(); } lastHandled = msgReceived; } if (msgReceived >= K * N) { dump("complete").GetAwaiter().GetResult(); done(); } }, null, 1000, 1000)) { var sw = Stopwatch.StartNew(); // start the workers workerSignal.Set(async: true); await Task.Delay(500); await dump("workers started"); // start the clients clientSignal.Set(async: true); await Task.WhenAll(clients); Console.WriteLine("Clients completed after " + sw.Elapsed); await Task.WhenAll(workers); Console.WriteLine("Workers completed after " + sw.Elapsed); } } finally { Console.WriteLine("---------------------------------------------------------------------------"); Console.WriteLine("Transaction logs:"); Console.WriteLine(); Console.WriteLine(sb.ToString()); } }
public static FdbDirectoryLayer Create(IFdbDynamicSubspace nodeSubspace, IFdbDynamicSubspace contentSubspace, IEnumerable<string> path = null) { if (nodeSubspace == null) throw new ArgumentNullException("nodeSubspace"); if (contentSubspace == null) throw new ArgumentNullException("contentSubspace"); var location = path != null ? ParsePath(path) : FdbTuple.Empty; //TODO: check that nodeSubspace != contentSubspace? return new FdbDirectoryLayer(nodeSubspace, contentSubspace, location); }
private async Task PushQueueAsync(IFdbTransaction tr, IFdbDynamicSubspace queue, Slice taskId) { //TODO: use a high contention algo ? // - must support Push and Pop // - an empty queue must correspond to an empty subspace // get the current size of the queue var range = queue.Keys.ToRange(); var lastKey = await tr.Snapshot.GetKeyAsync(FdbKeySelector.LastLessThan(range.End)).ConfigureAwait(false); int count = lastKey < range.Begin ? 0 : queue.Keys.DecodeFirst<int>(lastKey) + 1; // set the value tr.Set(queue.Keys.Encode(count, GetRandomId()), taskId); }
private async Task<KeyValuePair<Slice, Slice>> FindRandomItem(IFdbTransaction tr, IFdbDynamicSubspace ring) { var range = ring.Keys.ToRange(); // start from a random position around the ring Slice key = ring.Keys.Encode(GetRandomId()); // We want to find the next item in the clockwise direction. If we reach the end of the ring, we "wrap around" by starting again from the start // => So we do find_next(key <= x < MAX) and if that does not produce any result, we do a find_next(MIN <= x < key) // When the ring only contains a few items (or is empty), there is more than 50% change that we wont find anything in the first read. // To reduce the latency for this case, we will issue both range reads at the same time, and discard the second one if the first returned something. // This should reduce the latency in half when the ring is empty, or when it contains only items before the random key. var candidate = await tr.GetRange(key, range.End).FirstOrDefaultAsync(); if (!candidate.Key.IsPresent) { candidate = await tr.GetRange(range.Begin, key).FirstOrDefaultAsync(); } return candidate; }
private async Task RunAsync(IFdbDatabase db, IFdbDynamicSubspace location, CancellationToken ct, Action done, int N, int K, int W) { if (db == null) throw new ArgumentNullException("db"); StringBuilder sb = new StringBuilder(); db = new FdbLoggedDatabase(db, false, false, (log) => { sb.AppendLine(log.Log.GetTimingsReport(true)); //Console.WriteLine(log.Log.GetTimingsReport(true)); }); try { var workerPool = new FdbWorkerPool(location); Console.WriteLine("workerPool at " + location.Key.ToAsciiOrHexaString()); var workerSignal = new AsyncCancelableMutex(ct); var clientSignal = new AsyncCancelableMutex(ct); int taskCounter = 0; int msgSent = 0; int msgReceived = 0; Func<FdbWorkerMessage, CancellationToken, Task> handler = async (msg, _ct) => { Interlocked.Increment(ref msgReceived); //await Task.Delay(10 + Math.Abs(msg.Id.GetHashCode()) % 50); await Task.Delay(10).ConfigureAwait(false); }; Func<int, Task> worker = async (id) => { await workerSignal.Task.ConfigureAwait(false); Console.WriteLine("Worker #" + id + " is starting"); try { await workerPool.RunWorkerAsync(db, handler, ct).ConfigureAwait(false); } finally { Console.WriteLine("Worker #" + id + " has stopped"); } }; Func<int, Task> client = async (id) => { await clientSignal.Task.ConfigureAwait(false); await Task.Delay(10).ConfigureAwait(false); var rnd = new Random(id * 111); for (int i = 0; i < N; i++) { var taskId = Slice.FromString("T" + Interlocked.Increment(ref taskCounter)); var taskBody = Slice.FromString("Message " + (i + 1) + " of " + N + " from client #" + id); await workerPool.ScheduleTaskAsync(db, taskId, taskBody, ct).ConfigureAwait(false); Interlocked.Increment(ref msgSent); //if (i > 0 && i % 10 == 0) Console.WriteLine("@@@ Client#" + id + " pushed " + (i + 1) + " / " + N + " messages"); switch (rnd.Next(5)) { case 0: await Task.Delay(10).ConfigureAwait(false); break; case 1: await Task.Delay(100).ConfigureAwait(false); break; case 2: await Task.Delay(500).ConfigureAwait(false); break; } } Console.WriteLine("@@@ Client#" + id + " has finished!"); }; Func<string, Task> dump = async (label) => { Console.WriteLine("<dump label='" + label + "' key='" + location.Key.ToAsciiOrHexaString() + "'>"); using (var tr = db.BeginTransaction(ct)) { await tr.Snapshot .GetRange(FdbKeyRange.StartsWith(location.Key)) .ForEachAsync((kvp) => { Console.WriteLine(" - " + location.Keys.Unpack(kvp.Key) + " = " + kvp.Value.ToAsciiOrHexaString()); }).ConfigureAwait(false); } Console.WriteLine("</dump>"); }; var workers = Enumerable.Range(0, W).Select((i) => worker(i)).ToArray(); var clients = Enumerable.Range(0, K).Select((i) => client(i)).ToArray(); DateTime start = DateTime.Now; DateTime last = start; int lastHandled = -1; using (var timer = new Timer((_) => { var now = DateTime.Now; Console.WriteLine("@@@ T=" + now.Subtract(start) + ", sent: " + msgSent.ToString("N0") + ", recv: " + msgReceived.ToString("N0")); Console.WriteLine("### Workers: " + workerPool.IdleWorkers + " / " + workerPool.ActiveWorkers + " (" + new string('#', workerPool.IdleWorkers) + new string('.', workerPool.ActiveWorkers - workerPool.IdleWorkers) + "), sent: " + workerPool.MessageScheduled.ToString("N0") + ", recv: " + workerPool.MessageReceived.ToString("N0") + ", delta: " + (workerPool.MessageScheduled - workerPool.MessageReceived).ToString("N0") + ", busy: " + workerPool.WorkerBusyTime + " (avg " + workerPool.WorkerAverageBusyDuration.TotalMilliseconds.ToString("N3") + " ms)"); if (now.Subtract(last).TotalSeconds >= 10) { //dump("timer").GetAwaiter().GetResult(); last = now; if (lastHandled == msgReceived) { // STALL ? Console.WriteLine("STALL! "); done(); } lastHandled = msgReceived; } if (msgReceived >= K * N) { dump("complete").GetAwaiter().GetResult(); done(); } }, null, 1000, 1000)) { var sw = Stopwatch.StartNew(); // start the workers workerSignal.Set(async: true); await Task.Delay(500); await dump("workers started"); // start the clients clientSignal.Set(async: true); await Task.WhenAll(clients); Console.WriteLine("Clients completed after " + sw.Elapsed); await Task.WhenAll(workers); Console.WriteLine("Workers completed after " + sw.Elapsed); } } finally { Console.WriteLine("---------------------------------------------------------------------------"); Console.WriteLine("Transaction logs:"); Console.WriteLine(); Console.WriteLine(sb.ToString()); } }
/// <summary> /// Creates a new instance that will manages directories in FoudnationDB. /// </summary> /// <param name="nodeSubspace">Subspace where all the node metadata will be stored ('\xFE' by default)</param> /// <param name="contentSubspace">Subspace where all automatically allocated directories will be stored (empty by default)</param> /// <param name="location">Location of the root of all the directories managed by this Directory Layer. Ususally empty for the root partition of the database.</param> internal FdbDirectoryLayer(IFdbDynamicSubspace nodeSubspace, IFdbDynamicSubspace contentSubspace, IFdbTuple location) { Contract.Requires(nodeSubspace != null && contentSubspace != null); // If specified, new automatically allocated prefixes will all fall within content_subspace this.ContentSubspace = contentSubspace; this.NodeSubspace = nodeSubspace; // The root node is the one whose contents are the node subspace this.RootNode = nodeSubspace.Partition.ByKey(nodeSubspace.Key); this.Allocator = new FdbHighContentionAllocator(this.RootNode.Partition.ByKey(HcaKey)); if (location == null || location.Count == 0) { this.Location = FdbTuple.Empty; this.Path = new string[0]; } else { this.Location = location; this.Path = location.ToArray<string>(); } }