示例#1
0
 public FdbDynamicSubspacePartition([NotNull] IFdbDynamicSubspace subspace, [NotNull] IDynamicKeyEncoder encoder)
 {
     Contract.NotNull(subspace, nameof(subspace));
     Contract.NotNull(encoder, nameof(encoder));
     this.Subspace = subspace;
     this.Encoder  = encoder;
 }
		/// <summary>Create an allocator operating under a specific location</summary>
		/// <param name="subspace"></param>
		public FdbHighContentionAllocator(IFdbDynamicSubspace subspace)
		{
			if (subspace == null) throw new ArgumentException("subspace");

			this.Subspace = subspace;
			this.Counters = subspace.Partition.ByKey(COUNTERS);
			this.Recent = subspace.Partition.ByKey(RECENT);
		}
        /// <summary>Create an allocator operating under a specific location</summary>
        /// <param name="subspace"></param>
        public FdbHighContentionAllocator(IFdbDynamicSubspace subspace)
        {
            if (subspace == null)
            {
                throw new ArgumentException("subspace");
            }

            this.Subspace = subspace;
            this.Counters = subspace.Partition.ByKey(COUNTERS);
            this.Recent   = subspace.Partition.ByKey(RECENT);
        }
 /// <summary>Change the current global namespace.</summary>
 /// <remarks>Do NOT call this, unless you know exactly what you are doing !</remarks>
 internal void ChangeRoot(IFdbSubspace subspace, IFdbDirectory directory, bool readOnly)
 {
     //REVIEW: rename to "ChangeRootSubspace" ?
     subspace = subspace ?? FdbSubspace.Empty;
     lock (this)            //TODO: don't use this for locking
     {
         m_readOnly        = readOnly;
         m_globalSpace     = FdbSubspace.CopyDynamic(subspace, TypeSystem.Tuples);
         m_globalSpaceCopy = FdbSubspace.CopyDynamic(subspace, TypeSystem.Tuples);                 // keep another copy
         m_directory       = directory == null ? null : new FdbDatabasePartition(this, directory);
     }
 }
 public FdbDynamicSubspacePartition([NotNull] IFdbDynamicSubspace subspace, [NotNull] IDynamicKeyEncoder encoder)
 {
     if (subspace == null)
     {
         throw new ArgumentNullException("subspace");
     }
     if (encoder == null)
     {
         throw new ArgumentNullException("encoder");
     }
     this.Subspace = subspace;
     this.Encoder  = encoder;
 }
        /// <summary>
        /// Setup the initial state of the database
        /// </summary>
        public async Task Init(IFdbDatabase db, CancellationToken ct)
        {
            // open the folder where we will store everything
            this.Subspace = await db.Directory.CreateOrOpenAsync(new [] { "Benchmarks", "LeakTest" }, cancellationToken : ct);

            // clear all previous values
            await db.ClearRangeAsync(this.Subspace, ct);

            // insert all the classes
            await db.WriteAsync((tr) =>
            {
                tr.Set(this.Subspace.Key + FdbKey.MinValue, Slice.FromString("BEGIN"));
                tr.Set(this.Subspace.Key + FdbKey.MaxValue, Slice.FromString("END"));
            }, ct);
        }
示例#7
0
        private async Task PushQueueAsync(IFdbTransaction tr, IFdbDynamicSubspace queue, Slice taskId)
        {
            //TODO: use a high contention algo ?
            // - must support Push and Pop
            // - an empty queue must correspond to an empty subspace

            // get the current size of the queue
            var range   = queue.Keys.ToRange();
            var lastKey = await tr.Snapshot.GetKeyAsync(KeySelector.LastLessThan(range.End)).ConfigureAwait(false);

            int count = lastKey < range.Begin ? 0 : queue.Keys.DecodeFirst <int>(lastKey) + 1;

            // set the value
            tr.Set(queue.Keys.Encode(count, GetRandomId()), taskId);
        }
		/// <summary>
		/// Setup the initial state of the database
		/// </summary>
		public async Task Init(IFdbDatabase db, CancellationToken ct)
		{
			// open the folder where we will store everything
			this.Subspace = await db.Directory.CreateOrOpenAsync(new [] { "Benchmarks", "LeakTest" }, cancellationToken: ct);

			// clear all previous values
			await db.ClearRangeAsync(this.Subspace, ct);

			// insert all the classes
			await db.WriteAsync((tr) =>
			{
				tr.Set(this.Subspace.Key + FdbKey.MinValue, Slice.FromString("BEGIN"));
				tr.Set(this.Subspace.Key + FdbKey.MaxValue, Slice.FromString("END"));
			}, ct);
		}
示例#9
0
        /// <summary>
        /// Setup the initial state of the database
        /// </summary>
        public async Task Init(IFdbDatabase db, CancellationToken ct)
        {
            // open the folder where we will store everything
            this.Subspace = await db.Directory.CreateOrOpenAsync(new [] { "Tutorials", "ClassScheduling" }, cancellationToken : ct);

            // clear all previous values
            await db.ClearRangeAsync(this.Subspace, ct);

            // insert all the classes
            await db.WriteAsync((tr) =>
            {
                foreach (var c in this.ClassNames)
                {
                    tr.Set(ClassKey(c), Slice.FromAscii("100"));
                }
            }, ct);
        }
		/// <summary>
		/// Setup the initial state of the database
		/// </summary>
		public async Task Init(IFdbDatabase db, CancellationToken ct)
		{
			// open the folder where we will store everything
			this.Subspace = await db.Directory.CreateOrOpenAsync(new [] { "Tutorials", "ClassScheduling" }, cancellationToken: ct);

			// clear all previous values
			await db.ClearRangeAsync(this.Subspace, ct);
			// insert all the classes
			await db.WriteAsync((tr) =>
			{
				foreach (var c in this.ClassNames)
				{
					tr.Set(ClassKey(c), Slice.FromAscii("100"));
				}
			}, ct);


		}
示例#11
0
        private async Task <KeyValuePair <Slice, Slice> > FindRandomItem(IFdbTransaction tr, IFdbDynamicSubspace ring)
        {
            var range = ring.Keys.ToRange();

            // start from a random position around the ring
            Slice key = ring.Keys.Encode(GetRandomId());

            // We want to find the next item in the clockwise direction. If we reach the end of the ring, we "wrap around" by starting again from the start
            // => So we do find_next(key <= x < MAX) and if that does not produce any result, we do a find_next(MIN <= x < key)

            // When the ring only contains a few items (or is empty), there is more than 50% change that we wont find anything in the first read.
            // To reduce the latency for this case, we will issue both range reads at the same time, and discard the second one if the first returned something.
            // This should reduce the latency in half when the ring is empty, or when it contains only items before the random key.

            var candidate = await tr.GetRange(key, range.End).FirstOrDefaultAsync();

            if (!candidate.Key.IsPresent)
            {
                candidate = await tr.GetRange(range.Begin, key).FirstOrDefaultAsync();
            }

            return(candidate);
        }
        private async Task <long> GetNextIndexAsync([NotNull] IFdbReadOnlyTransaction tr, IFdbDynamicSubspace subspace)
        {
            var range = subspace.Keys.ToRange();

            var lastKey = await tr.GetKeyAsync(FdbKeySelector.LastLessThan(range.End)).ConfigureAwait(false);

            if (lastKey < range.Begin)
            {
                return(0);
            }

            return(subspace.Keys.DecodeFirst <long>(lastKey) + 1);
        }
			public Node(IFdbDynamicSubspace subspace, IFdbTuple path, IFdbTuple targetPath, Slice layer)
			{
				this.Subspace = subspace;
				this.Path = path;
				this.TargetPath = targetPath;
				this.Layer = layer;
			}
        private async Task RunAsync(IFdbDatabase db, IFdbDynamicSubspace location, CancellationToken ct, Action done, int N, int K, int W)
        {
            if (db == null)
            {
                throw new ArgumentNullException(nameof(db));
            }

            StringBuilder sb = new StringBuilder();

            db = new FdbLoggedDatabase(db, false, false, (log) =>
            {
                sb.AppendLine(log.Log.GetTimingsReport(true));
                //Console.WriteLine(log.Log.GetTimingsReport(true));
            });
            try
            {
                var workerPool = new FdbWorkerPool(location);
                Console.WriteLine("workerPool at " + location.Key.ToAsciiOrHexaString());

                var workerSignal = new AsyncCancelableMutex(ct);
                var clientSignal = new AsyncCancelableMutex(ct);

                int taskCounter = 0;

                int msgSent     = 0;
                int msgReceived = 0;

                Func <FdbWorkerMessage, CancellationToken, Task> handler = async(msg, _ct) =>
                {
                    Interlocked.Increment(ref msgReceived);

                    //await Task.Delay(10 + Math.Abs(msg.Id.GetHashCode()) % 50);
                    await Task.Delay(10).ConfigureAwait(false);
                };

                Func <int, Task> worker = async(id) =>
                {
                    await workerSignal.Task.ConfigureAwait(false);

                    Console.WriteLine("Worker #" + id + " is starting");
                    try
                    {
                        await workerPool.RunWorkerAsync(db, handler, ct).ConfigureAwait(false);
                    }
                    finally
                    {
                        Console.WriteLine("Worker #" + id + " has stopped");
                    }
                };

                Func <int, Task> client = async(id) =>
                {
                    await clientSignal.Task.ConfigureAwait(false);

                    await Task.Delay(10).ConfigureAwait(false);

                    var rnd = new Random(id * 111);
                    for (int i = 0; i < N; i++)
                    {
                        var taskId   = Slice.FromString("T" + Interlocked.Increment(ref taskCounter));
                        var taskBody = Slice.FromString("Message " + (i + 1) + " of " + N + " from client #" + id);

                        await workerPool.ScheduleTaskAsync(db, taskId, taskBody, ct).ConfigureAwait(false);

                        Interlocked.Increment(ref msgSent);

                        //if (i > 0 && i % 10 == 0) Console.WriteLine("@@@ Client#" + id + " pushed " + (i + 1) + " / " + N + " messages");

                        switch (rnd.Next(5))
                        {
                        case 0: await Task.Delay(10).ConfigureAwait(false); break;

                        case 1: await Task.Delay(100).ConfigureAwait(false); break;

                        case 2: await Task.Delay(500).ConfigureAwait(false); break;
                        }
                    }
                    Console.WriteLine("@@@ Client#" + id + " has finished!");
                };

                Func <string, Task> dump = async(label) =>
                {
                    Console.WriteLine("<dump label='" + label + "' key='" + location.Key.ToAsciiOrHexaString() + "'>");
                    using (var tr = db.BeginTransaction(ct))
                    {
                        await tr.Snapshot
                        .GetRange(KeyRange.StartsWith(location.Key))
                        .ForEachAsync((kvp) =>
                        {
                            Console.WriteLine(" - " + location.Keys.Unpack(kvp.Key) + " = " + kvp.Value.ToAsciiOrHexaString());
                        }).ConfigureAwait(false);
                    }
                    Console.WriteLine("</dump>");
                };

                var workers = Enumerable.Range(0, W).Select((i) => worker(i)).ToArray();
                var clients = Enumerable.Range(0, K).Select((i) => client(i)).ToArray();

                DateTime start       = DateTime.Now;
                DateTime last        = start;
                int      lastHandled = -1;
                using (var timer = new Timer((_) =>
                {
                    var now = DateTime.Now;
                    Console.WriteLine("@@@ T=" + now.Subtract(start) + ", sent: " + msgSent.ToString("N0") + ", recv: " + msgReceived.ToString("N0"));
                    Console.WriteLine("### Workers: " + workerPool.IdleWorkers + " / " + workerPool.ActiveWorkers + " (" + new string('#', workerPool.IdleWorkers) + new string('.', workerPool.ActiveWorkers - workerPool.IdleWorkers) + "), sent: " + workerPool.MessageScheduled.ToString("N0") + ", recv: " + workerPool.MessageReceived.ToString("N0") + ", delta: " + (workerPool.MessageScheduled - workerPool.MessageReceived).ToString("N0") + ", busy: " + workerPool.WorkerBusyTime + " (avg " + workerPool.WorkerAverageBusyDuration.TotalMilliseconds.ToString("N3") + " ms)");

                    if (now.Subtract(last).TotalSeconds >= 10)
                    {
                        //dump("timer").GetAwaiter().GetResult();
                        last = now;
                        if (lastHandled == msgReceived)
                        {                         // STALL ?
                            Console.WriteLine("STALL! ");
                            done();
                        }
                        lastHandled = msgReceived;
                    }

                    if (msgReceived >= K * N)
                    {
                        dump("complete").GetAwaiter().GetResult();
                        done();
                    }
                }, null, 1000, 1000))
                {
                    var sw = Stopwatch.StartNew();

                    // start the workers
                    workerSignal.Set(async: true);
                    await Task.Delay(500);

                    await dump("workers started");

                    // start the clients
                    clientSignal.Set(async: true);

                    await Task.WhenAll(clients);

                    Console.WriteLine("Clients completed after " + sw.Elapsed);

                    await Task.WhenAll(workers);

                    Console.WriteLine("Workers completed after " + sw.Elapsed);
                }
            }
            finally
            {
                Console.WriteLine("---------------------------------------------------------------------------");
                Console.WriteLine("Transaction logs:");
                Console.WriteLine();

                Console.WriteLine(sb.ToString());
            }
        }
		public static FdbDirectoryLayer Create(IFdbDynamicSubspace nodeSubspace, IFdbDynamicSubspace contentSubspace, IEnumerable<string> path = null)
		{
			if (nodeSubspace == null) throw new ArgumentNullException("nodeSubspace");
			if (contentSubspace == null) throw new ArgumentNullException("contentSubspace");

			var location = path != null ? ParsePath(path) : FdbTuple.Empty;
			//TODO: check that nodeSubspace != contentSubspace?
			return new FdbDirectoryLayer(nodeSubspace, contentSubspace, location);
		}
		private async Task PushQueueAsync(IFdbTransaction tr, IFdbDynamicSubspace queue, Slice taskId)
		{
			//TODO: use a high contention algo ?
			// - must support Push and Pop
			// - an empty queue must correspond to an empty subspace

			// get the current size of the queue
			var range = queue.Keys.ToRange();
			var lastKey = await tr.Snapshot.GetKeyAsync(FdbKeySelector.LastLessThan(range.End)).ConfigureAwait(false);
			int count = lastKey < range.Begin ? 0 : queue.Keys.DecodeFirst<int>(lastKey) + 1;

			// set the value
			tr.Set(queue.Keys.Encode(count, GetRandomId()), taskId);
		}
		private async Task<KeyValuePair<Slice, Slice>> FindRandomItem(IFdbTransaction tr, IFdbDynamicSubspace ring)
		{
			var range = ring.Keys.ToRange();

			// start from a random position around the ring
			Slice key = ring.Keys.Encode(GetRandomId());

			// We want to find the next item in the clockwise direction. If we reach the end of the ring, we "wrap around" by starting again from the start
			// => So we do find_next(key <= x < MAX) and if that does not produce any result, we do a find_next(MIN <= x < key)

			// When the ring only contains a few items (or is empty), there is more than 50% change that we wont find anything in the first read.
			// To reduce the latency for this case, we will issue both range reads at the same time, and discard the second one if the first returned something.
			// This should reduce the latency in half when the ring is empty, or when it contains only items before the random key.

			var candidate = await tr.GetRange(key, range.End).FirstOrDefaultAsync();

			if (!candidate.Key.IsPresent)
			{
				candidate = await tr.GetRange(range.Begin, key).FirstOrDefaultAsync();
			}

			return candidate;
		}
		private async Task RunAsync(IFdbDatabase db, IFdbDynamicSubspace location, CancellationToken ct, Action done, int N, int K, int W)
		{
			if (db == null) throw new ArgumentNullException("db");

			StringBuilder sb = new StringBuilder();

			db = new FdbLoggedDatabase(db, false, false, (log) =>
			{
				sb.AppendLine(log.Log.GetTimingsReport(true));
				//Console.WriteLine(log.Log.GetTimingsReport(true));
			});
			try
			{

				var workerPool = new FdbWorkerPool(location);
				Console.WriteLine("workerPool at " + location.Key.ToAsciiOrHexaString());

				var workerSignal = new AsyncCancelableMutex(ct);
				var clientSignal = new AsyncCancelableMutex(ct);

				int taskCounter = 0;

				int msgSent = 0;
				int msgReceived = 0;

				Func<FdbWorkerMessage, CancellationToken, Task> handler = async (msg, _ct) =>
				{
					Interlocked.Increment(ref msgReceived);

					//await Task.Delay(10 + Math.Abs(msg.Id.GetHashCode()) % 50);
					await Task.Delay(10).ConfigureAwait(false);

				};

				Func<int, Task> worker = async (id) =>
				{
					await workerSignal.Task.ConfigureAwait(false);
					Console.WriteLine("Worker #" + id + " is starting");
					try
					{
						await workerPool.RunWorkerAsync(db, handler, ct).ConfigureAwait(false);
					}
					finally
					{
						Console.WriteLine("Worker #" + id + " has stopped");
					}
				};

				Func<int, Task> client = async (id) =>
				{
					await clientSignal.Task.ConfigureAwait(false);
					await Task.Delay(10).ConfigureAwait(false);

					var rnd = new Random(id * 111);
					for (int i = 0; i < N; i++)
					{
						var taskId = Slice.FromString("T" + Interlocked.Increment(ref taskCounter));
						var taskBody = Slice.FromString("Message " + (i + 1) + " of " + N + " from client #" + id);

						await workerPool.ScheduleTaskAsync(db, taskId, taskBody, ct).ConfigureAwait(false);
						Interlocked.Increment(ref msgSent);

						//if (i > 0 && i % 10 == 0) Console.WriteLine("@@@ Client#" + id + " pushed " + (i + 1) + " / " + N + " messages");

						switch (rnd.Next(5))
						{
							case 0: await Task.Delay(10).ConfigureAwait(false); break;
							case 1: await Task.Delay(100).ConfigureAwait(false); break;
							case 2: await Task.Delay(500).ConfigureAwait(false); break;
						}
					}
					Console.WriteLine("@@@ Client#" + id + " has finished!");
				};

				Func<string, Task> dump = async (label) =>
				{
					Console.WriteLine("<dump label='" + label + "' key='" + location.Key.ToAsciiOrHexaString() + "'>");
					using (var tr = db.BeginTransaction(ct))
					{
						await tr.Snapshot
							.GetRange(FdbKeyRange.StartsWith(location.Key))
							.ForEachAsync((kvp) =>
							{
								Console.WriteLine(" - " + location.Keys.Unpack(kvp.Key) + " = " + kvp.Value.ToAsciiOrHexaString());
							}).ConfigureAwait(false);
					}
					Console.WriteLine("</dump>");
				};

				var workers = Enumerable.Range(0, W).Select((i) => worker(i)).ToArray();
				var clients = Enumerable.Range(0, K).Select((i) => client(i)).ToArray();

				DateTime start = DateTime.Now;
				DateTime last = start;
				int lastHandled = -1;
				using (var timer = new Timer((_) =>
				{
					var now = DateTime.Now;
					Console.WriteLine("@@@ T=" + now.Subtract(start) + ", sent: " + msgSent.ToString("N0") + ", recv: " + msgReceived.ToString("N0"));
					Console.WriteLine("### Workers: " + workerPool.IdleWorkers + " / " + workerPool.ActiveWorkers + " (" + new string('#', workerPool.IdleWorkers) + new string('.', workerPool.ActiveWorkers - workerPool.IdleWorkers) + "), sent: " + workerPool.MessageScheduled.ToString("N0") + ", recv: " + workerPool.MessageReceived.ToString("N0") + ", delta: " + (workerPool.MessageScheduled - workerPool.MessageReceived).ToString("N0") + ", busy: " + workerPool.WorkerBusyTime + " (avg " + workerPool.WorkerAverageBusyDuration.TotalMilliseconds.ToString("N3") + " ms)");

					if (now.Subtract(last).TotalSeconds >= 10)
					{
						//dump("timer").GetAwaiter().GetResult();
						last = now;
						if (lastHandled == msgReceived)
						{ // STALL ?
							Console.WriteLine("STALL! ");
							done();
						}
						lastHandled = msgReceived;
					}

					if (msgReceived >= K * N)
					{
						dump("complete").GetAwaiter().GetResult();
						done();
					}


				}, null, 1000, 1000))
				{

					var sw = Stopwatch.StartNew();

					// start the workers
					workerSignal.Set(async: true);
					await Task.Delay(500);

					await dump("workers started");

					// start the clients
					clientSignal.Set(async: true);

					await Task.WhenAll(clients);
					Console.WriteLine("Clients completed after " + sw.Elapsed);

					await Task.WhenAll(workers);
					Console.WriteLine("Workers completed after " + sw.Elapsed);
				}
			}
			finally
			{
				Console.WriteLine("---------------------------------------------------------------------------");
				Console.WriteLine("Transaction logs:");
				Console.WriteLine();

				Console.WriteLine(sb.ToString());
			}
		}
		/// <summary>
		/// Creates a new instance that will manages directories in FoudnationDB.
		/// </summary>
		/// <param name="nodeSubspace">Subspace where all the node metadata will be stored ('\xFE' by default)</param>
		/// <param name="contentSubspace">Subspace where all automatically allocated directories will be stored (empty by default)</param>
		/// <param name="location">Location of the root of all the directories managed by this Directory Layer. Ususally empty for the root partition of the database.</param>
		internal FdbDirectoryLayer(IFdbDynamicSubspace nodeSubspace, IFdbDynamicSubspace contentSubspace, IFdbTuple location)
		{
			Contract.Requires(nodeSubspace != null && contentSubspace != null);

			// If specified, new automatically allocated prefixes will all fall within content_subspace
			this.ContentSubspace = contentSubspace;
			this.NodeSubspace = nodeSubspace;

			// The root node is the one whose contents are the node subspace
			this.RootNode = nodeSubspace.Partition.ByKey(nodeSubspace.Key);
			this.Allocator = new FdbHighContentionAllocator(this.RootNode.Partition.ByKey(HcaKey));
			if (location == null || location.Count == 0)
			{
				this.Location = FdbTuple.Empty;
				this.Path = new string[0];
			}
			else
			{
				this.Location = location;
				this.Path = location.ToArray<string>();
			}
		}