public async Task Test_Allocator() { //FoundationDB.Client.Utils.Logging.SetLevel(System.Diagnostics.SourceLevels.Verbose); using (var db = await OpenTestDatabaseAsync()) { var location = db.Partition.ByKey(Slice.FromString("hca")); await db.ClearRangeAsync(location, this.Cancellation); #if ENABLE_LOGGING var list = new List<FdbTransactionLog>(); var logged = new FdbLoggedDatabase(db, false, false, (tr) => { list.Add(tr.Log); }); #else var logged = db; #endif var hpa = new FdbHighContentionAllocator(location); long id; var ids = new HashSet<long>(); // allocate a single new id using (var tr = logged.BeginTransaction(this.Cancellation)) { id = await hpa.AllocateAsync(tr); await tr.CommitAsync(); } ids.Add(id); await DumpSubspace(db, location); // allocate a batch of new ids for (int i = 0; i < 100; i++) { using (var tr = logged.BeginTransaction(this.Cancellation)) { id = await hpa.AllocateAsync(tr); await tr.CommitAsync(); } if (ids.Contains(id)) { await DumpSubspace(db, location); Assert.Fail("Duplicate key allocated: {0} (#{1})", id, i); } ids.Add(id); } await DumpSubspace(db, location); #if ENABLE_LOGGING foreach(var log in list) { Console.WriteLine(log.GetTimingsReport(true)); } #endif } }
public async Task Test_Can_Change_Layer_Of_Existing_Directory() { using (var db = await OpenTestDatabaseAsync()) { var location = db.Partition.ByKey("DL"); await db.ClearRangeAsync(location, this.Cancellation); var directory = FdbDirectoryLayer.Create(location); #if ENABLE_LOGGING var list = new List<FdbTransactionLog>(); var logged = new FdbLoggedDatabase(db, false, false, (tr) => { list.Add(tr.Log); }); #else var logged = db; #endif var folder = await directory.CreateAsync(logged, "Test", layer: Slice.FromString("foo"), cancellationToken: this.Cancellation); #if DEBUG await DumpSubspace(db, location); #endif Assert.That(folder, Is.Not.Null); Assert.That(folder.Layer.ToUnicode(), Is.EqualTo("foo")); // change the layer to 'bar' var folder2 = await folder.ChangeLayerAsync(logged, Slice.FromString("bar"), this.Cancellation); #if DEBUG await DumpSubspace(db, location); #endif Assert.That(folder2, Is.Not.Null); Assert.That(folder2.Layer.ToUnicode(), Is.EqualTo("bar")); Assert.That(folder2.FullName, Is.EqualTo("Test")); Assert.That(folder2.Path, Is.EqualTo(FdbTuple.Create("Test"))); Assert.That(folder2.Key, Is.EqualTo(folder.Key)); // opening the directory with the new layer should succeed var folder3 = await directory.OpenAsync(logged, "Test", layer: Slice.FromString("bar"), cancellationToken: this.Cancellation); Assert.That(folder3, Is.Not.Null); // opening the directory with the old layer should fail Assert.Throws<InvalidOperationException>(async () => await directory.OpenAsync(logged, "Test", layer: Slice.FromString("foo"), cancellationToken: this.Cancellation)); #if ENABLE_LOGGING foreach (var log in list) { Console.WriteLine(log.GetTimingsReport(true)); } #endif } }
public async Task Test_Remove_Folder() { using (var db = await OpenTestDatabaseAsync()) { var location = db.Partition.ByKey("DL"); await db.ClearRangeAsync(location, this.Cancellation); var directory = FdbDirectoryLayer.Create(location); #if ENABLE_LOGGING var list = new List<FdbTransactionLog>(); var logged = new FdbLoggedDatabase(db, false, false, (tr) => { list.Add(tr.Log); }); #else var logged = db; #endif // RemoveAsync string[] path = new[] { "CrashTestDummy" }; await directory.CreateAsync(logged, path, this.Cancellation); #if DEBUG await DumpSubspace(db, location); #endif // removing an existing folder should succeeed await directory.RemoveAsync(logged, path, this.Cancellation); #if DEBUG await DumpSubspace(db, location); #endif //TODO: call ExistsAsync(...) once it is implemented! // Removing it a second time should fail Assert.Throws<InvalidOperationException>(async () => await directory.RemoveAsync(logged, path, this.Cancellation), "Removing a non-existent directory should fail"); // TryRemoveAsync await directory.CreateAsync(logged, path, this.Cancellation); // attempting to remove a folder should return true bool res = await directory.TryRemoveAsync(logged, path, this.Cancellation); Assert.That(res, Is.True); // further attempts should return false res = await directory.TryRemoveAsync(logged, path, this.Cancellation); Assert.That(res, Is.False); // Corner Cases // removing the root folder is not allowed (too dangerous) Assert.Throws<InvalidOperationException>(async () => await directory.RemoveAsync(logged, new string[0], this.Cancellation), "Attempting to remove the root directory should fail"); #if ENABLE_LOGGING foreach (var log in list) { Console.WriteLine(log.GetTimingsReport(true)); } #endif } }
public async Task Test_List_SubFolders() { using (var db = await OpenTestDatabaseAsync()) { // we will put everything under a custom namespace var location = db.Partition.ByKey("DL"); await db.ClearRangeAsync(location, this.Cancellation); var directory = FdbDirectoryLayer.Create(location); #if ENABLE_LOGGING var list = new List<FdbTransactionLog>(); var logged = new FdbLoggedDatabase(db, false, false, (tr) => { list.Add(tr.Log); }); #else var logged = db; #endif // linear subtree "/Foo/Bar/Baz" await directory.CreateOrOpenAsync(logged, new[] { "Foo", "Bar", "Baz" }, this.Cancellation); // flat subtree "/numbers/0" to "/numbers/9" for (int i = 0; i < 10; i++) await directory.CreateOrOpenAsync(logged, new[] { "numbers", i.ToString() }, this.Cancellation); #if DEBUG await DumpSubspace(db, location); #endif var subdirs = await directory.ListAsync(logged, new[] { "Foo" }, this.Cancellation); Assert.That(subdirs, Is.Not.Null); Assert.That(subdirs.Count, Is.EqualTo(1)); Assert.That(subdirs[0], Is.EqualTo("Bar")); subdirs = await directory.ListAsync(logged, new[] { "Foo", "Bar" }, this.Cancellation); Assert.That(subdirs, Is.Not.Null); Assert.That(subdirs.Count, Is.EqualTo(1)); Assert.That(subdirs[0], Is.EqualTo("Baz")); subdirs = await directory.ListAsync(logged, new[] { "Foo", "Bar", "Baz" }, this.Cancellation); Assert.That(subdirs, Is.Not.Null); Assert.That(subdirs.Count, Is.EqualTo(0)); subdirs = await directory.ListAsync(logged, new[] { "numbers" }, this.Cancellation); Assert.That(subdirs, Is.Not.Null); Assert.That(subdirs.Count, Is.EqualTo(10)); Assert.That(subdirs, Is.EquivalentTo(Enumerable.Range(0, 10).Select(x => x.ToString()).ToList())); #if ENABLE_LOGGING foreach (var log in list) { Console.WriteLine(log.GetTimingsReport(true)); } #endif } }
public async Task Test_Move_Folder() { // Create a folder ("foo", "bar", "baz") and ensure that all the parent folder are also creating and linked properly using (var db = await OpenTestDatabaseAsync()) { // we will put everything under a custom namespace var location = db.Partition.ByKey("DL"); await db.ClearRangeAsync(location, this.Cancellation); #if ENABLE_LOGGING var list = new List<FdbTransactionLog>(); var logged = new FdbLoggedDatabase(db, false, false, (tr) => { list.Add(tr.Log); }); #else var logged = db; #endif // put the nodes under (..,"DL",\xFE,) and the content under (..,"DL",) var directory = FdbDirectoryLayer.Create(location); // create a folder at ('Foo',) var original = await directory.CreateOrOpenAsync(logged, "Foo", this.Cancellation); #if DEBUG await DumpSubspace(db, location); #endif Assert.That(original, Is.Not.Null); Assert.That(original.FullName, Is.EqualTo("Foo")); Assert.That(original.Path, Is.EqualTo(new[] { "Foo" })); // rename/move it as ('Bar',) var renamed = await original.MoveToAsync(logged, new[] { "Bar" }, this.Cancellation); #if DEBUG await DumpSubspace(db, location); #endif Assert.That(renamed, Is.Not.Null); Assert.That(renamed.FullName, Is.EqualTo("Bar")); Assert.That(renamed.Path, Is.EqualTo(new[] { "Bar" })); Assert.That(renamed.Key, Is.EqualTo(original.Key)); // opening the old path should fail Assert.Throws<InvalidOperationException>(async () => await directory.OpenAsync(logged, "Foo", this.Cancellation)); // opening the new path should succeed var folder = await directory.OpenAsync(logged, "Bar", this.Cancellation); Assert.That(folder, Is.Not.Null); Assert.That(folder.FullName, Is.EqualTo(renamed.FullName)); Assert.That(folder.Path, Is.EqualTo(renamed.Path)); Assert.That(folder.Key, Is.EqualTo(renamed.Key)); // moving the folder under itself should fail Assert.Throws<InvalidOperationException>(async () => await folder.MoveToAsync(logged, new[] { "Bar", "Baz" }, this.Cancellation)); #if ENABLE_LOGGING foreach (var log in list) { Console.WriteLine(log.GetTimingsReport(true)); } #endif } }
public async Task Test_CreateOrOpen_SubFolder() { // Create a folder ("foo", "bar", "baz") and ensure that all the parent folder are also creating and linked properly using (var db = await OpenTestDatabaseAsync()) { // we will put everything under a custom namespace var location = db.Partition.ByKey("DL"); await db.ClearRangeAsync(location, this.Cancellation); #if ENABLE_LOGGING var list = new List<FdbTransactionLog>(); var logged = new FdbLoggedDatabase(db, false, false, (tr) => { list.Add(tr.Log); }); #else var logged = db; #endif // put the nodes under (..,"DL",\xFE,) and the content under (..,"DL",) var directory = FdbDirectoryLayer.Create(location); FdbDirectorySubspace folder; using (var tr = logged.BeginTransaction(this.Cancellation)) { folder = await directory.CreateOrOpenAsync(tr, new [] { "Foo", "Bar", "Baz" }); await tr.CommitAsync(); } #if DEBUG await DumpSubspace(db, location); #endif Assert.That(folder, Is.Not.Null); Assert.That(folder.FullName, Is.EqualTo("Foo/Bar/Baz")); Assert.That(folder.Path, Is.EqualTo(new[] { "Foo", "Bar", "Baz" })); // all the parent folders should also now exist var foo = await directory.OpenAsync(logged, new[] { "Foo" }, this.Cancellation); var bar = await directory.OpenAsync(logged, new[] { "Foo", "Bar" }, this.Cancellation); Assert.That(foo, Is.Not.Null); Assert.That(bar, Is.Not.Null); #if ENABLE_LOGGING foreach (var log in list) { Console.WriteLine(log.GetTimingsReport(true)); } #endif } }
public async Task Test_CreateOrOpen_With_Layer() { using (var db = await OpenTestDatabaseAsync()) { // we will put everything under a custom namespace var location = db.Partition.ByKey("DL"); await db.ClearRangeAsync(location, this.Cancellation); #if ENABLE_LOGGING var list = new List<FdbTransactionLog>(); var logged = new FdbLoggedDatabase(db, false, false, (tr) => { list.Add(tr.Log); }); #else var logged = db; #endif // put the nodes under (..,"DL",\xFE,) and the content under (..,"DL",) var directory = FdbDirectoryLayer.Create(location); Assert.That(directory.ContentSubspace, Is.Not.Null); Assert.That(directory.ContentSubspace, Is.EqualTo(location)); Assert.That(directory.NodeSubspace, Is.Not.Null); Assert.That(directory.NodeSubspace.Key, Is.EqualTo(location.Key + Slice.FromByte(254))); // first call should create a new subspace (with a random prefix) var foo = await directory.CreateOrOpenAsync(logged, new[] { "Foo" }, Slice.FromString("AcmeLayer"), this.Cancellation); #if DEBUG await DumpSubspace(db, location); #endif Assert.That(foo, Is.Not.Null); Assert.That(foo.FullName, Is.EqualTo("Foo")); Assert.That(foo.Path, Is.EqualTo(new[] { "Foo" })); Assert.That(foo.Layer.ToUnicode(), Is.EqualTo("AcmeLayer")); Assert.That(foo.DirectoryLayer, Is.SameAs(directory)); // second call should return the same subspace var foo2 = await directory.OpenAsync(logged, new[] { "Foo" }, Slice.FromString("AcmeLayer"), this.Cancellation); Assert.That(foo2, Is.Not.Null); Assert.That(foo2.FullName, Is.EqualTo("Foo")); Assert.That(foo2.Path, Is.EqualTo(new[] { "Foo" })); Assert.That(foo2.Layer.ToUnicode(), Is.EqualTo("AcmeLayer")); Assert.That(foo2.DirectoryLayer, Is.SameAs(directory)); Assert.That(foo2.Key, Is.EqualTo(foo.Key), "Second call to CreateOrOpen should return the same subspace"); // opening it with wrong layer id should fail Assert.Throws<InvalidOperationException>(async () => await directory.OpenAsync(logged, new[] { "Foo" }, Slice.FromString("OtherLayer"), this.Cancellation), "Opening with invalid layer id should fail"); // opening without specifying a layer should disable the layer check var foo3 = await directory.OpenAsync(logged, "Foo", layer: Slice.Nil, cancellationToken: this.Cancellation); Assert.That(foo3, Is.Not.Null); Assert.That(foo3.Layer.ToUnicode(), Is.EqualTo("AcmeLayer")); // CheckLayer with the correct value should pass Assert.DoesNotThrow(() => foo3.CheckLayer(Slice.FromString("AcmeLayer")), "CheckLayer should not throw if the layer id is correct"); // CheckLayer with the incorrect value should fail Assert.Throws<InvalidOperationException>(() => foo3.CheckLayer(Slice.FromString("OtherLayer")), "CheckLayer should throw if the layer id is not correct"); // CheckLayer with empty string should do nothing foo3.CheckLayer(Slice.Empty); foo3.CheckLayer(Slice.Nil); #if ENABLE_LOGGING foreach (var log in list) { Console.WriteLine(log.GetTimingsReport(true)); } #endif } }
public async Task Test_CreateOrOpen_Simple() { using (var db = await OpenTestDatabaseAsync()) { // we will put everything under a custom namespace var location = db.Partition.ByKey("DL"); await db.ClearRangeAsync(location, this.Cancellation); #if ENABLE_LOGGING var list = new List<FdbTransactionLog>(); var logged = new FdbLoggedDatabase(db, false, false, (tr) => { list.Add(tr.Log); }); #else var logged = db; #endif // put the nodes under (..,"DL",\xFE,) and the content under (..,"DL",) var directory = FdbDirectoryLayer.Create(location); Assert.That(directory.ContentSubspace, Is.Not.Null); Assert.That(directory.ContentSubspace.Key, Is.EqualTo(location.Key)); Assert.That(directory.NodeSubspace, Is.Not.Null); Assert.That(directory.NodeSubspace.Key, Is.EqualTo(location.Key + Slice.FromByte(254))); // first call should create a new subspace (with a random prefix) FdbDirectorySubspace foo; using (var tr = logged.BeginTransaction(this.Cancellation)) { foo = await directory.CreateOrOpenAsync(tr, new[] { "Foo" }); await tr.CommitAsync(); } #if DEBUG await DumpSubspace(db, location); #endif Assert.That(foo, Is.Not.Null); Assert.That(foo.FullName, Is.EqualTo("Foo")); Assert.That(foo.Path, Is.EqualTo(new[] { "Foo" })); Assert.That(foo.Layer, Is.EqualTo(Slice.Empty)); Assert.That(foo.DirectoryLayer, Is.SameAs(directory)); // second call should return the same subspace FdbDirectorySubspace foo2; foo2 = await directory.OpenAsync(logged, new[] { "Foo" }, this.Cancellation); #if DEBUG await DumpSubspace(db, location); #endif Assert.That(foo2, Is.Not.Null); Assert.That(foo2.FullName, Is.EqualTo("Foo")); Assert.That(foo2.Path, Is.EqualTo(new[] { "Foo" })); Assert.That(foo2.Layer, Is.EqualTo(Slice.Empty)); Assert.That(foo2.DirectoryLayer, Is.SameAs(directory)); Assert.That(foo2.Key, Is.EqualTo(foo.Key), "Second call to CreateOrOpen should return the same subspace"); #if ENABLE_LOGGING foreach (var log in list) { Console.WriteLine(log.GetTimingsReport(true)); } #endif } }
public async Task Test_Log_Queue() { int NUM = 100; using (var db = await OpenTestPartitionAsync()) { var location = await GetCleanDirectory(db, "queue"); #if ENABLE_LOGGING var list = new List<FdbTransactionLog>(NUM); var logged = new FdbLoggedDatabase(db, false, false, (tr) => { lock (list) { list.Add(tr.Log); } }); #else var logged = db; #endif await RunMultiClientTest(logged, location, false, "simple queue", 4, NUM, this.Cancellation); #if ENABLE_LOGGING foreach (var log in list) { Console.WriteLine(log.GetTimingsReport(true)); } list.Clear(); #endif Console.WriteLine("------------------------------------------------"); await RunMultiClientTest(logged, location, true, "high contention queue", 4, NUM, this.Cancellation); #if ENABLE_LOGGING foreach (var log in list) { Console.WriteLine(log.GetTimingsReport(true)); } list.Clear(); #endif Console.WriteLine("------------------------------------------------"); } }
private async Task RunAsync(IFdbDatabase db, IFdbDynamicSubspace location, CancellationToken ct, Action done, int N, int K, int W) { if (db == null) throw new ArgumentNullException("db"); StringBuilder sb = new StringBuilder(); db = new FdbLoggedDatabase(db, false, false, (log) => { sb.AppendLine(log.Log.GetTimingsReport(true)); //Console.WriteLine(log.Log.GetTimingsReport(true)); }); try { var workerPool = new FdbWorkerPool(location); Console.WriteLine("workerPool at " + location.Key.ToAsciiOrHexaString()); var workerSignal = new AsyncCancelableMutex(ct); var clientSignal = new AsyncCancelableMutex(ct); int taskCounter = 0; int msgSent = 0; int msgReceived = 0; Func<FdbWorkerMessage, CancellationToken, Task> handler = async (msg, _ct) => { Interlocked.Increment(ref msgReceived); //await Task.Delay(10 + Math.Abs(msg.Id.GetHashCode()) % 50); await Task.Delay(10).ConfigureAwait(false); }; Func<int, Task> worker = async (id) => { await workerSignal.Task.ConfigureAwait(false); Console.WriteLine("Worker #" + id + " is starting"); try { await workerPool.RunWorkerAsync(db, handler, ct).ConfigureAwait(false); } finally { Console.WriteLine("Worker #" + id + " has stopped"); } }; Func<int, Task> client = async (id) => { await clientSignal.Task.ConfigureAwait(false); await Task.Delay(10).ConfigureAwait(false); var rnd = new Random(id * 111); for (int i = 0; i < N; i++) { var taskId = Slice.FromString("T" + Interlocked.Increment(ref taskCounter)); var taskBody = Slice.FromString("Message " + (i + 1) + " of " + N + " from client #" + id); await workerPool.ScheduleTaskAsync(db, taskId, taskBody, ct).ConfigureAwait(false); Interlocked.Increment(ref msgSent); //if (i > 0 && i % 10 == 0) Console.WriteLine("@@@ Client#" + id + " pushed " + (i + 1) + " / " + N + " messages"); switch (rnd.Next(5)) { case 0: await Task.Delay(10).ConfigureAwait(false); break; case 1: await Task.Delay(100).ConfigureAwait(false); break; case 2: await Task.Delay(500).ConfigureAwait(false); break; } } Console.WriteLine("@@@ Client#" + id + " has finished!"); }; Func<string, Task> dump = async (label) => { Console.WriteLine("<dump label='" + label + "' key='" + location.Key.ToAsciiOrHexaString() + "'>"); using (var tr = db.BeginTransaction(ct)) { await tr.Snapshot .GetRange(FdbKeyRange.StartsWith(location.Key)) .ForEachAsync((kvp) => { Console.WriteLine(" - " + location.Keys.Unpack(kvp.Key) + " = " + kvp.Value.ToAsciiOrHexaString()); }).ConfigureAwait(false); } Console.WriteLine("</dump>"); }; var workers = Enumerable.Range(0, W).Select((i) => worker(i)).ToArray(); var clients = Enumerable.Range(0, K).Select((i) => client(i)).ToArray(); DateTime start = DateTime.Now; DateTime last = start; int lastHandled = -1; using (var timer = new Timer((_) => { var now = DateTime.Now; Console.WriteLine("@@@ T=" + now.Subtract(start) + ", sent: " + msgSent.ToString("N0") + ", recv: " + msgReceived.ToString("N0")); Console.WriteLine("### Workers: " + workerPool.IdleWorkers + " / " + workerPool.ActiveWorkers + " (" + new string('#', workerPool.IdleWorkers) + new string('.', workerPool.ActiveWorkers - workerPool.IdleWorkers) + "), sent: " + workerPool.MessageScheduled.ToString("N0") + ", recv: " + workerPool.MessageReceived.ToString("N0") + ", delta: " + (workerPool.MessageScheduled - workerPool.MessageReceived).ToString("N0") + ", busy: " + workerPool.WorkerBusyTime + " (avg " + workerPool.WorkerAverageBusyDuration.TotalMilliseconds.ToString("N3") + " ms)"); if (now.Subtract(last).TotalSeconds >= 10) { //dump("timer").GetAwaiter().GetResult(); last = now; if (lastHandled == msgReceived) { // STALL ? Console.WriteLine("STALL! "); done(); } lastHandled = msgReceived; } if (msgReceived >= K * N) { dump("complete").GetAwaiter().GetResult(); done(); } }, null, 1000, 1000)) { var sw = Stopwatch.StartNew(); // start the workers workerSignal.Set(async: true); await Task.Delay(500); await dump("workers started"); // start the clients clientSignal.Set(async: true); await Task.WhenAll(clients); Console.WriteLine("Clients completed after " + sw.Elapsed); await Task.WhenAll(workers); Console.WriteLine("Workers completed after " + sw.Elapsed); } } finally { Console.WriteLine("---------------------------------------------------------------------------"); Console.WriteLine("Transaction logs:"); Console.WriteLine(); Console.WriteLine(sb.ToString()); } }