public async Task Test_Log_Queue() { int NUM = 100; using (var db = await OpenTestPartitionAsync()) { var location = await GetCleanDirectory(db, "queue"); #if ENABLE_LOGGING var list = new List <FdbTransactionLog>(NUM); var logged = new FdbLoggedDatabase(db, false, false, (tr) => { lock (list) { list.Add(tr.Log); } }); #else var logged = db; #endif await RunMultiClientTest(logged, location, false, "simple queue", 4, NUM, this.Cancellation); #if ENABLE_LOGGING foreach (var log in list) { Log(log.GetTimingsReport(true)); } list.Clear(); #endif Log("------------------------------------------------"); await RunMultiClientTest(logged, location, true, "high contention queue", 4, NUM, this.Cancellation); #if ENABLE_LOGGING foreach (var log in list) { Log(log.GetTimingsReport(true)); } list.Clear(); #endif Log("------------------------------------------------"); } }
private async Task RunAsync(IFdbDatabase db, FdbSubspace location, CancellationToken ct, Action done, int N, int K, int W) { if (db == null) { throw new ArgumentNullException("db"); } StringBuilder sb = new StringBuilder(); db = new FdbLoggedDatabase(db, false, false, (log) => { sb.AppendLine(log.Log.GetTimingsReport(true)); //Console.WriteLine(log.Log.GetTimingsReport(true)); }); try { var workerPool = new FdbWorkerPool(location); Console.WriteLine("workerPool at " + location.Key.ToAsciiOrHexaString()); var workerSignal = new AsyncCancelableMutex(ct); var clientSignal = new AsyncCancelableMutex(ct); int taskCounter = 0; int msgSent = 0; int msgReceived = 0; Func <FdbWorkerMessage, CancellationToken, Task> handler = async(msg, _ct) => { Interlocked.Increment(ref msgReceived); //await Task.Delay(10 + Math.Abs(msg.Id.GetHashCode()) % 50); await Task.Delay(10).ConfigureAwait(false); }; Func <int, Task> worker = async(id) => { await workerSignal.Task.ConfigureAwait(false); Console.WriteLine("Worker #" + id + " is starting"); try { await workerPool.RunWorkerAsync(db, handler, ct).ConfigureAwait(false); } finally { Console.WriteLine("Worker #" + id + " has stopped"); } }; Func <int, Task> client = async(id) => { await clientSignal.Task.ConfigureAwait(false); await Task.Delay(10).ConfigureAwait(false); var rnd = new Random(id * 111); for (int i = 0; i < N; i++) { var taskId = Slice.FromString("T" + Interlocked.Increment(ref taskCounter)); var taskBody = Slice.FromString("Message " + (i + 1) + " of " + N + " from client #" + id); await workerPool.ScheduleTaskAsync(db, taskId, taskBody, ct).ConfigureAwait(false); Interlocked.Increment(ref msgSent); //if (i > 0 && i % 10 == 0) Console.WriteLine("@@@ Client#" + id + " pushed " + (i + 1) + " / " + N + " messages"); switch (rnd.Next(5)) { case 0: await Task.Delay(10).ConfigureAwait(false); break; case 1: await Task.Delay(100).ConfigureAwait(false); break; case 2: await Task.Delay(500).ConfigureAwait(false); break; } } Console.WriteLine("@@@ Client#" + id + " has finished!"); }; Func <string, Task> dump = async(label) => { Console.WriteLine("<dump label='" + label + "' key='" + location.Key.ToAsciiOrHexaString() + "'>"); using (var tr = db.BeginTransaction(ct)) { await tr.Snapshot .GetRange(FdbKeyRange.StartsWith(location.Key)) .ForEachAsync((kvp) => { Console.WriteLine(" - " + FdbTuple.Unpack(location.Extract(kvp.Key)) + " = " + kvp.Value.ToAsciiOrHexaString()); }).ConfigureAwait(false); } Console.WriteLine("</dump>"); }; var workers = Enumerable.Range(0, W).Select((i) => worker(i)).ToArray(); var clients = Enumerable.Range(0, K).Select((i) => client(i)).ToArray(); DateTime start = DateTime.Now; DateTime last = start; int lastHandled = -1; using (var timer = new Timer((_) => { var now = DateTime.Now; Console.WriteLine("@@@ T=" + now.Subtract(start) + ", sent: " + msgSent.ToString("N0") + ", recv: " + msgReceived.ToString("N0")); Console.WriteLine("### Workers: " + workerPool.IdleWorkers + " / " + workerPool.ActiveWorkers + " (" + new string('#', workerPool.IdleWorkers) + new string('.', workerPool.ActiveWorkers - workerPool.IdleWorkers) + "), sent: " + workerPool.MessageScheduled.ToString("N0") + ", recv: " + workerPool.MessageReceived.ToString("N0") + ", delta: " + (workerPool.MessageScheduled - workerPool.MessageReceived).ToString("N0") + ", busy: " + workerPool.WorkerBusyTime + " (avg " + workerPool.WorkerAverageBusyDuration.TotalMilliseconds.ToString("N3") + " ms)"); if (now.Subtract(last).TotalSeconds >= 10) { //dump("timer").GetAwaiter().GetResult(); last = now; if (lastHandled == msgReceived) { // STALL ? Console.WriteLine("STALL! "); done(); } lastHandled = msgReceived; } if (msgReceived >= K * N) { dump("complete").GetAwaiter().GetResult(); done(); } }, null, 1000, 1000)) { var sw = Stopwatch.StartNew(); // start the workers workerSignal.Set(async: true); await Task.Delay(500); await dump("workers started"); // start the clients clientSignal.Set(async: true); await Task.WhenAll(clients); Console.WriteLine("Clients completed after " + sw.Elapsed); await Task.WhenAll(workers); Console.WriteLine("Workers completed after " + sw.Elapsed); } } finally { Console.WriteLine("---------------------------------------------------------------------------"); Console.WriteLine("Transaction logs:"); Console.WriteLine(); Console.WriteLine(sb.ToString()); } }
public async Task Test_Can_Log_A_Transaction() { const int N = 10; using (var db = await OpenTestPartitionAsync()) { // get a tuple view of the directory var location = (await GetCleanDirectory(db, "Logging")).Keys; // note: ensure that all methods are JITed await db.ReadWriteAsync(async (tr) => { await tr.GetReadVersionAsync(); tr.Set(location.Encode("Warmup", 0), Slice.FromInt32(1)); tr.Clear(location.Encode("Warmup", 1)); await tr.GetAsync(location.Encode("Warmup", 2)); await tr.GetRange(KeyRange.StartsWith(location.Encode("Warmup", 3))).ToListAsync(); tr.ClearRange(location.Encode("Warmup", 4), location.Encode("Warmup", 5)); }, this.Cancellation); await db.WriteAsync((tr) => { var rnd = new Random(); tr.Set(location.Encode("One"), Slice.FromString("111111")); tr.Set(location.Encode("Two"), Slice.FromString("222222")); for (int j = 0; j < 4; j++) { for (int i = 0; i < 100; i++) { tr.Set(location.Encode("Range", j, rnd.Next(1000)), Slice.Empty); } } for (int j = 0; j < N; j++) { tr.Set(location.Encode("X", j), Slice.FromInt32(j)); tr.Set(location.Encode("Y", j), Slice.FromInt32(j)); tr.Set(location.Encode("Z", j), Slice.FromInt32(j)); tr.Set(location.Encode("W", j), Slice.FromInt32(j)); } }, this.Cancellation); bool first = true; Action <FdbLoggedTransaction> logHandler = (tr) => { if (first) { Log(tr.Log.GetCommandsReport()); first = false; } Log(tr.Log.GetTimingsReport(true)); }; // create a logged version of the database var logged = new FdbLoggedDatabase(db, false, false, logHandler); for (int k = 0; k < N; k++) { Log("==== " + k + " ==== "); Log(); await logged.ReadWriteAsync(async (tr) => { Assert.That(tr, Is.InstanceOf <FdbLoggedTransaction>()); //tr.SetOption(FdbTransactionOption.CausalReadRisky); long ver = await tr.GetReadVersionAsync().ConfigureAwait(false); await tr.GetAsync(location.Encode("One")).ConfigureAwait(false); await tr.GetAsync(location.Encode("NotFound")).ConfigureAwait(false); tr.Set(location.Encode("Write"), Slice.FromString("abcdef" + k.ToString())); //tr.Annotate("BEFORE"); //await Task.Delay(TimeSpan.FromMilliseconds(10)); //tr.Annotate("AFTER"); //await tr.Snapshot.GetAsync(location.Pack("Snap")).ConfigureAwait(false); tr.Annotate("This is a comment"); //await tr.GetRangeAsync(KeySelector.LastLessOrEqual(location.Pack("A")), KeySelector.FirstGreaterThan(location.Pack("Z"))).ConfigureAwait(false); await Task.WhenAll( tr.GetRange(KeyRange.StartsWith(location.Encode("Range", 0))).ToListAsync(), tr.GetRange(location.Encode("Range", 1, 0), location.Encode("Range", 1, 200)).ToListAsync(), tr.GetRange(location.Encode("Range", 2, 400), location.Encode("Range", 2, 600)).ToListAsync(), tr.GetRange(location.Encode("Range", 3, 800), location.Encode("Range", 3, 1000)).ToListAsync() ).ConfigureAwait(false); await tr.GetAsync(location.Encode("Two")).ConfigureAwait(false); await tr.GetValuesAsync(Enumerable.Range(0, N).Select(x => location.Encode("X", x))).ConfigureAwait(false); for (int i = 0; i < N; i++) { await tr.GetAsync(location.Encode("Z", i)).ConfigureAwait(false); } await Task.WhenAll(Enumerable.Range(0, N / 2).Select(x => tr.GetAsync(location.Encode("Y", x)))).ConfigureAwait(false); await Task.WhenAll(Enumerable.Range(N / 2, N / 2).Select(x => tr.GetAsync(location.Encode("Y", x)))).ConfigureAwait(false); await Task.WhenAll( tr.GetAsync(location.Encode("W", 1)), tr.GetAsync(location.Encode("W", 2)), tr.GetAsync(location.Encode("W", 3)) ).ConfigureAwait(false); tr.Set(location.Encode("Write2"), Slice.FromString("ghijkl" + k.ToString())); tr.Clear(location.Encode("Clear", "0")); tr.ClearRange(location.Encode("Clear", "A"), location.Encode("Clear", "Z")); if (tr.Context.Retries == 0) { // make it fail //throw new FdbException(FdbError.PastVersion, "fake timeout"); } }, this.Cancellation); } } }
public async Task Test_Can_Log_A_Transaction() { const int N = 10; using (var db = await OpenTestPartitionAsync()) { var location = await GetCleanDirectory(db, "Logging"); // note: ensure that all methods are JITed await db.ReadWriteAsync(async (tr) => { await tr.GetReadVersionAsync(); tr.Set(location.Pack("Warmup", 0), Slice.FromInt32(1)); tr.Clear(location.Pack("Warmup", 1)); await tr.GetAsync(location.Pack("Warmup", 2)); await tr.GetRange(FdbKeyRange.StartsWith(location.Pack("Warmup", 3))).ToListAsync(); tr.ClearRange(location.Pack("Warmup", 4), location.Pack("Warmup", 5)); }, this.Cancellation); await db.WriteAsync((tr) => { var rnd = new Random(); tr.Set(location.Pack("One"), Slice.FromString("111111")); tr.Set(location.Pack("Two"), Slice.FromString("222222")); for (int j = 0; j < 4; j++) { for (int i = 0; i < 100; i++) { tr.Set(location.Pack("Range", j, rnd.Next(1000)), Slice.Empty); } } for (int j = 0; j < N; j++) { tr.Set(location.Pack("X", j), Slice.FromInt32(j)); tr.Set(location.Pack("Y", j), Slice.FromInt32(j)); tr.Set(location.Pack("Z", j), Slice.FromInt32(j)); tr.Set(location.Pack("W", j), Slice.FromInt32(j)); } }, this.Cancellation); bool first = true; Action<FdbLoggedTransaction> logHandler = (tr) => { if (first) { Console.WriteLine(tr.Log.GetCommandsReport()); first = false; } Console.WriteLine(tr.Log.GetTimingsReport(true)); }; // create a logged version of the database var logged = new FdbLoggedDatabase(db, false, false, logHandler); for (int k = 0; k < N; k++) { Console.WriteLine("==== " + k + " ==== "); Console.WriteLine(); await logged.ReadWriteAsync(async (tr) => { Assert.That(tr, Is.InstanceOf<FdbLoggedTransaction>()); //tr.SetOption(FdbTransactionOption.CausalReadRisky); long ver = await tr.GetReadVersionAsync().ConfigureAwait(false); await tr.GetAsync(location.Pack("One")).ConfigureAwait(false); await tr.GetAsync(location.Pack("NotFound")).ConfigureAwait(false); tr.Set(location.Pack("Write"), Slice.FromString("abcdef" + k.ToString())); //tr.Annotate("BEFORE"); //await Task.Delay(TimeSpan.FromMilliseconds(10)); //tr.Annotate("AFTER"); //await tr.Snapshot.GetAsync(location.Pack("Snap")).ConfigureAwait(false); tr.Annotate("This is a comment"); //await tr.GetRangeAsync(FdbKeySelector.LastLessOrEqual(location.Pack("A")), FdbKeySelector.FirstGreaterThan(location.Pack("Z"))).ConfigureAwait(false); await Task.WhenAll( tr.GetRange(FdbKeyRange.StartsWith(location.Pack("Range", 0))).ToListAsync(), tr.GetRange(location.Pack("Range", 1, 0), location.Pack("Range", 1, 200)).ToListAsync(), tr.GetRange(location.Pack("Range", 2, 400), location.Pack("Range", 2, 600)).ToListAsync(), tr.GetRange(location.Pack("Range", 3, 800), location.Pack("Range", 3, 1000)).ToListAsync() ).ConfigureAwait(false); await tr.GetAsync(location.Pack("Two")).ConfigureAwait(false); await tr.GetValuesAsync(Enumerable.Range(0, N).Select(x => location.Pack("X", x))).ConfigureAwait(false); for (int i = 0; i < N; i++) { await tr.GetAsync(location.Pack("Z", i)).ConfigureAwait(false); } await Task.WhenAll(Enumerable.Range(0, N / 2).Select(x => tr.GetAsync(location.Pack("Y", x)))).ConfigureAwait(false); await Task.WhenAll(Enumerable.Range(N / 2, N / 2).Select(x => tr.GetAsync(location.Pack("Y", x)))).ConfigureAwait(false); await Task.WhenAll( tr.GetAsync(location.Pack("W", 1)), tr.GetAsync(location.Pack("W", 2)), tr.GetAsync(location.Pack("W", 3)) ).ConfigureAwait(false); tr.Set(location.Pack("Write2"), Slice.FromString("ghijkl" + k.ToString())); tr.Clear(location.Pack("Clear", "0")); tr.ClearRange(location.Pack("Clear", "A"), location.Pack("Clear", "Z")); if (tr.Context.Retries == 0) { // make it fail //throw new FdbException(FdbError.PastVersion, "fake timeout"); } }, this.Cancellation); } } }