public static async Task <TimeSpan> RunConcurrentWorkersAsync(int workers,
Func <int, CancellationToken, Task> handler,
CancellationToken ct)
{
await Task.Delay(1).ConfigureAwait(false);
var signal = new AsyncCancelableMutex(ct);
var tasks = Enumerable.Range(0, workers).Select(async(i) =>
{
await signal.Task;
ct.ThrowIfCancellationRequested();
await handler(i, ct);
}).ToArray();
var sw = Stopwatch.StartNew();
signal.Set(async: true);
await Task.WhenAll(tasks);
sw.Stop();
return(sw.Elapsed);
}
private async Task RunAsync(IFdbDatabase db, FdbSubspace location, CancellationToken ct, Action done, int N, int K, int W)
{
if (db == null)
{
throw new ArgumentNullException("db");
}
StringBuilder sb = new StringBuilder();
db = new FdbLoggedDatabase(db, false, false, (log) =>
{
sb.AppendLine(log.Log.GetTimingsReport(true));
//Console.WriteLine(log.Log.GetTimingsReport(true));
});
try
{
var workerPool = new FdbWorkerPool(location);
Console.WriteLine("workerPool at " + location.Key.ToAsciiOrHexaString());
var workerSignal = new AsyncCancelableMutex(ct);
var clientSignal = new AsyncCancelableMutex(ct);
int taskCounter = 0;
int msgSent = 0;
int msgReceived = 0;
Func <FdbWorkerMessage, CancellationToken, Task> handler = async(msg, _ct) =>
{
Interlocked.Increment(ref msgReceived);
//await Task.Delay(10 + Math.Abs(msg.Id.GetHashCode()) % 50);
await Task.Delay(10).ConfigureAwait(false);
};
Func <int, Task> worker = async(id) =>
{
await workerSignal.Task.ConfigureAwait(false);
Console.WriteLine("Worker #" + id + " is starting");
try
{
await workerPool.RunWorkerAsync(db, handler, ct).ConfigureAwait(false);
}
finally
{
Console.WriteLine("Worker #" + id + " has stopped");
}
};
Func <int, Task> client = async(id) =>
{
await clientSignal.Task.ConfigureAwait(false);
await Task.Delay(10).ConfigureAwait(false);
var rnd = new Random(id * 111);
for (int i = 0; i < N; i++)
{
var taskId = Slice.FromString("T" + Interlocked.Increment(ref taskCounter));
var taskBody = Slice.FromString("Message " + (i + 1) + " of " + N + " from client #" + id);
await workerPool.ScheduleTaskAsync(db, taskId, taskBody, ct).ConfigureAwait(false);
Interlocked.Increment(ref msgSent);
//if (i > 0 && i % 10 == 0) Console.WriteLine("@@@ Client#" + id + " pushed " + (i + 1) + " / " + N + " messages");
switch (rnd.Next(5))
{
case 0: await Task.Delay(10).ConfigureAwait(false); break;
case 1: await Task.Delay(100).ConfigureAwait(false); break;
case 2: await Task.Delay(500).ConfigureAwait(false); break;
}
}
Console.WriteLine("@@@ Client#" + id + " has finished!");
};
Func <string, Task> dump = async(label) =>
{
Console.WriteLine("<dump label='" + label + "' key='" + location.Key.ToAsciiOrHexaString() + "'>");
using (var tr = db.BeginTransaction(ct))
{
await tr.Snapshot
.GetRange(FdbKeyRange.StartsWith(location.Key))
.ForEachAsync((kvp) =>
{
Console.WriteLine(" - " + FdbTuple.Unpack(location.Extract(kvp.Key)) + " = " + kvp.Value.ToAsciiOrHexaString());
}).ConfigureAwait(false);
}
Console.WriteLine("</dump>");
};
var workers = Enumerable.Range(0, W).Select((i) => worker(i)).ToArray();
var clients = Enumerable.Range(0, K).Select((i) => client(i)).ToArray();
DateTime start = DateTime.Now;
DateTime last = start;
int lastHandled = -1;
using (var timer = new Timer((_) =>
{
var now = DateTime.Now;
Console.WriteLine("@@@ T=" + now.Subtract(start) + ", sent: " + msgSent.ToString("N0") + ", recv: " + msgReceived.ToString("N0"));
Console.WriteLine("### Workers: " + workerPool.IdleWorkers + " / " + workerPool.ActiveWorkers + " (" + new string('#', workerPool.IdleWorkers) + new string('.', workerPool.ActiveWorkers - workerPool.IdleWorkers) + "), sent: " + workerPool.MessageScheduled.ToString("N0") + ", recv: " + workerPool.MessageReceived.ToString("N0") + ", delta: " + (workerPool.MessageScheduled - workerPool.MessageReceived).ToString("N0") + ", busy: " + workerPool.WorkerBusyTime + " (avg " + workerPool.WorkerAverageBusyDuration.TotalMilliseconds.ToString("N3") + " ms)");
if (now.Subtract(last).TotalSeconds >= 10)
{
//dump("timer").GetAwaiter().GetResult();
last = now;
if (lastHandled == msgReceived)
{ // STALL ?
Console.WriteLine("STALL! ");
done();
}
lastHandled = msgReceived;
}
if (msgReceived >= K * N)
{
dump("complete").GetAwaiter().GetResult();
done();
}
}, null, 1000, 1000))
{
var sw = Stopwatch.StartNew();
// start the workers
workerSignal.Set(async: true);
await Task.Delay(500);
await dump("workers started");
// start the clients
clientSignal.Set(async: true);
await Task.WhenAll(clients);
Console.WriteLine("Clients completed after " + sw.Elapsed);
await Task.WhenAll(workers);
Console.WriteLine("Workers completed after " + sw.Elapsed);
}
}
finally
{
Console.WriteLine("---------------------------------------------------------------------------");
Console.WriteLine("Transaction logs:");
Console.WriteLine();
Console.WriteLine(sb.ToString());
}
}
private async Task RunMultiClientTest(IFdbDatabase db, FdbDirectorySubspaceLocation location, string desc, int K, int NUM, CancellationToken ct)
{
Log($"Starting {desc} test with {K} threads and {NUM} iterations");
await CleanLocation(db, location);
var queue = new FdbQueue <string>(location);
// use a CTS to ensure that everything will stop in case of problems...
using (var go = new CancellationTokenSource(TimeSpan.FromSeconds(30)))
{
var tok = go.Token;
var pushLock = new AsyncCancelableMutex(tok);
var popLock = new AsyncCancelableMutex(tok);
int pushCount = 0;
int popCount = 0;
int stalls = 0;
var start = DateTime.UtcNow;
var pushTreads = Enumerable.Range(0, K)
.Select(async id =>
{
int i = 0;
try
{
// wait for the signal
await pushLock.Task.ConfigureAwait(false);
var res = new List <string>(NUM);
for (; i < NUM; i++)
{
var item = $"{id}.{i}";
await queue.WriteAsync(db, (tr, state) => state.Push(tr, item), tok).ConfigureAwait(false);
Interlocked.Increment(ref pushCount);
res.Add(item);
}
Log($"PushThread[{id}] pushed {NUM:N0} items in {(DateTime.UtcNow - start).TotalSeconds:N1} sec");
return(res);
}
catch (Exception e)
{
Log($"PushThread[{id}] failed after {i} push and {(DateTime.UtcNow - start).TotalSeconds:N1} sec: {e}");
Assert.Fail($"PushThread[{id}] failed: {e.Message}");
throw;
}
}).ToArray();
var popThreads = Enumerable.Range(0, K)
.Select(async id =>
{
int i = 0;
try
{
// make everyone wait a bit, to ensure that they all start roughly at the same time
await popLock.Task.ConfigureAwait(false);
var res = new List <string>(NUM);
while (i < NUM)
{
var item = await queue.ReadWriteAsync(db, (tr, state) => state.PopAsync(tr), tok).ConfigureAwait(false);
if (item.HasValue)
{
Interlocked.Increment(ref popCount);
res.Add(item.Value);
++i;
}
else
{
Interlocked.Increment(ref stalls);
await Task.Delay(10, this.Cancellation).ConfigureAwait(false);
}
}
Log($"PopThread[{id}] popped {NUM:N0} items in {(DateTime.UtcNow - start).TotalSeconds:N1} sec");
return(res);
}
catch (Exception e)
{
Log($"PopThread[{id}] failed: {e}");
Assert.Fail($"PopThread[{id}] failed after {i} pops and {(DateTime.UtcNow - start).TotalSeconds:N1} sec: {e.Message}");
throw;
}
}).ToArray();
var sw = Stopwatch.StartNew();
pushLock.Set(async: true);
await Task.Delay(50, this.Cancellation);
popLock.Set(async: true);
await Task.WhenAll(pushTreads);
Log("Push threads are finished!");
await Task.WhenAll(popThreads);
sw.Stop();
Log($"> Finished {desc} test in {sw.Elapsed.TotalSeconds} seconds");
Log($"> Pushed {pushCount}, Popped {popCount} and Stalled {stalls}");
var pushedItems = pushTreads.SelectMany(t => t.Result).ToList();
var poppedItems = popThreads.SelectMany(t => t.Result).ToList();
Assert.That(pushCount, Is.EqualTo(K * NUM));
Assert.That(popCount, Is.EqualTo(K * NUM));
// all pushed items should have been popped (with no duplicates)
Assert.That(poppedItems, Is.EquivalentTo(pushedItems));
// the queue should be empty
bool empty = await queue.ReadAsync(db, (tr, state) => state.EmptyAsync(tr), ct);
Assert.That(empty, Is.True);
}
}
private static async Task RunMultiClientTest(IFdbDatabase db, KeySubspace location, bool highContention, string desc, int K, int NUM, CancellationToken ct)
{
Log("Starting {0} test with {1} threads and {2} iterations", desc, K, NUM);
var queue = new FdbQueue <string>(location, highContention);
await db.WriteAsync((tr) => queue.Clear(tr), ct);
// use a CTS to ensure that everything will stop in case of problems...
using (var go = new CancellationTokenSource(TimeSpan.FromSeconds(30)))
{
var tok = go.Token;
var pushLock = new AsyncCancelableMutex(tok);
var popLock = new AsyncCancelableMutex(tok);
int pushCount = 0;
int popCount = 0;
int stalls = 0;
var pushTreads = Enumerable.Range(0, K)
.Select(async id =>
{
try
{
// wait for the signal
await pushLock.Task.ConfigureAwait(false);
var res = new List <string>(NUM);
for (int i = 0; i < NUM; i++)
{
var item = id.ToString() + "." + i.ToString();
await db.ReadWriteAsync((tr) => queue.PushAsync(tr, item), tok).ConfigureAwait(false);
Interlocked.Increment(ref pushCount);
res.Add(item);
}
return(res);
}
catch (Exception e)
{
Log("PushThread[" + id + "] failed: " + e);
Assert.Fail("PushThread[" + id + "] failed: " + e.Message);
throw;
}
}).ToArray();
var popThreads = Enumerable.Range(0, K)
.Select(async id =>
{
try
{
// make everyone wait a bit, to ensure that they all start roughly at the same time
await popLock.Task.ConfigureAwait(false);
var res = new List <string>(NUM);
int i = 0;
while (i < NUM)
{
var item = await queue.PopAsync(db, tok).ConfigureAwait(false);
if (item.HasValue)
{
Interlocked.Increment(ref popCount);
res.Add(item.Value);
++i;
}
else
{
Interlocked.Increment(ref stalls);
await Task.Delay(10).ConfigureAwait(false);
}
}
return(res);
}
catch (Exception e)
{
Log("PopThread[" + id + "] failed: " + e);
Assert.Fail("PopThread[" + id + "] failed: " + e.Message);
throw;
}
}).ToArray();
var sw = Stopwatch.StartNew();
pushLock.Set(async: true);
await Task.Delay(100);
popLock.Set(async: true);
//using (var timer = new Timer((_) =>
//{
// var __ = TestHelpers.DumpSubspace(db, location);
//}, null, 1000, 4000))
{
await Task.WhenAll(pushTreads);
await Task.WhenAll(popThreads);
}
sw.Stop();
Log("> Finished {0} test in {1} seconds", desc, sw.Elapsed.TotalSeconds);
Log("> Pushed {0}, Popped {1} and Stalled {2}", pushCount, popCount, stalls);
var pushedItems = pushTreads.SelectMany(t => t.Result).ToList();
var poppedItems = popThreads.SelectMany(t => t.Result).ToList();
Assert.That(pushCount, Is.EqualTo(K * NUM));
Assert.That(popCount, Is.EqualTo(K * NUM));
// all pushed items should have been popped (with no duplicates)
Assert.That(poppedItems, Is.EquivalentTo(pushedItems));
// the queue should be empty
bool empty = await db.ReadAsync((tr) => queue.EmptyAsync(tr), ct);
Assert.That(empty, Is.True);
}
}
private static async Task RunMultiClientTest(IFdbDatabase db, FdbSubspace location, bool highContention, string desc, int K, int NUM, CancellationToken ct)
{
Console.WriteLine("Starting {0} test with {1} threads and {2} iterations", desc, K, NUM);
var queue = new FdbQueue<string>(location, highContention);
await db.WriteAsync((tr) => queue.Clear(tr), ct);
// use a CTS to ensure that everything will stop in case of problems...
using (var go = new CancellationTokenSource(TimeSpan.FromSeconds(30)))
{
var tok = go.Token;
var pushLock = new AsyncCancelableMutex(tok);
var popLock = new AsyncCancelableMutex(tok);
int pushCount = 0;
int popCount = 0;
int stalls = 0;
var pushTreads = Enumerable.Range(0, K)
.Select(async id =>
{
// wait for the signal
await pushLock.Task.ConfigureAwait(false);
var res = new List<string>(NUM);
for (int i = 0; i < NUM; i++)
{
var item = id.ToString() + "." + i.ToString();
await db.ReadWriteAsync((tr) => queue.PushAsync(tr, item), tok).ConfigureAwait(false);
Interlocked.Increment(ref pushCount);
res.Add(item);
}
return res;
}).ToArray();
var popThreads = Enumerable.Range(0, K)
.Select(async id =>
{
// make everyone wait a bit, to ensure that they all start roughly at the same time
await popLock.Task.ConfigureAwait(false);
var res = new List<string>(NUM);
int i = 0;
while (i < NUM)
{
var item = await queue.PopAsync(db, tok).ConfigureAwait(false);
if (item.HasValue)
{
Interlocked.Increment(ref popCount);
res.Add(item.Value);
++i;
}
else
{
Interlocked.Increment(ref stalls);
await Task.Delay(10).ConfigureAwait(false);
}
}
return res;
}).ToArray();
var sw = Stopwatch.StartNew();
pushLock.Set(async: true);
await Task.Delay(100);
popLock.Set(async: true);
//using (var timer = new Timer((_) =>
//{
// var __ = TestHelpers.DumpSubspace(db, location);
//}, null, 1000, 4000))
{
await Task.WhenAll(pushTreads);
await Task.WhenAll(popThreads);
}
sw.Stop();
Console.WriteLine("> Finished {0} test in {1} seconds", desc, sw.Elapsed.TotalSeconds);
Console.WriteLine("> Pushed {0}, Popped {1} and Stalled {2}", pushCount, popCount, stalls);
var pushedItems = pushTreads.SelectMany(t => t.Result).ToList();
var poppedItems = popThreads.SelectMany(t => t.Result).ToList();
Assert.That(pushCount, Is.EqualTo(K * NUM));
Assert.That(popCount, Is.EqualTo(K * NUM));
// all pushed items should have been popped (with no duplicates)
Assert.That(poppedItems, Is.EquivalentTo(pushedItems));
// the queue should be empty
bool empty = await db.ReadAsync((tr) => queue.EmptyAsync(tr), ct);
Assert.That(empty, Is.True);
}
}
private async Task RunAsync(IFdbDatabase db, IFdbDynamicSubspace location, CancellationToken ct, Action done, int N, int K, int W)
{
if (db == null) throw new ArgumentNullException("db");
StringBuilder sb = new StringBuilder();
db = new FdbLoggedDatabase(db, false, false, (log) =>
{
sb.AppendLine(log.Log.GetTimingsReport(true));
//Console.WriteLine(log.Log.GetTimingsReport(true));
});
try
{
var workerPool = new FdbWorkerPool(location);
Console.WriteLine("workerPool at " + location.Key.ToAsciiOrHexaString());
var workerSignal = new AsyncCancelableMutex(ct);
var clientSignal = new AsyncCancelableMutex(ct);
int taskCounter = 0;
int msgSent = 0;
int msgReceived = 0;
Func<FdbWorkerMessage, CancellationToken, Task> handler = async (msg, _ct) =>
{
Interlocked.Increment(ref msgReceived);
//await Task.Delay(10 + Math.Abs(msg.Id.GetHashCode()) % 50);
await Task.Delay(10).ConfigureAwait(false);
};
Func<int, Task> worker = async (id) =>
{
await workerSignal.Task.ConfigureAwait(false);
Console.WriteLine("Worker #" + id + " is starting");
try
{
await workerPool.RunWorkerAsync(db, handler, ct).ConfigureAwait(false);
}
finally
{
Console.WriteLine("Worker #" + id + " has stopped");
}
};
Func<int, Task> client = async (id) =>
{
await clientSignal.Task.ConfigureAwait(false);
await Task.Delay(10).ConfigureAwait(false);
var rnd = new Random(id * 111);
for (int i = 0; i < N; i++)
{
var taskId = Slice.FromString("T" + Interlocked.Increment(ref taskCounter));
var taskBody = Slice.FromString("Message " + (i + 1) + " of " + N + " from client #" + id);
await workerPool.ScheduleTaskAsync(db, taskId, taskBody, ct).ConfigureAwait(false);
Interlocked.Increment(ref msgSent);
//if (i > 0 && i % 10 == 0) Console.WriteLine("@@@ Client#" + id + " pushed " + (i + 1) + " / " + N + " messages");
switch (rnd.Next(5))
{
case 0: await Task.Delay(10).ConfigureAwait(false); break;
case 1: await Task.Delay(100).ConfigureAwait(false); break;
case 2: await Task.Delay(500).ConfigureAwait(false); break;
}
}
Console.WriteLine("@@@ Client#" + id + " has finished!");
};
Func<string, Task> dump = async (label) =>
{
Console.WriteLine("<dump label='" + label + "' key='" + location.Key.ToAsciiOrHexaString() + "'>");
using (var tr = db.BeginTransaction(ct))
{
await tr.Snapshot
.GetRange(FdbKeyRange.StartsWith(location.Key))
.ForEachAsync((kvp) =>
{
Console.WriteLine(" - " + location.Keys.Unpack(kvp.Key) + " = " + kvp.Value.ToAsciiOrHexaString());
}).ConfigureAwait(false);
}
Console.WriteLine("</dump>");
};
var workers = Enumerable.Range(0, W).Select((i) => worker(i)).ToArray();
var clients = Enumerable.Range(0, K).Select((i) => client(i)).ToArray();
DateTime start = DateTime.Now;
DateTime last = start;
int lastHandled = -1;
using (var timer = new Timer((_) =>
{
var now = DateTime.Now;
Console.WriteLine("@@@ T=" + now.Subtract(start) + ", sent: " + msgSent.ToString("N0") + ", recv: " + msgReceived.ToString("N0"));
Console.WriteLine("### Workers: " + workerPool.IdleWorkers + " / " + workerPool.ActiveWorkers + " (" + new string('#', workerPool.IdleWorkers) + new string('.', workerPool.ActiveWorkers - workerPool.IdleWorkers) + "), sent: " + workerPool.MessageScheduled.ToString("N0") + ", recv: " + workerPool.MessageReceived.ToString("N0") + ", delta: " + (workerPool.MessageScheduled - workerPool.MessageReceived).ToString("N0") + ", busy: " + workerPool.WorkerBusyTime + " (avg " + workerPool.WorkerAverageBusyDuration.TotalMilliseconds.ToString("N3") + " ms)");
if (now.Subtract(last).TotalSeconds >= 10)
{
//dump("timer").GetAwaiter().GetResult();
last = now;
if (lastHandled == msgReceived)
{ // STALL ?
Console.WriteLine("STALL! ");
done();
}
lastHandled = msgReceived;
}
if (msgReceived >= K * N)
{
dump("complete").GetAwaiter().GetResult();
done();
}
}, null, 1000, 1000))
{
var sw = Stopwatch.StartNew();
// start the workers
workerSignal.Set(async: true);
await Task.Delay(500);
await dump("workers started");
// start the clients
clientSignal.Set(async: true);
await Task.WhenAll(clients);
Console.WriteLine("Clients completed after " + sw.Elapsed);
await Task.WhenAll(workers);
Console.WriteLine("Workers completed after " + sw.Elapsed);
}
}
finally
{
Console.WriteLine("---------------------------------------------------------------------------");
Console.WriteLine("Transaction logs:");
Console.WriteLine();
Console.WriteLine(sb.ToString());
}
}
public static async Task<TimeSpan> RunConcurrentWorkersAsync(int workers,
Func<int, CancellationToken, Task> handler,
CancellationToken ct)
{
await Task.Delay(1).ConfigureAwait(false);
var signal = new AsyncCancelableMutex(ct);
var tasks = Enumerable.Range(0, workers).Select(async (i) =>
{
await signal.Task;
ct.ThrowIfCancellationRequested();
await handler(i, ct);
}).ToArray();
var sw = Stopwatch.StartNew();
signal.Set(async: true);
await Task.WhenAll(tasks);
sw.Stop();
return sw.Elapsed;
}