public async Task Test_Single_Client()
{
using (var db = await OpenTestPartitionAsync())
{
var location = await GetCleanDirectory(db, "queue");
var queue = new FdbQueue <int>(location, highContention: false);
await db.WriteAsync((tr) => queue.Clear(tr), this.Cancellation);
for (int i = 0; i < 10; i++)
{
await db.ReadWriteAsync((tr) => queue.PushAsync(tr, i), this.Cancellation);
}
for (int i = 0; i < 10; i++)
{
var r = await queue.PopAsync(db, this.Cancellation);
Assert.That(r.HasValue, Is.True);
Assert.That(r.Value, Is.EqualTo(i));
}
bool empty = await db.ReadAsync((tr) => queue.EmptyAsync(tr), this.Cancellation);
Assert.That(empty, Is.True);
}
}
public async Task Test_Queue_Fast()
{
// without high contention protecction
using (var db = await OpenTestPartitionAsync())
{
var location = await GetCleanDirectory(db, "queue");
var queue = new FdbQueue<int>(location, highContention: false);
Console.WriteLine("Clear Queue");
await queue.ClearAsync(db, this.Cancellation);
Console.WriteLine("Empty? " + await queue.EmptyAsync(db, this.Cancellation));
Console.WriteLine("Push 10, 8, 6");
await queue.PushAsync(db, 10, this.Cancellation);
await queue.PushAsync(db, 8, this.Cancellation);
await queue.PushAsync(db, 6, this.Cancellation);
#if DEBUG
await DumpSubspace(db, location);
#endif
Console.WriteLine("Empty? " + await queue.EmptyAsync(db, this.Cancellation));
Console.WriteLine("Pop item: " + await queue.PopAsync(db, this.Cancellation));
Console.WriteLine("Next item: " + await queue.PeekAsync(db, this.Cancellation));
#if DEBUG
await DumpSubspace(db, location);
#endif
Console.WriteLine("Pop item: " + await queue.PopAsync(db, this.Cancellation));
#if DEBUG
await DumpSubspace(db, location);
#endif
Console.WriteLine("Pop item: " + await queue.PopAsync(db, this.Cancellation));
#if DEBUG
await DumpSubspace(db, location);
#endif
Console.WriteLine("Empty? " + await queue.EmptyAsync(db, this.Cancellation));
Console.WriteLine("Push 5");
await queue.PushAsync(db, 5, this.Cancellation);
#if DEBUG
await DumpSubspace(db, location);
#endif
Console.WriteLine("Clear Queue");
await queue.ClearAsync(db, this.Cancellation);
#if DEBUG
await DumpSubspace(db, location);
#endif
Console.WriteLine("Empty? " + await queue.EmptyAsync(db, this.Cancellation));
}
}
public async Task Test_Queue_Fast()
{
using (var db = await OpenTestPartitionAsync())
{
var location = db.Root["queue"];
await CleanLocation(db, location);
#if ENABLE_LOGGING
var logged = db.Logged((tr) => Log(tr.Log.GetTimingsReport(true)));
#else
var logged = db;
#endif
var queue = new FdbQueue <int>(location);
Log("Empty? " + queue.ReadAsync(logged, (tr, state) => state.EmptyAsync(tr), this.Cancellation));
Log("Push 10, 8, 6 in separate transactions");
await queue.WriteAsync(logged, (tr, state) => state.Push(tr, 10), this.Cancellation);
await queue.WriteAsync(logged, (tr, state) => state.Push(tr, 8), this.Cancellation);
await queue.WriteAsync(logged, (tr, state) => state.Push(tr, 6), this.Cancellation);
#if DEBUG
await DumpSubspace(db, location);
#endif
// Empty?
bool empty = await queue.ReadAsync(logged, (tr, state) => state.EmptyAsync(tr), this.Cancellation);
Log("Empty? " + empty);
Assert.That(empty, Is.False);
var item = await queue.ReadWriteAsync(logged, (tr, state) => state.PopAsync(tr), this.Cancellation);
Log($"Pop item: {item}");
Assert.That(item.HasValue, Is.True);
Assert.That(item.Value, Is.EqualTo(10));
item = await queue.ReadWriteAsync(logged, (tr, state) => state.PeekAsync(tr), this.Cancellation);
Log($"Next item: {item}");
Assert.That(item.HasValue, Is.True);
Assert.That(item.Value, Is.EqualTo(8));
#if DEBUG
await DumpSubspace(db, location);
#endif
item = await queue.ReadWriteAsync(logged, (tr, state) => state.PopAsync(tr), this.Cancellation);
Log($"Pop item: {item}");
Assert.That(item.HasValue, Is.True);
Assert.That(item.Value, Is.EqualTo(8));
#if DEBUG
await DumpSubspace(db, location);
#endif
item = await queue.ReadWriteAsync(logged, (tr, state) => state.PopAsync(tr), this.Cancellation);
Log($"Pop item: {item}");
Assert.That(item.HasValue, Is.True);
Assert.That(item.Value, Is.EqualTo(6));
#if DEBUG
await DumpSubspace(db, location);
#endif
empty = await queue.ReadAsync(logged, (tr, state) => state.EmptyAsync(tr), this.Cancellation);
Log("Empty? " + empty);
Assert.That(empty, Is.True);
Log("Push 5");
await queue.WriteAsync(logged, (tr, state) => state.Push(tr, 5), this.Cancellation);
#if DEBUG
await DumpSubspace(db, location);
#endif
Log("Clear Queue");
await queue.WriteAsync(logged, (tr, state) => state.Clear(tr), this.Cancellation);
#if DEBUG
await DumpSubspace(db, location);
#endif
empty = await queue.ReadAsync(logged, (tr, state) => state.EmptyAsync(tr), this.Cancellation);
Log("Empty? " + empty);
Assert.That(empty, Is.True);
}
}
private async Task RunMultiClientTest(IFdbDatabase db, FdbDirectorySubspaceLocation location, string desc, int K, int NUM, CancellationToken ct)
{
Log($"Starting {desc} test with {K} threads and {NUM} iterations");
await CleanLocation(db, location);
var queue = new FdbQueue <string>(location);
// use a CTS to ensure that everything will stop in case of problems...
using (var go = new CancellationTokenSource(TimeSpan.FromSeconds(30)))
{
var tok = go.Token;
var pushLock = new AsyncCancelableMutex(tok);
var popLock = new AsyncCancelableMutex(tok);
int pushCount = 0;
int popCount = 0;
int stalls = 0;
var start = DateTime.UtcNow;
var pushTreads = Enumerable.Range(0, K)
.Select(async id =>
{
int i = 0;
try
{
// wait for the signal
await pushLock.Task.ConfigureAwait(false);
var res = new List <string>(NUM);
for (; i < NUM; i++)
{
var item = $"{id}.{i}";
await queue.WriteAsync(db, (tr, state) => state.Push(tr, item), tok).ConfigureAwait(false);
Interlocked.Increment(ref pushCount);
res.Add(item);
}
Log($"PushThread[{id}] pushed {NUM:N0} items in {(DateTime.UtcNow - start).TotalSeconds:N1} sec");
return(res);
}
catch (Exception e)
{
Log($"PushThread[{id}] failed after {i} push and {(DateTime.UtcNow - start).TotalSeconds:N1} sec: {e}");
Assert.Fail($"PushThread[{id}] failed: {e.Message}");
throw;
}
}).ToArray();
var popThreads = Enumerable.Range(0, K)
.Select(async id =>
{
int i = 0;
try
{
// make everyone wait a bit, to ensure that they all start roughly at the same time
await popLock.Task.ConfigureAwait(false);
var res = new List <string>(NUM);
while (i < NUM)
{
var item = await queue.ReadWriteAsync(db, (tr, state) => state.PopAsync(tr), tok).ConfigureAwait(false);
if (item.HasValue)
{
Interlocked.Increment(ref popCount);
res.Add(item.Value);
++i;
}
else
{
Interlocked.Increment(ref stalls);
await Task.Delay(10, this.Cancellation).ConfigureAwait(false);
}
}
Log($"PopThread[{id}] popped {NUM:N0} items in {(DateTime.UtcNow - start).TotalSeconds:N1} sec");
return(res);
}
catch (Exception e)
{
Log($"PopThread[{id}] failed: {e}");
Assert.Fail($"PopThread[{id}] failed after {i} pops and {(DateTime.UtcNow - start).TotalSeconds:N1} sec: {e.Message}");
throw;
}
}).ToArray();
var sw = Stopwatch.StartNew();
pushLock.Set(async: true);
await Task.Delay(50, this.Cancellation);
popLock.Set(async: true);
await Task.WhenAll(pushTreads);
Log("Push threads are finished!");
await Task.WhenAll(popThreads);
sw.Stop();
Log($"> Finished {desc} test in {sw.Elapsed.TotalSeconds} seconds");
Log($"> Pushed {pushCount}, Popped {popCount} and Stalled {stalls}");
var pushedItems = pushTreads.SelectMany(t => t.Result).ToList();
var poppedItems = popThreads.SelectMany(t => t.Result).ToList();
Assert.That(pushCount, Is.EqualTo(K * NUM));
Assert.That(popCount, Is.EqualTo(K * NUM));
// all pushed items should have been popped (with no duplicates)
Assert.That(poppedItems, Is.EquivalentTo(pushedItems));
// the queue should be empty
bool empty = await queue.ReadAsync(db, (tr, state) => state.EmptyAsync(tr), ct);
Assert.That(empty, Is.True);
}
}
public async Task Test_Queue_Batch()
{
using (var db = await OpenTestPartitionAsync())
{
var location = db.Root["queue"];
await CleanLocation(db, location);
#if ENABLE_LOGGING
var logged = db.Logged((tr) => Log(tr.Log.GetTimingsReport(true)));
#else
var logged = db;
#endif
var queue = new FdbQueue <int>(location);
Log("Pushing 10 items in a batch...");
await queue.WriteAsync(logged, (tr, state) =>
{
for (int i = 0; i < 10; i++)
{
state.Push(tr, i);
}
}, this.Cancellation);
#if DEBUG
await DumpSubspace(db, queue.Location);
#endif
Log("Popping 7 items in same transaction...");
await queue.WriteAsync(logged, async (tr, state) =>
{
for (int i = 0; i < 7; i++)
{
var r = await state.PopAsync(tr);
Log($"- ({r.Value}, {r.HasValue})");
Assert.That(r.HasValue, Is.True);
Assert.That(r.Value, Is.EqualTo(i));
}
}, this.Cancellation);
#if DEBUG
await DumpSubspace(db, queue.Location);
#endif
bool empty = await queue.ReadAsync(logged, (tr, state) => state.EmptyAsync(tr), this.Cancellation);
Assert.That(empty, Is.False);
Log("Popping 3 + 1 items in another transaction...");
await queue.WriteAsync(logged, async (tr, state) =>
{
// should be able to pop 3 items..
var r = await state.PopAsync(tr);
Log($"- ({r.Value}, {r.HasValue})");
Assert.That(r.HasValue, Is.True);
Assert.That(r.Value, Is.EqualTo(7));
r = await state.PopAsync(tr);
Log($"- ({r.Value}, {r.HasValue})");
Assert.That(r.HasValue, Is.True);
Assert.That(r.Value, Is.EqualTo(8));
r = await state.PopAsync(tr);
Log($"- ({r.Value}, {r.HasValue})");
Assert.That(r.HasValue, Is.True);
Assert.That(r.Value, Is.EqualTo(9));
// queue should now be empty!
r = await state.PopAsync(tr);
Log($"- ({r.Value}, {r.HasValue})");
Assert.That(r.HasValue, Is.False);
Assert.That(r.Value, Is.EqualTo(0));
}, this.Cancellation);
empty = await queue.ReadAsync(logged, (tr, state) => state.EmptyAsync(tr), this.Cancellation);
Assert.That(empty, Is.True);
}
}
public async Task Test_Queue_Fast()
{
// without high contention protecction
using (var db = await OpenTestPartitionAsync())
{
var location = await GetCleanDirectory(db, "queue");
var queue = new FdbQueue <int>(location, highContention: false);
Log("Clear Queue");
await db.WriteAsync((tr) => queue.Clear(tr), this.Cancellation);
Log("Empty? " + await db.ReadAsync((tr) => queue.EmptyAsync(tr), this.Cancellation));
Log("Push 10, 8, 6");
await db.ReadWriteAsync((tr) => queue.PushAsync(tr, 10), this.Cancellation);
await db.ReadWriteAsync((tr) => queue.PushAsync(tr, 8), this.Cancellation);
await db.ReadWriteAsync((tr) => queue.PushAsync(tr, 6), this.Cancellation);
#if DEBUG
await DumpSubspace(db, location);
#endif
// Empty?
bool empty = await db.ReadAsync((tr) => queue.EmptyAsync(tr), this.Cancellation);
Log("Empty? " + empty);
Assert.That(empty, Is.False);
var item = await queue.PopAsync(db, this.Cancellation);
Log($"Pop item: {item}");
Assert.That(item.HasValue, Is.True);
Assert.That(item.Value, Is.EqualTo(10));
item = await db.ReadWriteAsync((tr) => queue.PeekAsync(tr), this.Cancellation);
Log($"Next item: {item}");
Assert.That(item.HasValue, Is.True);
Assert.That(item.Value, Is.EqualTo(8));
#if DEBUG
await DumpSubspace(db, location);
#endif
item = await queue.PopAsync(db, this.Cancellation);
Log($"Pop item: {item}");
Assert.That(item.HasValue, Is.True);
Assert.That(item.Value, Is.EqualTo(8));
#if DEBUG
await DumpSubspace(db, location);
#endif
item = await queue.PopAsync(db, this.Cancellation);
Log($"Pop item: {item}");
Assert.That(item.HasValue, Is.True);
Assert.That(item.Value, Is.EqualTo(6));
#if DEBUG
await DumpSubspace(db, location);
#endif
empty = await db.ReadAsync((tr) => queue.EmptyAsync(tr), this.Cancellation);
Log("Empty? " + empty);
Assert.That(empty, Is.True);
Log("Push 5");
await db.ReadWriteAsync((tr) => queue.PushAsync(tr, 5), this.Cancellation);
#if DEBUG
await DumpSubspace(db, location);
#endif
Log("Clear Queue");
await db.WriteAsync((tr) => queue.Clear(tr), this.Cancellation);
#if DEBUG
await DumpSubspace(db, location);
#endif
empty = await db.ReadAsync((tr) => queue.EmptyAsync(tr), this.Cancellation);
Log("Empty? " + empty);
Assert.That(empty, Is.True);
}
}
private static async Task RunMultiClientTest(IFdbDatabase db, KeySubspace location, bool highContention, string desc, int K, int NUM, CancellationToken ct)
{
Log("Starting {0} test with {1} threads and {2} iterations", desc, K, NUM);
var queue = new FdbQueue <string>(location, highContention);
await db.WriteAsync((tr) => queue.Clear(tr), ct);
// use a CTS to ensure that everything will stop in case of problems...
using (var go = new CancellationTokenSource(TimeSpan.FromSeconds(30)))
{
var tok = go.Token;
var pushLock = new AsyncCancelableMutex(tok);
var popLock = new AsyncCancelableMutex(tok);
int pushCount = 0;
int popCount = 0;
int stalls = 0;
var pushTreads = Enumerable.Range(0, K)
.Select(async id =>
{
try
{
// wait for the signal
await pushLock.Task.ConfigureAwait(false);
var res = new List <string>(NUM);
for (int i = 0; i < NUM; i++)
{
var item = id.ToString() + "." + i.ToString();
await db.ReadWriteAsync((tr) => queue.PushAsync(tr, item), tok).ConfigureAwait(false);
Interlocked.Increment(ref pushCount);
res.Add(item);
}
return(res);
}
catch (Exception e)
{
Log("PushThread[" + id + "] failed: " + e);
Assert.Fail("PushThread[" + id + "] failed: " + e.Message);
throw;
}
}).ToArray();
var popThreads = Enumerable.Range(0, K)
.Select(async id =>
{
try
{
// make everyone wait a bit, to ensure that they all start roughly at the same time
await popLock.Task.ConfigureAwait(false);
var res = new List <string>(NUM);
int i = 0;
while (i < NUM)
{
var item = await queue.PopAsync(db, tok).ConfigureAwait(false);
if (item.HasValue)
{
Interlocked.Increment(ref popCount);
res.Add(item.Value);
++i;
}
else
{
Interlocked.Increment(ref stalls);
await Task.Delay(10).ConfigureAwait(false);
}
}
return(res);
}
catch (Exception e)
{
Log("PopThread[" + id + "] failed: " + e);
Assert.Fail("PopThread[" + id + "] failed: " + e.Message);
throw;
}
}).ToArray();
var sw = Stopwatch.StartNew();
pushLock.Set(async: true);
await Task.Delay(100);
popLock.Set(async: true);
//using (var timer = new Timer((_) =>
//{
// var __ = TestHelpers.DumpSubspace(db, location);
//}, null, 1000, 4000))
{
await Task.WhenAll(pushTreads);
await Task.WhenAll(popThreads);
}
sw.Stop();
Log("> Finished {0} test in {1} seconds", desc, sw.Elapsed.TotalSeconds);
Log("> Pushed {0}, Popped {1} and Stalled {2}", pushCount, popCount, stalls);
var pushedItems = pushTreads.SelectMany(t => t.Result).ToList();
var poppedItems = popThreads.SelectMany(t => t.Result).ToList();
Assert.That(pushCount, Is.EqualTo(K * NUM));
Assert.That(popCount, Is.EqualTo(K * NUM));
// all pushed items should have been popped (with no duplicates)
Assert.That(poppedItems, Is.EquivalentTo(pushedItems));
// the queue should be empty
bool empty = await db.ReadAsync((tr) => queue.EmptyAsync(tr), ct);
Assert.That(empty, Is.True);
}
}
public async Task Test_Queue_Fast()
{
// without high contention protecction
using (var db = await OpenTestPartitionAsync())
{
var location = await GetCleanDirectory(db, "queue");
var queue = new FdbQueue<int>(location, highContention: false);
Console.WriteLine("Clear Queue");
await db.WriteAsync((tr) => queue.Clear(tr), this.Cancellation);
Console.WriteLine("Empty? " + await db.ReadAsync((tr) => queue.EmptyAsync(tr), this.Cancellation));
Console.WriteLine("Push 10, 8, 6");
await db.ReadWriteAsync((tr) => queue.PushAsync(tr, 10), this.Cancellation);
await db.ReadWriteAsync((tr) => queue.PushAsync(tr, 8), this.Cancellation);
await db.ReadWriteAsync((tr) => queue.PushAsync(tr, 6), this.Cancellation);
#if DEBUG
await DumpSubspace(db, location);
#endif
// Empty?
bool empty = await db.ReadAsync((tr) => queue.EmptyAsync(tr), this.Cancellation);
Console.WriteLine("Empty? " + empty);
Assert.That(empty, Is.False);
Optional<int> item = await queue.PopAsync(db, this.Cancellation);
Console.WriteLine("Pop item: " + item);
Assert.That((int)item, Is.EqualTo(10));
item = await db.ReadWriteAsync((tr) => queue.PeekAsync(tr), this.Cancellation);
Console.WriteLine("Next item: " + item);
Assert.That((int)item, Is.EqualTo(8));
#if DEBUG
await DumpSubspace(db, location);
#endif
item = await queue.PopAsync(db, this.Cancellation);
Console.WriteLine("Pop item: " + item);
Assert.That((int)item, Is.EqualTo(8));
#if DEBUG
await DumpSubspace(db, location);
#endif
item = await queue.PopAsync(db, this.Cancellation);
Console.WriteLine("Pop item: " + item);
Assert.That((int)item, Is.EqualTo(6));
#if DEBUG
await DumpSubspace(db, location);
#endif
empty = await db.ReadAsync((tr) => queue.EmptyAsync(tr), this.Cancellation);
Console.WriteLine("Empty? " + empty);
Assert.That(empty, Is.True);
Console.WriteLine("Push 5");
await db.ReadWriteAsync((tr) => queue.PushAsync(tr, 5), this.Cancellation);
#if DEBUG
await DumpSubspace(db, location);
#endif
Console.WriteLine("Clear Queue");
await db.WriteAsync((tr) => queue.Clear(tr), this.Cancellation);
#if DEBUG
await DumpSubspace(db, location);
#endif
empty = await db.ReadAsync((tr) => queue.EmptyAsync(tr), this.Cancellation);
Console.WriteLine("Empty? " + empty);
Assert.That(empty, Is.True);
}
}
private static async Task RunMultiClientTest(IFdbDatabase db, FdbSubspace location, bool highContention, string desc, int K, int NUM, CancellationToken ct)
{
Console.WriteLine("Starting {0} test with {1} threads and {2} iterations", desc, K, NUM);
var queue = new FdbQueue<string>(location, highContention);
await db.WriteAsync((tr) => queue.Clear(tr), ct);
// use a CTS to ensure that everything will stop in case of problems...
using (var go = new CancellationTokenSource(TimeSpan.FromSeconds(30)))
{
var tok = go.Token;
var pushLock = new AsyncCancelableMutex(tok);
var popLock = new AsyncCancelableMutex(tok);
int pushCount = 0;
int popCount = 0;
int stalls = 0;
var pushTreads = Enumerable.Range(0, K)
.Select(async id =>
{
// wait for the signal
await pushLock.Task.ConfigureAwait(false);
var res = new List<string>(NUM);
for (int i = 0; i < NUM; i++)
{
var item = id.ToString() + "." + i.ToString();
await db.ReadWriteAsync((tr) => queue.PushAsync(tr, item), tok).ConfigureAwait(false);
Interlocked.Increment(ref pushCount);
res.Add(item);
}
return res;
}).ToArray();
var popThreads = Enumerable.Range(0, K)
.Select(async id =>
{
// make everyone wait a bit, to ensure that they all start roughly at the same time
await popLock.Task.ConfigureAwait(false);
var res = new List<string>(NUM);
int i = 0;
while (i < NUM)
{
var item = await queue.PopAsync(db, tok).ConfigureAwait(false);
if (item.HasValue)
{
Interlocked.Increment(ref popCount);
res.Add(item.Value);
++i;
}
else
{
Interlocked.Increment(ref stalls);
await Task.Delay(10).ConfigureAwait(false);
}
}
return res;
}).ToArray();
var sw = Stopwatch.StartNew();
pushLock.Set(async: true);
await Task.Delay(100);
popLock.Set(async: true);
//using (var timer = new Timer((_) =>
//{
// var __ = TestHelpers.DumpSubspace(db, location);
//}, null, 1000, 4000))
{
await Task.WhenAll(pushTreads);
await Task.WhenAll(popThreads);
}
sw.Stop();
Console.WriteLine("> Finished {0} test in {1} seconds", desc, sw.Elapsed.TotalSeconds);
Console.WriteLine("> Pushed {0}, Popped {1} and Stalled {2}", pushCount, popCount, stalls);
var pushedItems = pushTreads.SelectMany(t => t.Result).ToList();
var poppedItems = popThreads.SelectMany(t => t.Result).ToList();
Assert.That(pushCount, Is.EqualTo(K * NUM));
Assert.That(popCount, Is.EqualTo(K * NUM));
// all pushed items should have been popped (with no duplicates)
Assert.That(poppedItems, Is.EquivalentTo(pushedItems));
// the queue should be empty
bool empty = await db.ReadAsync((tr) => queue.EmptyAsync(tr), ct);
Assert.That(empty, Is.True);
}
}
public async Task Test_Single_Client()
{
using (var db = await OpenTestPartitionAsync())
{
var location = await GetCleanDirectory(db, "queue");
var queue = new FdbQueue<int>(location, highContention: false);
await db.WriteAsync((tr) => queue.Clear(tr), this.Cancellation);
for (int i = 0; i < 10; i++)
{
await db.ReadWriteAsync((tr) => queue.PushAsync(tr, i), this.Cancellation);
}
for (int i = 0; i < 10; i++)
{
var r = await queue.PopAsync(db, this.Cancellation);
Assert.That(r.HasValue, Is.True);
Assert.That(r.Value, Is.EqualTo(i));
}
bool empty = await db.ReadAsync((tr) => queue.EmptyAsync(tr), this.Cancellation);
Assert.That(empty, Is.True);
}
}
