public async Task Test_Log_Queue()
{
int NUM = 100;
using (var db = await OpenTestPartitionAsync())
{
var location = await GetCleanDirectory(db, "queue");
#if ENABLE_LOGGING
var list = new List <FdbTransactionLog>(NUM);
var logged = new FdbLoggedDatabase(db, false, false, (tr) => { lock (list) { list.Add(tr.Log); } });
#else
var logged = db;
#endif
await RunMultiClientTest(logged, location, false, "simple queue", 4, NUM, this.Cancellation);
#if ENABLE_LOGGING
foreach (var log in list)
{
Log(log.GetTimingsReport(true));
}
list.Clear();
#endif
Log("------------------------------------------------");
await RunMultiClientTest(logged, location, true, "high contention queue", 4, NUM, this.Cancellation);
#if ENABLE_LOGGING
foreach (var log in list)
{
Log(log.GetTimingsReport(true));
}
list.Clear();
#endif
Log("------------------------------------------------");
}
}
private async Task RunAsync(IFdbDatabase db, FdbSubspace location, CancellationToken ct, Action done, int N, int K, int W)
{
if (db == null)
{
throw new ArgumentNullException("db");
}
StringBuilder sb = new StringBuilder();
db = new FdbLoggedDatabase(db, false, false, (log) =>
{
sb.AppendLine(log.Log.GetTimingsReport(true));
//Console.WriteLine(log.Log.GetTimingsReport(true));
});
try
{
var workerPool = new FdbWorkerPool(location);
Console.WriteLine("workerPool at " + location.Key.ToAsciiOrHexaString());
var workerSignal = new AsyncCancelableMutex(ct);
var clientSignal = new AsyncCancelableMutex(ct);
int taskCounter = 0;
int msgSent = 0;
int msgReceived = 0;
Func <FdbWorkerMessage, CancellationToken, Task> handler = async(msg, _ct) =>
{
Interlocked.Increment(ref msgReceived);
//await Task.Delay(10 + Math.Abs(msg.Id.GetHashCode()) % 50);
await Task.Delay(10).ConfigureAwait(false);
};
Func <int, Task> worker = async(id) =>
{
await workerSignal.Task.ConfigureAwait(false);
Console.WriteLine("Worker #" + id + " is starting");
try
{
await workerPool.RunWorkerAsync(db, handler, ct).ConfigureAwait(false);
}
finally
{
Console.WriteLine("Worker #" + id + " has stopped");
}
};
Func <int, Task> client = async(id) =>
{
await clientSignal.Task.ConfigureAwait(false);
await Task.Delay(10).ConfigureAwait(false);
var rnd = new Random(id * 111);
for (int i = 0; i < N; i++)
{
var taskId = Slice.FromString("T" + Interlocked.Increment(ref taskCounter));
var taskBody = Slice.FromString("Message " + (i + 1) + " of " + N + " from client #" + id);
await workerPool.ScheduleTaskAsync(db, taskId, taskBody, ct).ConfigureAwait(false);
Interlocked.Increment(ref msgSent);
//if (i > 0 && i % 10 == 0) Console.WriteLine("@@@ Client#" + id + " pushed " + (i + 1) + " / " + N + " messages");
switch (rnd.Next(5))
{
case 0: await Task.Delay(10).ConfigureAwait(false); break;
case 1: await Task.Delay(100).ConfigureAwait(false); break;
case 2: await Task.Delay(500).ConfigureAwait(false); break;
}
}
Console.WriteLine("@@@ Client#" + id + " has finished!");
};
Func <string, Task> dump = async(label) =>
{
Console.WriteLine("<dump label='" + label + "' key='" + location.Key.ToAsciiOrHexaString() + "'>");
using (var tr = db.BeginTransaction(ct))
{
await tr.Snapshot
.GetRange(FdbKeyRange.StartsWith(location.Key))
.ForEachAsync((kvp) =>
{
Console.WriteLine(" - " + FdbTuple.Unpack(location.Extract(kvp.Key)) + " = " + kvp.Value.ToAsciiOrHexaString());
}).ConfigureAwait(false);
}
Console.WriteLine("</dump>");
};
var workers = Enumerable.Range(0, W).Select((i) => worker(i)).ToArray();
var clients = Enumerable.Range(0, K).Select((i) => client(i)).ToArray();
DateTime start = DateTime.Now;
DateTime last = start;
int lastHandled = -1;
using (var timer = new Timer((_) =>
{
var now = DateTime.Now;
Console.WriteLine("@@@ T=" + now.Subtract(start) + ", sent: " + msgSent.ToString("N0") + ", recv: " + msgReceived.ToString("N0"));
Console.WriteLine("### Workers: " + workerPool.IdleWorkers + " / " + workerPool.ActiveWorkers + " (" + new string('#', workerPool.IdleWorkers) + new string('.', workerPool.ActiveWorkers - workerPool.IdleWorkers) + "), sent: " + workerPool.MessageScheduled.ToString("N0") + ", recv: " + workerPool.MessageReceived.ToString("N0") + ", delta: " + (workerPool.MessageScheduled - workerPool.MessageReceived).ToString("N0") + ", busy: " + workerPool.WorkerBusyTime + " (avg " + workerPool.WorkerAverageBusyDuration.TotalMilliseconds.ToString("N3") + " ms)");
if (now.Subtract(last).TotalSeconds >= 10)
{
//dump("timer").GetAwaiter().GetResult();
last = now;
if (lastHandled == msgReceived)
{ // STALL ?
Console.WriteLine("STALL! ");
done();
}
lastHandled = msgReceived;
}
if (msgReceived >= K * N)
{
dump("complete").GetAwaiter().GetResult();
done();
}
}, null, 1000, 1000))
{
var sw = Stopwatch.StartNew();
// start the workers
workerSignal.Set(async: true);
await Task.Delay(500);
await dump("workers started");
// start the clients
clientSignal.Set(async: true);
await Task.WhenAll(clients);
Console.WriteLine("Clients completed after " + sw.Elapsed);
await Task.WhenAll(workers);
Console.WriteLine("Workers completed after " + sw.Elapsed);
}
}
finally
{
Console.WriteLine("---------------------------------------------------------------------------");
Console.WriteLine("Transaction logs:");
Console.WriteLine();
Console.WriteLine(sb.ToString());
}
}
public async Task Test_Can_Log_A_Transaction()
{
const int N = 10;
using (var db = await OpenTestPartitionAsync())
{
// get a tuple view of the directory
var location = (await GetCleanDirectory(db, "Logging")).Keys;
// note: ensure that all methods are JITed
await db.ReadWriteAsync(async (tr) =>
{
await tr.GetReadVersionAsync();
tr.Set(location.Encode("Warmup", 0), Slice.FromInt32(1));
tr.Clear(location.Encode("Warmup", 1));
await tr.GetAsync(location.Encode("Warmup", 2));
await tr.GetRange(KeyRange.StartsWith(location.Encode("Warmup", 3))).ToListAsync();
tr.ClearRange(location.Encode("Warmup", 4), location.Encode("Warmup", 5));
}, this.Cancellation);
await db.WriteAsync((tr) =>
{
var rnd = new Random();
tr.Set(location.Encode("One"), Slice.FromString("111111"));
tr.Set(location.Encode("Two"), Slice.FromString("222222"));
for (int j = 0; j < 4; j++)
{
for (int i = 0; i < 100; i++)
{
tr.Set(location.Encode("Range", j, rnd.Next(1000)), Slice.Empty);
}
}
for (int j = 0; j < N; j++)
{
tr.Set(location.Encode("X", j), Slice.FromInt32(j));
tr.Set(location.Encode("Y", j), Slice.FromInt32(j));
tr.Set(location.Encode("Z", j), Slice.FromInt32(j));
tr.Set(location.Encode("W", j), Slice.FromInt32(j));
}
}, this.Cancellation);
bool first = true;
Action <FdbLoggedTransaction> logHandler = (tr) =>
{
if (first)
{
Log(tr.Log.GetCommandsReport());
first = false;
}
Log(tr.Log.GetTimingsReport(true));
};
// create a logged version of the database
var logged = new FdbLoggedDatabase(db, false, false, logHandler);
for (int k = 0; k < N; k++)
{
Log("==== " + k + " ==== ");
Log();
await logged.ReadWriteAsync(async (tr) =>
{
Assert.That(tr, Is.InstanceOf <FdbLoggedTransaction>());
//tr.SetOption(FdbTransactionOption.CausalReadRisky);
long ver = await tr.GetReadVersionAsync().ConfigureAwait(false);
await tr.GetAsync(location.Encode("One")).ConfigureAwait(false);
await tr.GetAsync(location.Encode("NotFound")).ConfigureAwait(false);
tr.Set(location.Encode("Write"), Slice.FromString("abcdef" + k.ToString()));
//tr.Annotate("BEFORE");
//await Task.Delay(TimeSpan.FromMilliseconds(10));
//tr.Annotate("AFTER");
//await tr.Snapshot.GetAsync(location.Pack("Snap")).ConfigureAwait(false);
tr.Annotate("This is a comment");
//await tr.GetRangeAsync(KeySelector.LastLessOrEqual(location.Pack("A")), KeySelector.FirstGreaterThan(location.Pack("Z"))).ConfigureAwait(false);
await Task.WhenAll(
tr.GetRange(KeyRange.StartsWith(location.Encode("Range", 0))).ToListAsync(),
tr.GetRange(location.Encode("Range", 1, 0), location.Encode("Range", 1, 200)).ToListAsync(),
tr.GetRange(location.Encode("Range", 2, 400), location.Encode("Range", 2, 600)).ToListAsync(),
tr.GetRange(location.Encode("Range", 3, 800), location.Encode("Range", 3, 1000)).ToListAsync()
).ConfigureAwait(false);
await tr.GetAsync(location.Encode("Two")).ConfigureAwait(false);
await tr.GetValuesAsync(Enumerable.Range(0, N).Select(x => location.Encode("X", x))).ConfigureAwait(false);
for (int i = 0; i < N; i++)
{
await tr.GetAsync(location.Encode("Z", i)).ConfigureAwait(false);
}
await Task.WhenAll(Enumerable.Range(0, N / 2).Select(x => tr.GetAsync(location.Encode("Y", x)))).ConfigureAwait(false);
await Task.WhenAll(Enumerable.Range(N / 2, N / 2).Select(x => tr.GetAsync(location.Encode("Y", x)))).ConfigureAwait(false);
await Task.WhenAll(
tr.GetAsync(location.Encode("W", 1)),
tr.GetAsync(location.Encode("W", 2)),
tr.GetAsync(location.Encode("W", 3))
).ConfigureAwait(false);
tr.Set(location.Encode("Write2"), Slice.FromString("ghijkl" + k.ToString()));
tr.Clear(location.Encode("Clear", "0"));
tr.ClearRange(location.Encode("Clear", "A"), location.Encode("Clear", "Z"));
if (tr.Context.Retries == 0)
{
// make it fail
//throw new FdbException(FdbError.PastVersion, "fake timeout");
}
}, this.Cancellation);
}
}
}
public async Task Test_Can_Log_A_Transaction()
{
const int N = 10;
using (var db = await OpenTestPartitionAsync())
{
var location = await GetCleanDirectory(db, "Logging");
// note: ensure that all methods are JITed
await db.ReadWriteAsync(async (tr) =>
{
await tr.GetReadVersionAsync();
tr.Set(location.Pack("Warmup", 0), Slice.FromInt32(1));
tr.Clear(location.Pack("Warmup", 1));
await tr.GetAsync(location.Pack("Warmup", 2));
await tr.GetRange(FdbKeyRange.StartsWith(location.Pack("Warmup", 3))).ToListAsync();
tr.ClearRange(location.Pack("Warmup", 4), location.Pack("Warmup", 5));
}, this.Cancellation);
await db.WriteAsync((tr) =>
{
var rnd = new Random();
tr.Set(location.Pack("One"), Slice.FromString("111111"));
tr.Set(location.Pack("Two"), Slice.FromString("222222"));
for (int j = 0; j < 4; j++)
{
for (int i = 0; i < 100; i++)
{
tr.Set(location.Pack("Range", j, rnd.Next(1000)), Slice.Empty);
}
}
for (int j = 0; j < N; j++)
{
tr.Set(location.Pack("X", j), Slice.FromInt32(j));
tr.Set(location.Pack("Y", j), Slice.FromInt32(j));
tr.Set(location.Pack("Z", j), Slice.FromInt32(j));
tr.Set(location.Pack("W", j), Slice.FromInt32(j));
}
}, this.Cancellation);
bool first = true;
Action<FdbLoggedTransaction> logHandler = (tr) =>
{
if (first)
{
Console.WriteLine(tr.Log.GetCommandsReport());
first = false;
}
Console.WriteLine(tr.Log.GetTimingsReport(true));
};
// create a logged version of the database
var logged = new FdbLoggedDatabase(db, false, false, logHandler);
for (int k = 0; k < N; k++)
{
Console.WriteLine("==== " + k + " ==== ");
Console.WriteLine();
await logged.ReadWriteAsync(async (tr) =>
{
Assert.That(tr, Is.InstanceOf<FdbLoggedTransaction>());
//tr.SetOption(FdbTransactionOption.CausalReadRisky);
long ver = await tr.GetReadVersionAsync().ConfigureAwait(false);
await tr.GetAsync(location.Pack("One")).ConfigureAwait(false);
await tr.GetAsync(location.Pack("NotFound")).ConfigureAwait(false);
tr.Set(location.Pack("Write"), Slice.FromString("abcdef" + k.ToString()));
//tr.Annotate("BEFORE");
//await Task.Delay(TimeSpan.FromMilliseconds(10));
//tr.Annotate("AFTER");
//await tr.Snapshot.GetAsync(location.Pack("Snap")).ConfigureAwait(false);
tr.Annotate("This is a comment");
//await tr.GetRangeAsync(FdbKeySelector.LastLessOrEqual(location.Pack("A")), FdbKeySelector.FirstGreaterThan(location.Pack("Z"))).ConfigureAwait(false);
await Task.WhenAll(
tr.GetRange(FdbKeyRange.StartsWith(location.Pack("Range", 0))).ToListAsync(),
tr.GetRange(location.Pack("Range", 1, 0), location.Pack("Range", 1, 200)).ToListAsync(),
tr.GetRange(location.Pack("Range", 2, 400), location.Pack("Range", 2, 600)).ToListAsync(),
tr.GetRange(location.Pack("Range", 3, 800), location.Pack("Range", 3, 1000)).ToListAsync()
).ConfigureAwait(false);
await tr.GetAsync(location.Pack("Two")).ConfigureAwait(false);
await tr.GetValuesAsync(Enumerable.Range(0, N).Select(x => location.Pack("X", x))).ConfigureAwait(false);
for (int i = 0; i < N; i++)
{
await tr.GetAsync(location.Pack("Z", i)).ConfigureAwait(false);
}
await Task.WhenAll(Enumerable.Range(0, N / 2).Select(x => tr.GetAsync(location.Pack("Y", x)))).ConfigureAwait(false);
await Task.WhenAll(Enumerable.Range(N / 2, N / 2).Select(x => tr.GetAsync(location.Pack("Y", x)))).ConfigureAwait(false);
await Task.WhenAll(
tr.GetAsync(location.Pack("W", 1)),
tr.GetAsync(location.Pack("W", 2)),
tr.GetAsync(location.Pack("W", 3))
).ConfigureAwait(false);
tr.Set(location.Pack("Write2"), Slice.FromString("ghijkl" + k.ToString()));
tr.Clear(location.Pack("Clear", "0"));
tr.ClearRange(location.Pack("Clear", "A"), location.Pack("Clear", "Z"));
if (tr.Context.Retries == 0)
{
// make it fail
//throw new FdbException(FdbError.PastVersion, "fake timeout");
}
}, this.Cancellation);
}
}
}
