/// <summary>
/// Setup the initial state of the database
/// </summary>
public async Task Init(IFdbDatabase db, CancellationToken ct)
{
// open the folder where we will store everything
this.Subspace = await db.ReadWriteAsync(tr => db.Directory.CreateOrOpenAsync(tr, new [] { "Samples", "MessageQueueTest" }), ct);
this.WorkerPool = new FdbWorkerPool(this.Subspace);
}
private static async Task BurnerThread(IFdbDatabase db, CancellationToken ct)
{
var folder = await db.ReadWriteAsync(async tr =>
{
var x = await db.Root["Benchmarks"]["Burner"]["Sequential"].CreateOrOpenAsync(tr);
tr.ClearRange(x);
return(x);
}, ct);
long pos = 0;
Random rnd;
lock (Rnd)
{
rnd = new Random(Rnd.Next());
}
using (var tr = await db.BeginTransactionAsync(ct))
{
while (!ct.IsCancellationRequested)
{
FdbException error = null;
try
{
tr.Reset();
for (int i = 0; i < N; i++)
{
long x = Randomized
? rnd.Next()
: pos + i;
tr.Set(folder.Encode(x, Suffix), Value);
Interlocked.Increment(ref Keys);
}
pos += N;
await tr.CommitAsync();
Interlocked.Increment(ref Transactions);
Interlocked.Add(ref Bytes, tr.Size);
}
catch (FdbException e)
{
error = e;
}
if (error != null && !ct.IsCancellationRequested)
{
await tr.OnErrorAsync(error.Code);
}
}
}
}
public static async Task Clear(string[] path, IVarTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
if (path == null || path.Length == 0)
{
Program.Error(log, "Cannot directory list the content of Root Partition.");
return;
}
var folder = await db.Directory.TryOpenAsync(db, path, ct : ct);
if (folder == null)
{
Program.Error(log, "The directory does not exist anymore");
return;
}
if (folder.Layer == FdbDirectoryPartition.LayerId)
{
Program.Error(log, "Cannot clear the content of a Directory Partition!");
return;
}
if (extras.Count == 0)
{
Program.Error(log, "You must specify a key of range of keys!");
return;
}
object key = extras[0];
Slice k = MakeKey(folder, key);
bool empty = await db.ReadWriteAsync(async tr =>
{
var v = await tr.GetAsync(k);
if (v.IsNullOrEmpty)
{
return(true);
}
tr.Clear(k);
return(false);
}, ct);
if (empty)
{
Program.StdOut(log, "Key did not exist in the database.", ConsoleColor.Cyan);
}
else
{
Program.Success(log, $"Key {key} has been cleared from Directory {String.Join("/", path)}.");
}
}
public static async Task ChangeDirectoryLayer(string[] path, string layer, IVarTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
var dir = await BasicCommands.TryOpenCurrentDirectoryAsync(path, db, ct);
if (dir == null)
{
Program.Error(log, $"# Directory {String.Join("/", path)} does not exist anymore");
}
else
{
dir = await db.ReadWriteAsync((tr) => dir.ChangeLayerAsync(tr, Slice.FromString(layer)), ct);
Program.Success(log, $"# Directory {String.Join("/", path)} layer changed to {dir.Layer:P}");
}
}
public static async Task ChangeDirectoryLayer(string[] path, string layer, IFdbTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
var dir = await BasicCommands.TryOpenCurrentDirectoryAsync(path, db, ct);
if (dir == null)
{
log.WriteLine("# Directory {0} does not exist anymore", String.Join("/", path));
}
else
{
dir = await db.ReadWriteAsync((tr) => dir.ChangeLayerAsync(tr, Slice.FromString(layer)), ct);
log.WriteLine("# Directory {0} layer changed to {1}", String.Join("/", path), dir.Layer.ToAsciiOrHexaString());
}
}
/// <summary>Remove a directory and all its data</summary>
public static async Task RemoveDirectory(FdbPath path, IVarTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
if (log == null)
{
log = Console.Out;
}
// "-r|--recursive" is used to allow removing an entire sub-tree
string[] args = extras.ToArray <string>();
bool recursive = args.Contains("-r", StringComparer.Ordinal) || args.Contains("--recursive", StringComparer.Ordinal);
bool force = args.Contains("-f", StringComparer.Ordinal) || args.Contains("--force", StringComparer.Ordinal);
var res = await db.ReadWriteAsync(async tr =>
{
var folder = await db.DirectoryLayer.TryOpenAsync(tr, path);
if (folder == null)
{
Program.Error(log, $"# Directory {path} does not exist");
return(false);
}
// are there any subdirectories ?
if (!recursive)
{
var subDirs = await folder.TryListAsync(tr);
if (subDirs != null && subDirs.Count > 0)
{
//TODO: "-r" flag ?
Program.Error(log, $"# Cannot remove {path} because it still contains {subDirs.Count:N0} sub-directories.");
Program.StdOut(log, "Use the -r|--recursive flag to override this warning.");
return(false);
}
}
if (!force)
{
//TODO: ask for confirmation?
}
await folder.RemoveAsync(tr);
return(true);
}, ct);
if (res)
{
Program.Success(log, $"Deleted directory {path}");
}
}
/// <summary>
/// Setup the initial state of the database
/// </summary>
public async Task Init(IFdbDatabase db, CancellationToken ct)
{
this.Subspace = await db.ReadWriteAsync(async tr =>
{
// open the folder where we will store everything
var subspace = await db.Root["Benchmarks"]["LeakTest"].CreateOrOpenAsync(tr);
// clear all previous values
await db.ClearRangeAsync(subspace, ct);
// insert all the classes
tr.Set(subspace.GetPrefix() + FdbKey.MinValue, Slice.FromString("BEGIN"));
tr.Set(subspace.GetPrefix() + FdbKey.MaxValue, Slice.FromString("END"));
return(subspace);
}, ct);
}
/// <summary>
/// Setup the initial state of the database
/// </summary>
public async Task Init(IFdbDatabase db, CancellationToken ct)
{
// open the folder where we will store everything
//this.Subspace = db.GlobalSpace;
var subspace = db.GlobalSpace;
await db.ReadWriteAsync(tr =>
{
tr.ClearRange(subspace);
// insert all the classes
foreach (var c in this.ClassNames)
{
tr.Set(ClassKey(c, subspace), Slice.FromStringAscii("100"));
}
Console.WriteLine("inserted range..");
return(subspace);
}, ct);
/*
* this.Subspace = await db.ReadWriteAsync(async tr =>
* {
* var subspace = db.GlobalSpace;
*
* // clear all previous values
* tr.ClearRange(subspace);
*
* // insert all the classes
* foreach (var c in this.ClassNames)
* {
* tr.Set(ClassKey(c), Slice.FromString("100"));
* }
*
* return subspace;
* }, ct);
*
*/
}
/// <summary>
/// Setup the initial state of the database
/// </summary>
public async Task Init(IFdbDatabase db, CancellationToken ct)
{
// open the folder where we will store everything
this.Subspace = await db.ReadWriteAsync(async tr =>
{
var subspace = await db.Directory.CreateOrOpenAsync(tr, new[] { "Tutorials", "ClassScheduling" });
// clear all previous values
tr.ClearRange(subspace);
// insert all the classes
foreach (var c in this.ClassNames)
{
tr.Set(ClassKey(c), Slice.FromStringAscii("100"));
}
return(subspace);
}, ct);
}
public static async Task <FdbDirectorySubspace> GetCleanDirectory([NotNull] IFdbDatabase db, [NotNull] string[] path, CancellationToken ct)
{
Assert.That(db, Is.Not.Null, "null db");
Assert.That(path, Is.Not.Null.And.Length.GreaterThan(0), "invalid path");
// do not log
db = db.WithoutLogging();
var subspace = await db.ReadWriteAsync(async tr =>
{
// remove previous
await db.Directory.TryRemoveAsync(tr, path);
// create new
return(await db.Directory.CreateAsync(tr, path));
}, ct);
Assert.That(subspace, Is.Not.Null);
Assert.That(db.GlobalSpace.Contains(subspace.GetPrefix()), Is.True);
return(subspace);
}
/// <summary>Pop the next item from the queue. Cannot be composed with other functions in a single transaction.</summary>
public Task <Optional <T> > PopAsync([NotNull] IFdbDatabase db, CancellationToken cancellationToken)
{
if (db == null)
{
throw new ArgumentNullException("db");
}
if (cancellationToken.IsCancellationRequested)
{
return(TaskHelpers.FromCancellation <Optional <T> >(cancellationToken));
}
if (this.HighContention)
{
return(PopHighContentionAsync(db, cancellationToken));
}
else
{
return(db.ReadWriteAsync((tr) => this.PopSimpleAsync(tr), cancellationToken));
}
}
/// <summary>Pop the next item from the queue. Cannot be composed with other functions in a single transaction.</summary>
public Task <(T Value, bool HasValue)> PopAsync([NotNull] IFdbDatabase db, CancellationToken ct)
{
if (db == null)
{
throw new ArgumentNullException(nameof(db));
}
if (ct.IsCancellationRequested)
{
return(Task.FromCanceled <(T, bool)>(ct));
}
if (this.HighContention)
{
return(PopHighContentionAsync(db, ct));
}
else
{
return(db.ReadWriteAsync((tr) => PopSimpleAsync(tr), ct));
}
}
/// <summary>Move/Rename a directory</summary>
public static async Task MoveDirectory(string[] srcPath, string[] dstPath, IVarTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
await db.ReadWriteAsync(async tr =>
{
var folder = await db.Directory.TryOpenAsync(tr, srcPath);
if (folder == null)
{
Program.Error(log, $"# Source directory /{string.Join("/", srcPath)} does not exist!");
return;
}
folder = await db.Directory.TryOpenAsync(tr, dstPath);
if (folder != null)
{
Program.Error(log, $"# Destination directory /{string.Join("/", dstPath)} already exists!");
return;
}
await db.Directory.MoveAsync(tr, srcPath, dstPath);
}, ct);
Program.Success(log, $"Moved /{string.Join("/", srcPath)} to {string.Join("/", dstPath)}");
}
private Task <Slice> AddConflictedPopAsync([NotNull] IFdbDatabase db, bool forced, CancellationToken ct)
{
return(db.ReadWriteAsync((tr) => AddConflictedPopAsync(tr, forced), ct));
}
/// <summary>
/// Simulate a student that is really indecisive
/// </summary>
public async Task IndecisiveStudent(IFdbDatabase db, int id, int ops, CancellationToken ct)
{
string student = "s" + id.ToString("D04");
var allClasses = new List <string>(this.ClassNames);
var myClasses = new List <string>();
var rnd = new Random(id * 7);
for (int i = 0; i < ops && !ct.IsCancellationRequested; i++)
{
int classCount = myClasses.Count;
var moods = new List <string>();
if (classCount > 0)
{
moods.AddRange(new[] { "drop", "switch" });
}
if (classCount < 5)
{
moods.Add("add");
}
string mood = moods[rnd.Next(moods.Count)];
try
{
if (allClasses == null)
{
allClasses = await db.ReadAsync((tr) => AvailableClasses(tr), ct);
}
switch (mood)
{
case "add":
{
string @class = allClasses[rnd.Next(allClasses.Count)];
await db.ReadWriteAsync((tr) => Signup(tr, student, @class), ct);
myClasses.Add(@class);
break;
}
case "drop":
{
string @class = allClasses[rnd.Next(allClasses.Count)];
await db.ReadWriteAsync((tr) => Drop(tr, student, @class), ct);
myClasses.Remove(@class);
break;
}
case "switch":
{
string oldClass = allClasses[rnd.Next(allClasses.Count)];
string newClass = allClasses[rnd.Next(allClasses.Count)];
await db.ReadWriteAsync((tr) => Switch(tr, student, oldClass, newClass), ct);
myClasses.Remove(oldClass);
myClasses.Add(newClass);
break;
}
default:
{
throw new InvalidOperationException("Ooops");
}
}
}
catch (Exception e)
{
if (e is TaskCanceledException || e is OperationCanceledException)
{
throw;
}
allClasses = null;
}
}
ct.ThrowIfCancellationRequested();
}
/// <summary>Run the worker loop</summary>
public async Task RunWorkerAsync(IFdbDatabase db, Func<FdbWorkerMessage, CancellationToken, Task> handler, CancellationToken ct)
{
int num = Interlocked.Increment(ref counter);
Slice workerId = Slice.Nil;
Slice previousTaskId = Slice.Nil;
FdbWatch watch = default(FdbWatch);
FdbWorkerMessage msg = null;
Interlocked.Increment(ref m_workers);
try
{
while (true)
{
//TODO: how do we clear the previousTaskId from the db in case of cancellation ?
ct.ThrowIfCancellationRequested();
Slice myId = Slice.Nil;
await db.ReadWriteAsync(
async (tr) =>
{
tr.Annotate("I'm worker #{0} with id {1}", num, workerId.ToAsciiOrHexaString());
myId = workerId;
watch = default(FdbWatch);
msg = new FdbWorkerMessage();
if (previousTaskId != null)
{ // we need to clean up the previous task
ClearTask(tr, previousTaskId);
}
else if (myId.IsPresent)
{ // look for an already assigned task
tr.Annotate("Look for already assigned task");
msg.Id = await tr.GetAsync(this.BusyRing.Pack(myId)).ConfigureAwait(false);
}
if (!msg.Id.IsPresent)
{ // We aren't already assigned a task, so get an item from a random queue
tr.Annotate("Look for next queued item");
// Find the next task on the queue
var item = await tr.GetRange(this.UnassignedTaskRing.ToRange()).FirstOrDefaultAsync().ConfigureAwait(false);
if (item.Key != null)
{ // pop the Task from the queue
msg.Id = item.Value;
tr.Clear(item.Key);
}
if (msg.Id.IsPresent)
{ // mark this worker as busy
// note: we need a random id so generate one if it is the first time...
if (!myId.IsPresent) myId = GetRandomId();
tr.Annotate("Found {0}, switch to busy with id {1}", msg.Id.ToAsciiOrHexaString(), myId.ToAsciiOrHexaString());
tr.Set(this.BusyRing.Pack(myId), msg.Id);
this.Counters.Increment(tr, COUNTER_BUSY);
}
else if (myId.IsPresent)
{ // remove ourselves from the busy ring
tr.Annotate("Found nothing, switch to idle with id {0}", myId.ToAsciiOrHexaString());
//tr.Clear(this.BusyRing.Pack(myId));
}
}
if (msg.Id.IsPresent)
{ // get the task body
tr.Annotate("Fetching body for task {0}", msg.Id.ToAsciiOrHexaString());
var prefix = this.TaskStore.Partition(msg.Id);
//TODO: replace this with a get_range ?
var data = await tr.GetValuesAsync(new [] {
prefix.Key,
prefix.Pack(TASK_META_SCHEDULED)
}).ConfigureAwait(false);
msg.Body = data[0];
msg.Scheduled = new DateTime(data[1].ToInt64(), DateTimeKind.Utc);
msg.Received = DateTime.UtcNow;
}
else
{ // There are no unassigned task, so enter the idle_worker_ring and wait for a task to be asssigned to us
// remove us from the busy ring
if (myId.IsPresent)
{
tr.Clear(this.BusyRing.Pack(myId));
this.Counters.Decrement(tr, COUNTER_BUSY);
}
// choose a new random position on the idle ring
myId = GetRandomId();
// the idle key will also be used as the watch key to wake us up
var watchKey = this.IdleRing.Pack(myId);
tr.Annotate("Will start watching on key {0} with id {1}", watchKey.ToAsciiOrHexaString(), myId.ToAsciiOrHexaString());
tr.Set(watchKey, Slice.Empty);
this.Counters.Increment(tr, COUNTER_IDLE);
watch = tr.Watch(watchKey, ct);
}
},
onDone: (tr) =>
{ // we have successfully acquired some work, or got a watch
previousTaskId = Slice.Nil;
workerId = myId;
},
cancellationToken: ct
).ConfigureAwait(false);
if (msg.Id.IsNullOrEmpty)
{ // wait for someone to wake us up...
Interlocked.Increment(ref m_idleWorkers);
try
{
await watch.Task;
//Console.WriteLine("Worker #" + num + " woken up!");
}
finally
{
Interlocked.Decrement(ref m_idleWorkers);
}
}
else
{
//Console.WriteLine("Got task " + taskId);
previousTaskId = msg.Id;
if (msg.Body.IsNull)
{ // the task has been dropped?
// TODO: loggin?
Console.WriteLine("[####] Task[" + msg.Id.ToAsciiOrHexaString() + "] has vanished?");
}
else
{
try
{
await RunTask(db, msg, handler, ct);
}
catch (Exception e)
{
//TODO: logging?
Console.Error.WriteLine("Task[" + msg.Id.ToAsciiOrHexaString() + "] failed: " + e.ToString());
}
}
}
}
}
finally
{
//TODO: we should ensure that the last processed task is properly removed from the db before leaving !
Interlocked.Decrement(ref m_workers);
}
}
private static async Task RunMultiClientTest(IFdbDatabase db, FdbSubspace location, bool highContention, string desc, int K, int NUM, CancellationToken ct)
{
Console.WriteLine("Starting {0} test with {1} threads and {2} iterations", desc, K, NUM);
var queue = new FdbQueue<string>(location, highContention);
await db.WriteAsync((tr) => queue.Clear(tr), ct);
// use a CTS to ensure that everything will stop in case of problems...
using (var go = new CancellationTokenSource(TimeSpan.FromSeconds(30)))
{
var tok = go.Token;
var pushLock = new AsyncCancelableMutex(tok);
var popLock = new AsyncCancelableMutex(tok);
int pushCount = 0;
int popCount = 0;
int stalls = 0;
var pushTreads = Enumerable.Range(0, K)
.Select(async id =>
{
// wait for the signal
await pushLock.Task.ConfigureAwait(false);
var res = new List<string>(NUM);
for (int i = 0; i < NUM; i++)
{
var item = id.ToString() + "." + i.ToString();
await db.ReadWriteAsync((tr) => queue.PushAsync(tr, item), tok).ConfigureAwait(false);
Interlocked.Increment(ref pushCount);
res.Add(item);
}
return res;
}).ToArray();
var popThreads = Enumerable.Range(0, K)
.Select(async id =>
{
// make everyone wait a bit, to ensure that they all start roughly at the same time
await popLock.Task.ConfigureAwait(false);
var res = new List<string>(NUM);
int i = 0;
while (i < NUM)
{
var item = await queue.PopAsync(db, tok).ConfigureAwait(false);
if (item.HasValue)
{
Interlocked.Increment(ref popCount);
res.Add(item.Value);
++i;
}
else
{
Interlocked.Increment(ref stalls);
await Task.Delay(10).ConfigureAwait(false);
}
}
return res;
}).ToArray();
var sw = Stopwatch.StartNew();
pushLock.Set(async: true);
await Task.Delay(100);
popLock.Set(async: true);
//using (var timer = new Timer((_) =>
//{
// var __ = TestHelpers.DumpSubspace(db, location);
//}, null, 1000, 4000))
{
await Task.WhenAll(pushTreads);
await Task.WhenAll(popThreads);
}
sw.Stop();
Console.WriteLine("> Finished {0} test in {1} seconds", desc, sw.Elapsed.TotalSeconds);
Console.WriteLine("> Pushed {0}, Popped {1} and Stalled {2}", pushCount, popCount, stalls);
var pushedItems = pushTreads.SelectMany(t => t.Result).ToList();
var poppedItems = popThreads.SelectMany(t => t.Result).ToList();
Assert.That(pushCount, Is.EqualTo(K * NUM));
Assert.That(popCount, Is.EqualTo(K * NUM));
// all pushed items should have been popped (with no duplicates)
Assert.That(poppedItems, Is.EquivalentTo(pushedItems));
// the queue should be empty
bool empty = await db.ReadAsync((tr) => queue.EmptyAsync(tr), ct);
Assert.That(empty, Is.True);
}
}
public static async Task ChangeDirectoryLayer(string[] path, string layer, IFdbTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
var dir = await BasicCommands.TryOpenCurrentDirectoryAsync(path, db, ct);
if (dir == null)
{
log.WriteLine("# Directory {0} does not exist anymore", String.Join("/", path));
}
else
{
dir = await db.ReadWriteAsync((tr) => dir.ChangeLayerAsync(tr, Slice.FromString(layer)), ct);
log.WriteLine("# Directory {0} layer changed to {1}", String.Join("/", path), dir.Layer.ToAsciiOrHexaString());
}
}
/// <summary>Run the worker loop</summary>
public async Task RunWorkerAsync(IFdbDatabase db, Func <FdbWorkerMessage, CancellationToken, Task> handler, CancellationToken ct)
{
int num = Interlocked.Increment(ref s_counter);
Slice workerId = Slice.Nil;
Slice previousTaskId = Slice.Nil;
FdbWatch watch = null;
FdbWorkerMessage msg = null;
Interlocked.Increment(ref m_workers);
try
{
while (true)
{
//TODO: how do we clear the previousTaskId from the db in case of cancellation ?
ct.ThrowIfCancellationRequested();
Slice myId = Slice.Nil;
await db.ReadWriteAsync(
async (tr) =>
{
tr.Annotate("I'm worker #{0} with id {1:P}", num, workerId);
myId = workerId;
watch = null;
msg = new FdbWorkerMessage();
if (previousTaskId != null)
{ // we need to clean up the previous task
ClearTask(tr, previousTaskId);
}
else if (myId.IsPresent)
{ // look for an already assigned task
tr.Annotate("Look for already assigned task");
msg.Id = await tr.GetAsync(this.BusyRing.Keys.Encode(myId)).ConfigureAwait(false);
}
if (!msg.Id.IsPresent)
{ // We aren't already assigned a task, so get an item from a random queue
tr.Annotate("Look for next queued item");
// Find the next task on the queue
var item = await tr.GetRange(this.UnassignedTaskRing.Keys.ToRange()).FirstOrDefaultAsync().ConfigureAwait(false);
if (item.Key != null)
{ // pop the Task from the queue
msg.Id = item.Value;
tr.Clear(item.Key);
}
if (msg.Id.IsPresent)
{ // mark this worker as busy
// note: we need a random id so generate one if it is the first time...
if (!myId.IsPresent)
{
myId = GetRandomId();
}
tr.Annotate("Found {0:P}, switch to busy with id {1:P}", msg.Id, myId);
tr.Set(this.BusyRing.Keys.Encode(myId), msg.Id);
this.Counters.Increment(tr, COUNTER_BUSY);
}
else if (myId.IsPresent)
{ // remove ourselves from the busy ring
tr.Annotate("Found nothing, switch to idle with id {0:P}", myId);
//tr.Clear(this.BusyRing.Pack(myId));
}
}
if (msg.Id.IsPresent)
{ // get the task body
tr.Annotate("Fetching body for task {0:P}", msg.Id);
var prefix = this.TaskStore.Partition.ByKey(msg.Id);
//TODO: replace this with a get_range ?
var data = await tr.GetValuesAsync(new [] {
prefix.GetPrefix(),
prefix.Keys.Encode(TASK_META_SCHEDULED)
}).ConfigureAwait(false);
msg.Body = data[0];
msg.Scheduled = new DateTime(data[1].ToInt64(), DateTimeKind.Utc);
msg.Received = DateTime.UtcNow;
}
else
{ // There are no unassigned task, so enter the idle_worker_ring and wait for a task to be asssigned to us
// remove us from the busy ring
if (myId.IsPresent)
{
tr.Clear(this.BusyRing.Keys.Encode(myId));
this.Counters.Decrement(tr, COUNTER_BUSY);
}
// choose a new random position on the idle ring
myId = GetRandomId();
// the idle key will also be used as the watch key to wake us up
var watchKey = this.IdleRing.Keys.Encode(myId);
tr.Annotate("Will start watching on key {0:P} with id {1:P}", watchKey, myId);
tr.Set(watchKey, Slice.Empty);
this.Counters.Increment(tr, COUNTER_IDLE);
watch = tr.Watch(watchKey, ct);
}
},
success : (tr) =>
{ // we have successfully acquired some work, or got a watch
previousTaskId = Slice.Nil;
workerId = myId;
},
ct : ct
).ConfigureAwait(false);
if (msg.Id.IsNullOrEmpty)
{ // wait for someone to wake us up...
Interlocked.Increment(ref m_idleWorkers);
try
{
await watch.Task;
//Console.WriteLine("Worker #" + num + " woken up!");
}
finally
{
Interlocked.Decrement(ref m_idleWorkers);
}
}
else
{
//Console.WriteLine("Got task " + taskId);
previousTaskId = msg.Id;
if (msg.Body.IsNull)
{ // the task has been dropped?
// TODO: loggin?
#if DEBUG
Console.WriteLine($"[####] Task[{msg.Id:P}] has vanished?");
#endif
}
else
{
try
{
await RunTask(db, msg, handler, ct);
}
catch (Exception e)
{
//TODO: logging?
#if DEBUG
Console.Error.WriteLine($"Task[{msg.Id:P}] failed: {e}");
#endif
}
}
}
}
}
finally
{
//TODO: we should ensure that the last processed task is properly removed from the db before leaving !
Interlocked.Decrement(ref m_workers);
}
}
/// <summary>
/// Simulate a student that is really indecisive
/// </summary>
public async Task IndecisiveStudent(IFdbDatabase db, int id, int ops, CancellationToken ct)
{
string student = "s" + id.ToString("D04");
var allClasses = new List<string>(this.ClassNames);
var myClasses = new List<string>();
var rnd = new Random(id * 7);
for (int i = 0; i < ops && !ct.IsCancellationRequested; i++)
{
int classCount = myClasses.Count;
var moods = new List<string>();
if (classCount > 0) moods.AddRange(new[] { "drop", "switch" });
if (classCount < 5) moods.Add("add");
string mood = moods[rnd.Next(moods.Count)];
try
{
if (allClasses == null)
{
allClasses = await db.ReadAsync((tr) => AvailableClasses(tr), ct);
}
switch (mood)
{
case "add":
{
string @class = allClasses[rnd.Next(allClasses.Count)];
await db.ReadWriteAsync((tr) => Signup(tr, student, @class), ct);
myClasses.Add(@class);
break;
}
case "drop":
{
string @class = allClasses[rnd.Next(allClasses.Count)];
await db.ReadWriteAsync((tr) => Drop(tr, student, @class), ct);
myClasses.Remove(@class);
break;
}
case "switch":
{
string oldClass = allClasses[rnd.Next(allClasses.Count)];
string newClass = allClasses[rnd.Next(allClasses.Count)];
await db.ReadWriteAsync((tr) => Switch(tr, student, oldClass, newClass), ct);
myClasses.Remove(oldClass);
myClasses.Add(newClass);
break;
}
default:
{
throw new InvalidOperationException("Ooops");
}
}
}
catch (Exception e)
{
if (e is TaskCanceledException || e is OperationCanceledException) throw;
allClasses = null;
}
}
ct.ThrowIfCancellationRequested();
}
/// <summary>
/// Setup the initial state of the database
/// </summary>
public async Task Init(IFdbDatabase db, CancellationToken ct)
{
// open the folder where we will store everything
this.Subspace = await db.ReadWriteAsync(tr => db.Root["Benchmarks"].CreateOrOpenAsync(tr), ct);
}
public static async Task Get(string[] path, IVarTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
if (path == null || path.Length == 0)
{
Program.Error(log, "Cannot read keys of the Root Partition.");
return;
}
if (extras.Count == 0)
{
Program.Error(log, "You must specify a key of range of keys!");
return;
}
var folder = await db.Directory.TryOpenAsync(db, path, ct : ct);
if (folder == null)
{
Program.Error(log, "The directory does not exist anymore");
return;
}
if (folder.Layer == FdbDirectoryPartition.LayerId)
{
Program.Error(log, "Cannot clear the content of a Directory Partition!");
return;
}
object key = extras[0];
Slice k = MakeKey(folder, key);
Program.Comment(log, "# Reading key: " + k.ToString("K"));
Slice v = await db.ReadWriteAsync(tr => tr.GetAsync(k), ct);
if (v.IsNull)
{
Program.StdOut(log, "# Key does not exist in the database.", ConsoleColor.Red);
return;
}
if (v.IsEmpty)
{
Program.StdOut(log, "# Key exists but is empty.", ConsoleColor.Gray);
return;
}
Program.StdOut(log, $"# Size: {v.Count:N0}", ConsoleColor.Gray);
string format = extras.Count > 1 ? extras.Get <string>(1) : null;
switch (format)
{
case "--text":
case "--json":
case "--utf8":
{
Program.StdOut(log, v.ToStringUtf8(), ConsoleColor.Gray);
break;
}
case "--hex":
case "--hexa":
{
Program.StdOut(log, v.ToHexaString(), ConsoleColor.White);
break;
}
case "--dump":
{
var sb = new StringBuilder(v.Count * 3 + (v.Count >> 4) * 2 + 16);
for (int i = 0; i < v.Count; i += 16)
{
sb.AppendLine(v.Substring(i, 16).ToHexaString(' '));
}
Program.StdOut(log, sb.ToString(), ConsoleColor.White);
break;
}
case "--int":
{
if (v.Count <= 8)
{
long he = v.ToInt64BE();
long le = v.ToInt64();
Program.StdOut(log, $"BE: {he:X016} ({he:N0})", ConsoleColor.White);
Program.StdOut(log, $"LE: {le:X016} ({le:N0})", ConsoleColor.White);
}
else
{
Program.StdOut(log, $"Value is too large ({v.Count} bytes)", ConsoleColor.DarkRed);
Program.StdOut(log, v.ToHexaString(' '), ConsoleColor.Gray);
}
break;
}
case "--uint":
{
if (v.Count <= 8)
{
ulong he = v.ToUInt64BE();
ulong le = v.ToUInt64();
Program.StdOut(log, $"BE: {he:X016} ({he:N0})", ConsoleColor.White);
Program.StdOut(log, $"LE: {le:X016} ({le:N0})", ConsoleColor.White);
}
else
{
Program.StdOut(log, $"Value is too large ({v.Count} bytes)", ConsoleColor.DarkRed);
Program.StdOut(log, v.ToHexaString(' '), ConsoleColor.Gray);
}
break;
}
case "--tuple":
{
try
{
var t = TuPack.Unpack(v);
Program.StdOut(log, t.ToString(), ConsoleColor.Gray);
}
catch (Exception e)
{
Program.Error(log, "Key value does not seem to be a valid Tuple: " + e.Message);
Program.StdOut(log, v.ToHexaString(' '), ConsoleColor.Gray);
}
break;
}
default:
{
Program.StdOut(log, v.ToString("V"), ConsoleColor.White);
break;
}
}
}
private static async Task RunMultiClientTest(IFdbDatabase db, KeySubspace location, bool highContention, string desc, int K, int NUM, CancellationToken ct)
{
Log("Starting {0} test with {1} threads and {2} iterations", desc, K, NUM);
var queue = new FdbQueue <string>(location, highContention);
await db.WriteAsync((tr) => queue.Clear(tr), ct);
// use a CTS to ensure that everything will stop in case of problems...
using (var go = new CancellationTokenSource(TimeSpan.FromSeconds(30)))
{
var tok = go.Token;
var pushLock = new AsyncCancelableMutex(tok);
var popLock = new AsyncCancelableMutex(tok);
int pushCount = 0;
int popCount = 0;
int stalls = 0;
var pushTreads = Enumerable.Range(0, K)
.Select(async id =>
{
try
{
// wait for the signal
await pushLock.Task.ConfigureAwait(false);
var res = new List <string>(NUM);
for (int i = 0; i < NUM; i++)
{
var item = id.ToString() + "." + i.ToString();
await db.ReadWriteAsync((tr) => queue.PushAsync(tr, item), tok).ConfigureAwait(false);
Interlocked.Increment(ref pushCount);
res.Add(item);
}
return(res);
}
catch (Exception e)
{
Log("PushThread[" + id + "] failed: " + e);
Assert.Fail("PushThread[" + id + "] failed: " + e.Message);
throw;
}
}).ToArray();
var popThreads = Enumerable.Range(0, K)
.Select(async id =>
{
try
{
// make everyone wait a bit, to ensure that they all start roughly at the same time
await popLock.Task.ConfigureAwait(false);
var res = new List <string>(NUM);
int i = 0;
while (i < NUM)
{
var item = await queue.PopAsync(db, tok).ConfigureAwait(false);
if (item.HasValue)
{
Interlocked.Increment(ref popCount);
res.Add(item.Value);
++i;
}
else
{
Interlocked.Increment(ref stalls);
await Task.Delay(10).ConfigureAwait(false);
}
}
return(res);
}
catch (Exception e)
{
Log("PopThread[" + id + "] failed: " + e);
Assert.Fail("PopThread[" + id + "] failed: " + e.Message);
throw;
}
}).ToArray();
var sw = Stopwatch.StartNew();
pushLock.Set(async: true);
await Task.Delay(100);
popLock.Set(async: true);
//using (var timer = new Timer((_) =>
//{
// var __ = TestHelpers.DumpSubspace(db, location);
//}, null, 1000, 4000))
{
await Task.WhenAll(pushTreads);
await Task.WhenAll(popThreads);
}
sw.Stop();
Log("> Finished {0} test in {1} seconds", desc, sw.Elapsed.TotalSeconds);
Log("> Pushed {0}, Popped {1} and Stalled {2}", pushCount, popCount, stalls);
var pushedItems = pushTreads.SelectMany(t => t.Result).ToList();
var poppedItems = popThreads.SelectMany(t => t.Result).ToList();
Assert.That(pushCount, Is.EqualTo(K * NUM));
Assert.That(popCount, Is.EqualTo(K * NUM));
// all pushed items should have been popped (with no duplicates)
Assert.That(poppedItems, Is.EquivalentTo(pushedItems));
// the queue should be empty
bool empty = await db.ReadAsync((tr) => queue.EmptyAsync(tr), ct);
Assert.That(empty, Is.True);
}
}
public static async Task ClearRange(string[] path, IVarTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
if (path == null || path.Length == 0)
{
Program.Error(log, "Cannot directory list the content of Root Partition.");
return;
}
var folder = await db.Directory.TryOpenAsync(db, path, ct : ct);
if (folder == null)
{
Program.Error(log, "The directory does not exist anymore");
return;
}
if (folder.Layer == FdbDirectoryPartition.LayerId)
{
Program.Error(log, "Cannot clear the content of a Directory Partition!");
return;
}
if (extras.Count == 0)
{
Program.Error(log, "You must specify a key of range of keys!");
return;
}
KeyRange range;
if (extras.Get <string>(0) == "*")
{ // clear all!
range = folder.ToRange();
}
else
{
object from = extras[0];
object to = extras.Count > 1 ? extras[1] : null;
if (to == null)
{
range = KeyRange.StartsWith(MakeKey(folder, from));
}
else
{
range = new KeyRange(
MakeKey(folder, from),
MakeKey(folder, to)
);
}
}
Program.Comment(log, "Clearing range " + range.ToString());
bool empty = await db.ReadWriteAsync(async tr =>
{
var any = await tr.GetRangeAsync(range, new FdbRangeOptions {
Limit = 1
});
if (any.Count == 0)
{
return(true);
}
tr.ClearRange(folder.ToRange());
return(false);
}, ct);
if (empty)
{
Program.StdOut(log, $"Directory {String.Join("/", path)} was already empty.", ConsoleColor.Cyan);
}
else
{
Program.Success(log, $"Cleared all content of Directory {String.Join("/", path)}.");
}
}
/// <summary>Opens a subdirectory with the given path.
/// If the subdirectory does not exist, it is created (creating intermediate subdirectories if necessary).
/// If layer is specified, it is checked against the layer of an existing subdirectory or set as the layer of a new subdirectory.
/// </summary>
public Task <FdbDirectorySubspace> CreateOrOpenAsync([NotNull] string name, CancellationToken cancellationToken)
{
return(m_database.ReadWriteAsync((tr) => m_directory.CreateOrOpenAsync(tr, new [] { name }, Slice.Nil), cancellationToken));
}