/// <summary>
/// Simulate a student that is really indecisive
/// </summary>
public async Task RunProducer(IFdbDatabase db, CancellationToken ct)
{
int cnt = 0;
var rnd = new Random(123456);
DateTime last = DateTime.Now;
rnd = new Random();
this.TimeLine.Start();
while (!ct.IsCancellationRequested)
{
int k = cnt++;
Slice taskId = FdbTuple.EncodeKey(this.Id.GetHashCode(), k);
var ts = Stopwatch.GetTimestamp();
string msg = "Message #" + k + " from producer " + this.Id + " (" + DateTime.UtcNow.ToString("O") + ")";
var latency = Stopwatch.StartNew();
await this.WorkerPool.ScheduleTaskAsync(db, taskId, Slice.FromString(msg), ct).ConfigureAwait(false);
latency.Stop();
Console.Write(k.ToString("N0") + "\r");
this.TimeLine.Add(latency.Elapsed.TotalMilliseconds);
TimeSpan delay = TimeSpan.FromTicks(rnd.Next((int)this.DelayMin.Ticks, (int)this.DelayMax.Ticks));
await Task.Delay(delay).ConfigureAwait(false);
}
this.TimeLine.Stop();
ct.ThrowIfCancellationRequested();
}
/// <summary>
/// Setup the initial state of the database
/// </summary>
public async Task Init(IFdbDatabase db, CancellationToken ct)
{
// open the folder where we will store everything
this.Subspace = await db.Directory.CreateOrOpenAsync(new [] { "Samples", "MessageQueueTest" }, cancellationToken: ct);
this.WorkerPool = new FdbWorkerPool(this.Subspace);
}
/// <summary>Wrap an existing database with a root directory</summary>
public FdbDatabasePartition(IFdbDatabase database, IFdbDirectory directory)
{
if (database == null) throw new ArgumentNullException("database");
if (directory == null) throw new ArgumentNullException("directory");
m_database = database;
m_directory = directory;
}
public static async Task Dir(string[] path, IFdbTuple extras, DirectoryBrowseOptions options, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
if (log == null) log = Console.Out;
log.WriteLine("# Listing {0}:", String.Join("/", path));
var parent = await TryOpenCurrentDirectoryAsync(path, db, ct);
if (parent == null)
{
log.WriteLine(" Directory not found.");
return;
}
if (parent.Layer.IsPresent)
{
log.WriteLine("# Layer: {0}", parent.Layer.ToAsciiOrHexaString());
}
var folders = await Fdb.Directory.BrowseAsync(db, parent, ct);
if (folders != null && folders.Count > 0)
{
foreach (var kvp in folders)
{
var name = kvp.Key;
var subfolder = kvp.Value;
if (subfolder != null)
{
if ((options & DirectoryBrowseOptions.ShowCount) != 0)
{
if (!(subfolder is FdbDirectoryPartition))
{
long count = await Fdb.System.EstimateCountAsync(db, subfolder.ToRange(), ct);
log.WriteLine(" {0,-12} {1,-12} {3,9:N0} {2}", FdbKey.Dump(subfolder.Copy().Key), subfolder.Layer.IsNullOrEmpty ? "-" : ("<" + subfolder.Layer.ToUnicode() + ">"), name, count);
}
else
{
log.WriteLine(" {0,-12} {1,-12} {3,9} {2}", FdbKey.Dump(subfolder.Copy().Key), subfolder.Layer.IsNullOrEmpty ? "-" : ("<" + subfolder.Layer.ToUnicode() + ">"), name, "-");
}
}
else
{
log.WriteLine(" {0,-12} {1,-12} {2}", FdbKey.Dump(subfolder.Copy().Key), subfolder.Layer.IsNullOrEmpty ? "-" : ("<" + subfolder.Layer.ToUnicode() + ">"), name);
}
}
else
{
log.WriteLine(" WARNING: {0} seems to be missing!", name);
}
}
log.WriteLine(" {0} sub-directorie(s).", folders.Count);
}
else
{
//TODO: test if it contains data?
log.WriteLine(" No sub-directories.");
}
}
/// <summary>Create a new High Contention counter, using a specific value encoder.</summary>
/// <param name="db">Database used by this layer</param>
/// <param name="subspace">Subspace to be used for storing the counter</param>
/// <param name="encoder">Encoder for the counter values</param>
public FdbHighContentionCounter(IFdbDatabase db, FdbSubspace subspace, IValueEncoder<long> encoder)
{
if (db == null) throw new ArgumentNullException("db");
if (subspace == null) throw new ArgumentNullException("subspace");
if (encoder == null) throw new ArgumentNullException("encoder");
this.Database = db;
this.Subspace = subspace;
this.Encoder = encoder;
}
public static async Task<IFdbDirectory> TryOpenCurrentDirectoryAsync(string[] path, IFdbDatabase db, CancellationToken ct)
{
if (path != null && path.Length > 0)
{
return await db.Directory.TryOpenAsync(path, cancellationToken: ct);
}
else
{
return db.Directory;
}
}
internal static Exception InvalidKeyOutsideDatabaseNamespace(IFdbDatabase db, Slice key)
{
Contract.Requires(db != null);
return new FdbException(
FdbError.KeyOutsideLegalRange,
#if DEBUG
String.Format("An attempt was made to use a key '{2}' that is outside of the global namespace {0} of database '{1}'", db.GlobalSpace, db.Name, FdbKey.Dump(key))
#else
String.Format("An attempt was made to use a key that is outside of the global namespace {0} of database '{1}'", db.GlobalSpace, db.Name)
#endif
);
}
private static async Task BurnerThread(IFdbDatabase db, CancellationToken ct)
{
var folder = await db.Directory.CreateOrOpenAsync(new[] { "Benchmarks", "Burner", "Sequential" }, ct);
await db.WriteAsync((tr) => tr.ClearRange(folder), ct);
long pos = 0;
Random rnd;
lock(Rnd)
{
rnd = new Random(Rnd.Next());
}
using (var tr = db.BeginTransaction(ct))
{
while (!ct.IsCancellationRequested)
{
FdbException error = null;
try
{
tr.Reset();
for(int i = 0; i < N; i++)
{
long x = Randomized
? rnd.Next()
: pos + i;
tr.Set(folder.Keys.Encode(x, Suffix), Value);
Interlocked.Increment(ref Keys);
}
pos += N;
await tr.CommitAsync();
Interlocked.Increment(ref Transactions);
Interlocked.Add(ref Bytes, tr.Size);
}
catch (FdbException e)
{
error = e;
}
if (error != null && !ct.IsCancellationRequested)
{
await tr.OnErrorAsync(error.Code);
}
}
}
}
/// <summary>
/// Setup the initial state of the database
/// </summary>
public async Task Init(IFdbDatabase db, CancellationToken ct)
{
// open the folder where we will store everything
this.Subspace = await db.Directory.CreateOrOpenAsync(new [] { "Benchmarks", "LeakTest" }, cancellationToken: ct);
// clear all previous values
await db.ClearRangeAsync(this.Subspace, ct);
// insert all the classes
await db.WriteAsync((tr) =>
{
tr.Set(this.Subspace.Concat(FdbKey.MinValue), Slice.FromString("BEGIN"));
tr.Set(this.Subspace.Concat(FdbKey.MaxValue), Slice.FromString("END"));
}, ct);
}
/// <summary>Checks that one or more keys are inside the global namespace of this database, and contained in the optional legal key space specified by the user</summary>
/// <param name="db">Database instance</param>
/// <param name="keys">Array of keys to verify</param>
/// <param name="endExclusive">If true, the keys are allowed to be one past the maximum key allowed by the global namespace</param>
/// <exception cref="FdbException">If at least on key is outside of the allowed keyspace, throws an FdbException with code FdbError.KeyOutsideLegalRange</exception>
internal static void EnsureKeysAreValid(this IFdbDatabase db, Slice[] keys, bool endExclusive = false)
{
if (keys == null)
{
throw new ArgumentNullException("keys");
}
for (int i = 0; i < keys.Length; i++)
{
Exception ex;
if (!FdbDatabase.ValidateKey(db, ref keys[i], endExclusive, false, out ex))
{
throw ex;
}
}
}
/// <summary>
/// Setup the initial state of the database
/// </summary>
public async Task Init(IFdbDatabase db, CancellationToken ct)
{
// open the folder where we will store everything
this.Subspace = await db.Directory.CreateOrOpenAsync(new [] { "Benchmarks", "LeakTest" }, ct : ct);
// clear all previous values
await db.ClearRangeAsync(this.Subspace, ct);
// insert all the classes
await db.WriteAsync((tr) =>
{
tr.Set(this.Subspace.GetPrefix() + FdbKey.MinValue, Slice.FromString("BEGIN"));
tr.Set(this.Subspace.GetPrefix() + FdbKey.MaxValue, Slice.FromString("END"));
}, ct);
}
/// <summary>Set the size of the client location cache. Raising this value can boost performance in very large databases where clients access data in a near-random pattern. Defaults to 100000.</summary>
/// <param name="db">Database instance</param>
/// <param name="size">Max location cache entries</param>
public static void SetLocationCacheSize(this IFdbDatabase db, int size)
{
if (db == null)
{
throw new ArgumentNullException("db");
}
if (size < 0)
{
throw new FdbException(FdbError.InvalidOptionValue, "Location cache size must be a positive integer");
}
//REVIEW: we can't really change this to a Property, because we don't have a way to get the current value for the getter, and set only properties are weird...
//TODO: cache this into a local variable ?
db.SetOption(FdbDatabaseOption.LocationCacheSize, size);
}
/// <summary>Set the maximum number of watches allowed to be outstanding on a database connection. Increasing this number could result in increased resource usage. Reducing this number will not cancel any outstanding watches. Defaults to 10000 and cannot be larger than 1000000.</summary>
/// <param name="db">Database instance</param>
/// <param name="count">Max outstanding watches</param>
public static void SetMaxWatches(this IFdbDatabase db, int count)
{
if (db == null)
{
throw new ArgumentNullException("db");
}
if (count < 0)
{
throw new FdbException(FdbError.InvalidOptionValue, "Maximum outstanding watches count must be a positive integer");
}
//REVIEW: we can't really change this to a Property, because we don't have a way to get the current value for the getter, and set only properties are weird...
//TODO: cache this into a local variable ?
db.SetOption(FdbDatabaseOption.MaxWatches, count);
}
/// <summary>Remove a directory and all its data</summary>
public static async Task RemoveDirectory(FdbPath path, IVarTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
if (log == null)
{
log = Console.Out;
}
// "-r|--recursive" is used to allow removing an entire sub-tree
string[] args = extras.ToArray <string>();
bool recursive = args.Contains("-r", StringComparer.Ordinal) || args.Contains("--recursive", StringComparer.Ordinal);
bool force = args.Contains("-f", StringComparer.Ordinal) || args.Contains("--force", StringComparer.Ordinal);
var res = await db.ReadWriteAsync(async tr =>
{
var folder = await db.DirectoryLayer.TryOpenAsync(tr, path);
if (folder == null)
{
Program.Error(log, $"# Directory {path} does not exist");
return(false);
}
// are there any subdirectories ?
if (!recursive)
{
var subDirs = await folder.TryListAsync(tr);
if (subDirs != null && subDirs.Count > 0)
{
//TODO: "-r" flag ?
Program.Error(log, $"# Cannot remove {path} because it still contains {subDirs.Count:N0} sub-directories.");
Program.StdOut(log, "Use the -r|--recursive flag to override this warning.");
return(false);
}
}
if (!force)
{
//TODO: ask for confirmation?
}
await folder.RemoveAsync(tr);
return(true);
}, ct);
if (res)
{
Program.Success(log, $"Deleted directory {path}");
}
}
/// <summary>
/// Simulate a student that is really indecisive
/// </summary>
public async Task RunWorker(IFdbDatabase db, int id, CancellationToken ct)
{
string student = "WORKER" + id.ToString("D04");
var rnd = new Random(id * 7);
var values = new string[this.M];
for (int i = 0; i < values.Length;i++)
{
values[i] = "initial_value_" + rnd.Next();
}
var prefix = this.Subspace.Partition(student);
for (int i = 0; i < 1/*this.N*/ && !ct.IsCancellationRequested; i++)
{
// randomly mutate values
var n = rnd.Next(values.Length / 2);
for (int j = 0; j < n;j++)
{
values[rnd.Next(values.Length)] = "value_" + i.ToString() + "_" + rnd.Next().ToString();
}
long now = DateTime.Now.Ticks / TimeSpan.TicksPerMillisecond;
// write everything
await db.WriteAsync((tr) =>
{
if (tr.Context.Retries > 0) Console.Write("!");
for (int j = 0; j < values.Length; j++)
{
tr.Set(prefix.Pack(j, now), Slice.FromString(values[j] + new string('A', 100)));
}
}, ct);
Console.Write(".");
//var r = await db.ReadAsync(async (tr) =>
//{
// if (tr.Context.Retries > 0) Console.Write("!");
// return await Task.WhenAll(Enumerable.Range(0, values.Length).Select(x => tr.GetRange(FdbKeyRange.StartsWith(prefix.Pack(x))).LastOrDefaultAsync()));
//}, ct);
//if (i % 10 == 0) Console.Write(":");
//await Task.Delay(this.Delay);
}
ct.ThrowIfCancellationRequested();
}
/// <summary>Shows the first few keys of a directory</summary>
public static async Task Show(string[] path, IFdbTuple extras, bool reverse, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
int count = 20;
if (extras.Count > 0)
{
int x = extras.Get <int>(0);
if (x > 0)
{
count = x;
}
}
// look if there is something under there
var folder = await db.Directory.TryOpenAsync(path, cancellationToken : ct);
if (folder != null)
{
log.WriteLine("# Content of {0} [{1}]", FdbKey.Dump(folder.Key), folder.Key.ToHexaString(' '));
var keys = await db.QueryAsync((tr) =>
{
var query = tr.GetRange(folder.ToRange());
return(reverse
? query.Reverse().Take(count)
: query.Take(count + 1));
}, cancellationToken : ct);
if (keys.Count > 0)
{
if (reverse)
{
keys.Reverse();
}
foreach (var key in keys.Take(count))
{
log.WriteLine("...{0} = {1}", FdbKey.Dump(folder.Extract(key.Key)), key.Value.ToAsciiOrHexaString());
}
if (!reverse && keys.Count == count + 1)
{
log.WriteLine("... more");
}
}
else
{
log.WriteLine(" no content found");
}
}
}
private static async Task TestSimpleTransactionAsync(IFdbDatabase db, CancellationToken ct)
{
Console.WriteLine("=== TestSimpleTransaction() ===");
var location = db.GlobalSpace;
Console.WriteLine("Starting new transaction...");
using (var trans = db.BeginTransaction(ct))
{
Console.WriteLine("> Transaction ready");
Console.WriteLine("Getting read version...");
var readVersion = await trans.GetReadVersionAsync();
Console.WriteLine("> Read Version = " + readVersion);
Console.WriteLine("Getting 'hello'...");
trans.Set(location.Keys.Encode("hello"), Slice.FromString("hello"));
var result = await trans.GetAsync(location.Keys.Encode("hello"));
if (result.IsNull)
{
Console.WriteLine("> hello NOT FOUND");
}
else
{
Console.WriteLine($"> hello = {result:V}");
}
Console.WriteLine("Setting 'Foo' = 'Bar'");
trans.Set(location.Keys.Encode("Foo"), Slice.FromString("Bar"));
Console.WriteLine("Setting 'TopSecret' = rnd(512)");
var data = new byte[512];
new Random(1234).NextBytes(data);
trans.Set(location.Keys.Encode("TopSecret"), data.AsSlice());
Console.WriteLine("Committing transaction...");
await trans.CommitAsync();
//trans.Commit();
Console.WriteLine("> Committed!");
Console.WriteLine("Getting comitted version...");
var writeVersion = trans.GetCommittedVersion();
Console.WriteLine("> Commited Version = " + writeVersion);
}
}
public static IFdbDatabase WithoutLogging([NotNull] this IFdbDatabase database)
{
if (database == null)
{
throw new ArgumentNullException("database");
}
var logged = database as FdbLoggedDatabase;
if (logged != null)
{
return(logged.GetInnerDatabase());
}
return(database);
}
public static FdbLoggedDatabase Logged([NotNull] this IFdbDatabase database, [NotNull] Action <FdbLoggedTransaction> handler, FdbLoggingOptions options = FdbLoggingOptions.Default)
{
if (database == null)
{
throw new ArgumentNullException(nameof(database));
}
if (handler == null)
{
throw new ArgumentNullException(nameof(handler));
}
// prevent multiple logging
database = WithoutLogging(database);
return(new FdbLoggedDatabase(database, false, false, handler, options));
}
/// <summary>Creates a new directory</summary>
public static async Task CreateDirectory(FdbPath path, IVarTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
if (log == null)
{
log = Console.Out;
}
string layer = extras.Count > 0 ? extras.Get <string>(0).Trim() : string.Empty;
if (path.LayerId != layer)
{
path.WithLayer(layer);
}
log.WriteLine($"# Creating directory {path}");
(var prefix, var created) = await db.ReadWriteAsync(async tr =>
{
var folder = await db.DirectoryLayer.TryOpenAsync(tr, path);
if (folder != null)
{
return(folder.GetPrefix(), false);
}
folder = await db.DirectoryLayer.TryCreateAsync(tr, path);
return(folder.Copy().GetPrefix(), true);
}, ct);
if (!created)
{
log.WriteLine($"- Directory {path} already exists at {FdbKey.Dump(prefix)} [{prefix.ToHexaString(' ')}]");
return;
}
log.WriteLine($"- Created under {FdbKey.Dump(prefix)} [{prefix.ToHexaString(' ')}]");
// look if there is already stuff under there
var stuff = await db.ReadAsync(async tr =>
{
var folder = await db.DirectoryLayer.TryOpenAsync(tr, path);
return(await tr.GetRange(folder.ToRange()).FirstOrDefaultAsync());
}, ct);
if (stuff.Key.IsPresent)
{
Program.Error(log, $"CAUTION: There is already some data under {path} !");
Program.Error(log, $" {FdbKey.Dump(stuff.Key)} = {stuff.Value:V}");
}
}
public async Task RunStatus(IFdbDatabase db, CancellationToken ct)
{
var countersLocation = this.WorkerPool.Subspace.Partition.ByKey(Slice.FromChar('C'));
var idleLocation = this.WorkerPool.Subspace.Partition.ByKey(Slice.FromChar('I'));
var busyLocation = this.WorkerPool.Subspace.Partition.ByKey(Slice.FromChar('B'));
var tasksLocation = this.WorkerPool.Subspace.Partition.ByKey(Slice.FromChar('T'));
var unassignedLocation = this.WorkerPool.Subspace.Partition.ByKey(Slice.FromChar('U'));
using (var tr = db.BeginTransaction(ct))
{
var counters = await tr.Snapshot.GetRange(countersLocation.Keys.ToRange()).Select(kvp => new KeyValuePair <string, long>(countersLocation.Keys.DecodeLast <string>(kvp.Key), kvp.Value.ToInt64())).ToListAsync().ConfigureAwait(false);
Console.WriteLine("Status at " + DateTimeOffset.Now.ToString("O"));
foreach (var counter in counters)
{
Console.WriteLine(" - " + counter.Key + " = " + counter.Value);
}
Console.WriteLine("Dump:");
Console.WriteLine("> Idle");
await tr.Snapshot.GetRange(idleLocation.Keys.ToRange()).ForEachAsync((kvp) =>
{
Console.WriteLine($"- Idle.{idleLocation.Keys.Unpack(kvp.Key)} = {kvp.Value:V}");
});
Console.WriteLine("> Busy");
await tr.Snapshot.GetRange(busyLocation.Keys.ToRange()).ForEachAsync((kvp) =>
{
Console.WriteLine($"- Busy.{busyLocation.Keys.Unpack(kvp.Key)} = {kvp.Value:V}");
});
Console.WriteLine("> Unassigned");
await tr.Snapshot.GetRange(unassignedLocation.Keys.ToRange()).ForEachAsync((kvp) =>
{
Console.WriteLine($"- Unassigned.{unassignedLocation.Keys.Unpack(kvp.Key)} = {kvp.Value:V}");
});
Console.WriteLine("> Tasks");
await tr.Snapshot.GetRange(tasksLocation.Keys.ToRange()).ForEachAsync((kvp) =>
{
Console.WriteLine($"- Tasks.{tasksLocation.Keys.Unpack(kvp.Key)} = {kvp.Value:V}");
});
Console.WriteLine("<");
}
}
/// <summary>Display a tree of a directory's children</summary>
public static async Task Tree(string[] path, IVarTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
log = log ?? Console.Out;
Program.Comment(log, $"# Tree of {String.Join("/", path)}:");
FdbDirectorySubspace root = null;
if (path.Length > 0)
{
root = await db.Directory.TryOpenAsync(db, path, ct : ct);
}
await TreeDirectoryWalk(root, new List <bool>(), db, log, ct);
Program.Comment(log, "# done");
}
public static async Task <FdbSystemStatus> GetStatusAsync([NotNull] IFdbDatabase db, CancellationToken ct)
{
if (db == null)
{
throw new ArgumentNullException(nameof(db));
}
// we should not retry the read to the status key!
using (var trans = db.BeginReadOnlyTransaction(ct))
{
trans.WithPrioritySystemImmediate();
//note: in v3.x, the status key does not need the access to system key option.
//TODO: set a custom timeout?
return(await GetStatusAsync(trans));
}
}
/// <summary>
/// Setup the initial state of the database
/// </summary>
public async Task Init(IFdbDatabase db, CancellationToken ct)
{
// open the folder where we will store everything
this.Subspace = await db.Directory.CreateOrOpenAsync(new [] { "Tutorials", "ClassScheduling" }, ct : ct);
// clear all previous values
await db.ClearRangeAsync(this.Subspace, ct);
// insert all the classes
await db.WriteAsync((tr) =>
{
foreach (var c in this.ClassNames)
{
tr.Set(ClassKey(c), Slice.FromStringAscii("100"));
}
}, ct);
}
/// <summary>
/// Setup the initial state of the database
/// </summary>
public async Task Init(IFdbDatabase db, CancellationToken ct)
{
this.Subspace = await db.ReadWriteAsync(async tr =>
{
// open the folder where we will store everything
var subspace = await db.Root["Benchmarks"]["LeakTest"].CreateOrOpenAsync(tr);
// clear all previous values
await db.ClearRangeAsync(subspace, ct);
// insert all the classes
tr.Set(subspace.GetPrefix() + FdbKey.MinValue, Slice.FromString("BEGIN"));
tr.Set(subspace.GetPrefix() + FdbKey.MaxValue, Slice.FromString("END"));
return(subspace);
}, ct);
}
/// <summary>Imports a potentially large sequence of items into the map.</summary>
/// <param name="db">Database used for the operation</param>
/// <param name="items">Sequence of items to import. If the item already exists in the map, its value will be overwritten.</param>
/// <param name="cancellationToken">Token used to cancel the operation</param>
/// <remarks>
/// <p>Any previously existing items in the map will remain. If you want to get from the previous content, you need to clear the map before hand.</p>
/// <p>Other transactions may see a partial view of the map while the sequence is being imported. If this is a problem, you may need to import the map into a temporary subspace, and then 'publish' the final result using an indirection layer (like the Directory Layer)</p>
/// <p>If the import operation fails midway, all items that have already been successfully imported will be kept in the database.</p>
/// </remarks>
public Task ImportAsync([NotNull] IFdbDatabase db, [NotNull] IEnumerable <KeyValuePair <TKey, TValue> > items, CancellationToken cancellationToken)
{
if (db == null)
{
throw new ArgumentNullException("db");
}
if (items == null)
{
throw new ArgumentNullException("items");
}
return(Fdb.Bulk.InsertAsync(
db,
items,
(item, tr) => this.Set(tr, item.Key, item.Value),
cancellationToken
));
}
/// <summary>Exports the content of this map out of the database, by using as many transactions as necessary.</summary>
/// <param name="db">Database used for the operation</param>
/// <param name="handler">Handler called for each batch of items in the map. Calls to the handler are serialized, so it does not need to take locks. Any exception will abort the export and be thrown to the caller</param>
/// <param name="cancellationToken">Token used to cancel the operation.</param>
/// <returns>Task that completes once all the entries have been processed.</returns>
/// <remarks>This method does not guarantee that the export will be a complete and coherent snapshot of the map, except that all the items in a single batch are from the same snapshot. Any change made to the map while the export is running may be partially exported.</remarks>
public Task ExportAsync([NotNull] IFdbDatabase db, [NotNull] Func <KeyValuePair <TKey, TValue>[], CancellationToken, Task> handler, CancellationToken cancellationToken)
{
if (db == null)
{
throw new ArgumentNullException("db");
}
if (handler == null)
{
throw new ArgumentNullException("handler");
}
return(Fdb.Bulk.ExportAsync(
db,
this.Location.ToRange(),
(batch, _, ct) => handler(DecodeItems(batch), ct),
cancellationToken
));
}
/// <summary>Imports a potentially large sequence of items into the map.</summary>
/// <param name="db">Database used for the operation</param>
/// <param name="items">Sequence of items to import. If the item already exists in the map, its value will be overwritten.</param>
/// <param name="ct">Token used to cancel the operation</param>
/// <remarks>
/// <p>Any previously existing items in the map will remain. If you want to get from the previous content, you need to clear the map before hand.</p>
/// <p>Other transactions may see a partial view of the map while the sequence is being imported. If this is a problem, you may need to import the map into a temporary subspace, and then 'publish' the final result using an indirection layer (like the Directory Layer)</p>
/// <p>If the import operation fails midway, all items that have already been successfully imported will be kept in the database.</p>
/// </remarks>
public Task ImportAsync(IFdbDatabase db, IEnumerable <KeyValuePair <TKey, TValue> > items, CancellationToken ct)
{
if (db == null)
{
throw new ArgumentNullException(nameof(db));
}
if (items == null)
{
throw new ArgumentNullException(nameof(items));
}
return(Fdb.Bulk.InsertAsync(
db,
items,
(item, tr) => Set(tr, item.Key, item.Value),
ct
));
}
/// <summary>Exports the content of this map out of the database, by using as many transactions as necessary.</summary>
/// <param name="db">Database used for the operation</param>
/// <param name="handler">Handler called for each batch of items in the map. Calls to the handler are serialized, so it does not need to take locks. Any exception will abort the export and be thrown to the caller</param>
/// <param name="ct">Token used to cancel the operation.</param>
/// <returns>Task that completes once all the entries have been processed.</returns>
/// <remarks>This method does not guarantee that the export will be a complete and coherent snapshot of the map, except that all the items in a single batch are from the same snapshot. Any change made to the map while the export is running may be partially exported.</remarks>
public Task ExportAsync([NotNull] IFdbDatabase db, [NotNull] Func <KeyValuePair <TKey, TValue>[], CancellationToken, Task> handler, CancellationToken ct)
{
if (db == null)
{
throw new ArgumentNullException(nameof(db));
}
if (handler == null)
{
throw new ArgumentNullException(nameof(handler));
}
return(Fdb.Bulk.ExportAsync(
db,
this.Subspace.ToRange(),
(batch, _, tok) => handler(DecodeItems(batch), tok),
ct
));
}
/// <summary>Exports the content of this map out of the database, by using as many transactions as necessary.</summary>
/// <param name="db">Database used for the operation</param>
/// <param name="handler">Handler called for each batch of items in the map. Calls to the handler are serialized, so it does not need to take locks. Any exception will abort the export and be thrown to the caller</param>
/// <param name="ct">Token used to cancel the operation.</param>
/// <returns>Task that completes once all the entries have been processed.</returns>
/// <remarks>This method does not guarantee that the export will be a complete and coherent snapshot of the map, except that all the items in a single batch are from the same snapshot. Any change made to the map while the export is running may be partially exported.</remarks>
public Task ExportAsync(IFdbDatabase db, Func <KeyValuePair <TKey, TValue>[], CancellationToken, Task> handler, CancellationToken ct)
{
if (db == null)
{
throw new ArgumentNullException(nameof(db));
}
if (handler == null)
{
throw new ArgumentNullException(nameof(handler));
}
return(Fdb.Bulk.ExportAsync(
db,
this.Location,
(batch, loc, _, tok) => handler(DecodeItems(loc, this.ValueEncoder, batch), tok),
ct
));
}
/// <summary>
/// Setup the initial state of the database
/// </summary>
public async Task Init(IFdbDatabase db, CancellationToken ct)
{
// open the folder where we will store everything
this.Subspace = await db.Directory.CreateOrOpenAsync(new [] { "Tutorials", "ClassScheduling" }, cancellationToken: ct);
// clear all previous values
await db.ClearRangeAsync(this.Subspace, ct);
// insert all the classes
await db.WriteAsync((tr) =>
{
foreach (var c in this.ClassNames)
{
tr.Set(ClassKey(c), Slice.FromAscii("100"));
}
}, ct);
}
public async Task Run(IFdbDatabase db, TextWriter log, CancellationToken ct)
{
const int STUDENTS = 10;
const int OPS_PER_STUDENTS = 10;
await Init(db, ct);
log.WriteLine("# Class sheduling test initialized");
// run multiple students
var elapsed = await Program.RunConcurrentWorkersAsync(
STUDENTS,
(i, _ct) => IndecisiveStudent(db, i, OPS_PER_STUDENTS, _ct),
ct
);
log.WriteLine("# Ran {0} transactions in {1:0.0##} sec", (STUDENTS * OPS_PER_STUDENTS), elapsed.TotalSeconds);
}
public static async Task <FdbDirectorySubspace> GetCleanDirectory([NotNull] IFdbDatabase db, [NotNull] string[] path, CancellationToken ct)
{
Assert.That(db, Is.Not.Null, "null db");
Assert.That(path, Is.Not.Null.And.Length.GreaterThan(0), "invalid path");
// do not log
db = db.WithoutLogging();
// remove previous
await db.Directory.TryRemoveAsync(path, ct);
// create new
var subspace = await db.Directory.CreateAsync(path, ct);
Assert.That(subspace, Is.Not.Null);
Assert.That(db.GlobalSpace.Contains(subspace.Key), Is.True);
return(subspace);
}
public static async Task <List <KeyRange> > GetChunksAsync([NotNull] IFdbDatabase db, Slice beginInclusive, Slice endExclusive, CancellationToken ct)
{
//REVIEW: maybe rename this to SplitIntoChunksAsync or SplitIntoShardsAsync or GetFragmentsAsync ?
if (db == null)
{
throw new ArgumentNullException(nameof(db));
}
if (endExclusive < beginInclusive)
{
throw new ArgumentException("The end key cannot be less than the begin key", nameof(endExclusive));
}
var boundaries = await GetBoundaryKeysAsync(db, beginInclusive, endExclusive, ct).ConfigureAwait(false);
int count = boundaries.Count;
var chunks = new List <KeyRange>(count + 2);
if (count == 0)
{ // the range does not cross any boundary, and is contained in just one chunk
chunks.Add(new KeyRange(beginInclusive, endExclusive));
return(chunks);
}
var k = boundaries[0];
if (k != beginInclusive)
{
chunks.Add(new KeyRange(beginInclusive, k));
}
for (int i = 1; i < boundaries.Count; i++)
{
chunks.Add(new KeyRange(k, boundaries[i]));
k = boundaries[i];
}
if (k != endExclusive)
{
chunks.Add(new KeyRange(k, endExclusive));
}
return(chunks);
}
/// <summary>
/// Setup the initial state of the database
/// </summary>
public async Task Init(IFdbDatabase db, CancellationToken ct)
{
// open the folder where we will store everything
//this.Subspace = db.GlobalSpace;
var subspace = db.GlobalSpace;
await db.ReadWriteAsync(tr =>
{
tr.ClearRange(subspace);
// insert all the classes
foreach (var c in this.ClassNames)
{
tr.Set(ClassKey(c, subspace), Slice.FromStringAscii("100"));
}
Console.WriteLine("inserted range..");
return(subspace);
}, ct);
/*
* this.Subspace = await db.ReadWriteAsync(async tr =>
* {
* var subspace = db.GlobalSpace;
*
* // clear all previous values
* tr.ClearRange(subspace);
*
* // insert all the classes
* foreach (var c in this.ClassNames)
* {
* tr.Set(ClassKey(c), Slice.FromString("100"));
* }
*
* return subspace;
* }, ct);
*
*/
}
/// <summary>Create a new retry loop operation context</summary>
/// <param name="db">Database that will be used by the retry loop</param>
/// <param name="mode">Operation mode of the retry loop</param>
/// <param name="ct">Optional cancellation token that will abort the retry loop if triggered.</param>
public FdbOperationContext([NotNull] IFdbDatabase db, FdbTransactionMode mode, CancellationToken ct)
{
Contract.NotNull(db, nameof(db));
this.Database = db;
this.Mode = mode;
// note: we don't start the clock yet, only when the context starts executing...
// by default, we hook ourselves to the database's CancellationToken, but we may need to also
// hook with a different, caller-provided, token and respond to cancellation from both sites.
var token = db.Cancellation;
if (ct.CanBeCanceled && !ct.Equals(token))
{
this.TokenSource = CancellationTokenSource.CreateLinkedTokenSource(token, ct);
token = this.TokenSource.Token;
}
this.Cancellation = token;
}
private async Task RunTask(IFdbDatabase db, FdbWorkerMessage msg, Func <FdbWorkerMessage, CancellationToken, Task> handler, CancellationToken ct)
{
var sw = Stopwatch.StartNew();
try
{
Interlocked.Increment(ref m_workerTasksReceived);
//TODO: custom TaskScheduler for task execution ?
await handler(msg, ct).ConfigureAwait(false);
Interlocked.Increment(ref m_workerTasksCompleted);
}
finally
{
sw.Stop();
Interlocked.Add(ref m_workerBusyTime, sw.ElapsedTicks);
}
}
/// <summary>Run a write operation until it suceeds, timeouts, or fail with non-retryable error</summary>
public static Task RunWriteAsync([NotNull] IFdbDatabase db, [NotNull] Action <IFdbTransaction> handler, Action <IFdbTransaction> onDone, CancellationToken ct)
{
if (db == null)
{
throw new ArgumentNullException(nameof(db));
}
if (handler == null)
{
throw new ArgumentNullException(nameof(handler));
}
if (ct.IsCancellationRequested)
{
return(Task.FromCanceled(ct));
}
var context = new FdbOperationContext(db, FdbTransactionMode.Default | FdbTransactionMode.InsideRetryLoop, ct);
return(ExecuteInternal(db, context, handler, onDone));
}
/// <summary>Run a read/write operation until it succeeds, timeouts, or fails with a non retry-able error</summary>
public static async Task <TResult> RunWriteWithResultAsync <TResult>([NotNull] IFdbDatabase db, [NotNull] Func <IFdbTransaction, Task <TResult> > handler, CancellationToken ct)
{
Contract.NotNull(db, nameof(db));
Contract.NotNull(handler, nameof(handler));
ct.ThrowIfCancellationRequested();
TResult result = default;
async Task Handler(IFdbTransaction tr)
{
result = await handler(tr).ConfigureAwait(false);
}
var context = new FdbOperationContext(db, FdbTransactionMode.Default | FdbTransactionMode.InsideRetryLoop, ct);
await ExecuteInternal(db, context, (Func <IFdbTransaction, Task>) Handler, null).ConfigureAwait(false);
return(result);
}
/// <summary>Run a read/write operation until it suceeds, timeouts, or fail with non-retryable error</summary>
public static Task RunWriteAsync([NotNull] IFdbDatabase db, [NotNull] Func <IFdbTransaction, Task> asyncHandler, Action <IFdbTransaction> onDone, CancellationToken cancellationToken)
{
if (db == null)
{
throw new ArgumentNullException("db");
}
if (asyncHandler == null)
{
throw new ArgumentNullException("asyncHandler");
}
if (cancellationToken.IsCancellationRequested)
{
return(TaskHelpers.FromCancellation <object>(cancellationToken));
}
var context = new FdbOperationContext(db, FdbTransactionMode.Default | FdbTransactionMode.InsideRetryLoop, cancellationToken);
return(ExecuteInternal(db, context, asyncHandler, onDone));
}
/// <summary>Create a new High Contention counter, using a specific value encoder.</summary>
/// <param name="db">Database used by this layer</param>
/// <param name="subspace">Subspace to be used for storing the counter</param>
/// <param name="encoder">Encoder for the counter values</param>
public FdbHighContentionCounter(IFdbDatabase db, FdbSubspace subspace, IValueEncoder <long> encoder)
{
if (db == null)
{
throw new ArgumentNullException("db");
}
if (subspace == null)
{
throw new ArgumentNullException("subspace");
}
if (encoder == null)
{
throw new ArgumentNullException("encoder");
}
this.Database = db;
this.Subspace = subspace;
this.Encoder = encoder;
}
protected async Task DeleteSubspace(IFdbDatabase db, IFdbSubspace subspace)
{
using (var tr = db.BeginTransaction(this.Cancellation))
{
tr.ClearRange(subspace);
await tr.CommitAsync();
}
}
protected Task DumpSubspace(IFdbDatabase db, IFdbSubspace subspace)
{
return TestHelpers.DumpSubspace(db, subspace, this.Cancellation);
}
protected Task<FdbDirectorySubspace> GetCleanDirectory(IFdbDatabase db, params string[] path)
{
return TestHelpers.GetCleanDirectory(db, path, this.Cancellation);
}
public static async Task ShowDirectoryLayer(string[] path, IFdbTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
var dir = await BasicCommands.TryOpenCurrentDirectoryAsync(path, db, ct);
if (dir == null)
{
log.WriteLine("# Directory {0} does not exist anymore", String.Join("/", path));
}
else
{
if (dir.Layer == FdbDirectoryPartition.LayerId)
log.WriteLine("# Directory {0} is a partition", String.Join("/", path));
else if (dir.Layer.IsPresent)
log.WriteLine("# Directory {0} has layer {1}", String.Join("/", path), dir.Layer.ToAsciiOrHexaString());
else
log.WriteLine("# Directory {0} does not have a layer defined", String.Join("/", path));
}
}
/// <summary>Performs (x OP y) and ensure that the result is correct</summary>
private async Task PerformAtomicOperationAndCheck(IFdbDatabase db, Slice key, int x, FdbMutationType type, int y)
{
int expected = 0;
switch(type)
{
case FdbMutationType.BitAnd: expected = x & y; break;
case FdbMutationType.BitOr: expected = x | y; break;
case FdbMutationType.BitXor: expected = x ^ y; break;
case FdbMutationType.Add: expected = x + y; break;
default: Assert.Fail("Invalid operation type"); break;
}
// set key = x
using (var tr = db.BeginTransaction(this.Cancellation))
{
tr.Set(key, Slice.FromFixed32(x));
await tr.CommitAsync();
}
// atomic key op y
using (var tr = db.BeginTransaction(this.Cancellation))
{
tr.Atomic(key, Slice.FromFixed32(y), type);
await tr.CommitAsync();
}
// read key
using (var tr = db.BeginTransaction(this.Cancellation))
{
var data = await tr.GetAsync(key);
Assert.That(data.Count, Is.EqualTo(4), "data.Count");
Assert.That(data.ToInt32(), Is.EqualTo(expected), "0x{0} {1} 0x{2} = 0x{3}", x.ToString("X8"), type.ToString(), y.ToString("X8"), expected.ToString("X8"));
}
}
/// <summary>Creates a new directory</summary>
public static async Task CreateDirectory(string[] path, IFdbTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
if (log == null) log = Console.Out;
string layer = extras.Count > 0 ? extras.Get<string>(0) : null;
log.WriteLine("# Creating directory {0} with layer '{1}'", String.Join("/", path), layer);
var folder = await db.Directory.TryOpenAsync(path, cancellationToken: ct);
if (folder != null)
{
log.WriteLine("- Directory {0} already exists!", string.Join("/", path));
return;
}
folder = await db.Directory.TryCreateAsync(path, Slice.FromString(layer), cancellationToken: ct);
log.WriteLine("- Created under {0} [{1}]", FdbKey.Dump(folder.Key), folder.Key.ToHexaString(' '));
// look if there is already stuff under there
var stuff = await db.ReadAsync((tr) => tr.GetRange(folder.ToRange()).FirstOrDefaultAsync(), cancellationToken: ct);
if (stuff.Key.IsPresent)
{
log.WriteLine("CAUTION: There is already some data under {0} !");
log.WriteLine(" {0} = {1}", FdbKey.Dump(stuff.Key), stuff.Value.ToAsciiOrHexaString());
}
}
public static async Task Shards(string[] path, IFdbTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
var ranges = await Fdb.System.GetChunksAsync(db, FdbKey.MinValue, FdbKey.MaxValue, ct);
Console.WriteLine("Found {0} shards in the whole cluster", ranges.Count);
// look if there is something under there
var folder = (await TryOpenCurrentDirectoryAsync(path, db, ct)) as FdbDirectorySubspace;
if (folder != null)
{
var r = FdbKeyRange.StartsWith(folder.Copy().Key);
Console.WriteLine("Searching for shards that intersect with /{0} ...", String.Join("/", path));
ranges = await Fdb.System.GetChunksAsync(db, r, ct);
Console.WriteLine("Found {0} ranges intersecting {1}:", ranges.Count, r);
var last = Slice.Empty;
foreach (var range in ranges)
{
Console.Write("> " + FdbKey.Dump(range.Begin) + " ...");
long count = await Fdb.System.EstimateCountAsync(db, range, ct);
Console.WriteLine(" {0:N0}", count);
last = range.End;
//TODO: we can probably get more details on this shard looking in the system keyspace (where it is, how many replicas, ...)
}
Console.WriteLine("> ... " + FdbKey.Dump(last));
}
//Console.WriteLine("Found " + ranges.Count + " shards in the cluster");
//TODO: shards that intersect the current directory
}
/// <summary>Find the DCs, machines and processes in the cluster</summary>
public static async Task Topology(string[] path, IFdbTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
var coords = await Fdb.System.GetCoordinatorsAsync(db, ct);
log.WriteLine("[Cluster] {0}", coords.Id);
var servers = await db.QueryAsync(tr => tr
.WithReadAccessToSystemKeys()
.GetRange(FdbKeyRange.StartsWith(Fdb.System.ServerList))
.Select(kvp => new
{
// Offsets Size Type Name Description
// 0 2 Word Version? 0100 (1.0 ?)
// 2 4 DWord ??? 0x00 0x20 0xA2 0x00
// 6 2 Word FDBMagic 0xDB 0x0F "FDB"
// 8 16 Guid NodeId Unique Process ID
// 24 16 Guid Machine "machine_id" field in foundationdb.conf (ends with 8x0 if manually specified)
// 40 16 Guid DataCenter "datacenter_id" field in foundationdb.conf (ends with 8x0 if manually specified)
// 56 4 ??? ?? 4 x 0
// 60 12 x24 ARRAY[] ?? array of 12x the same 24-byte struct defined below
// ...0 4 DWord IPAddress 01 00 00 7F => 127.0.0.1
// ...4 4 DWord Port 94 11 00 00 -> 4500
// ...8 4 DWord ?? randomish, changes every reboot
// ..12 4 DWord ?? randomish, changes every reboot
// ..16 4 DWord Size? small L-E integer, usually between 0x20 and 0x40...
// ..20 4 DWord ?? randmoish, changes every reboot
ProcessId = kvp.Value.Substring(8, 16).ToHexaString(),
MachineId = kvp.Value.Substring(24, 16).ToHexaString(),
DataCenterId = kvp.Value.Substring(40, 16).ToHexaString(),
Parts = Enumerable.Range(0, 12).Select(i =>
{
int p = 60 + 24 * i;
return new
{
Address = new IPAddress(kvp.Value.Substring(p, 4).GetBytes().Reverse().ToArray()),
Port = kvp.Value.Substring(p + 4, 4).ToInt32(),
Unknown1 = kvp.Value.Substring(p + 8, 4).ToInt32(),
Unknown2 = kvp.Value.Substring(p + 12, 4).ToInt32(),
Unknown3 = kvp.Value.Substring(p + 16, 4).ToInt32(),
Unknown4 = kvp.Value.Substring(p + 20, 4).ToInt32(),
};
}).ToList(),
Raw = kvp.Value,
}),
ct
);
var numNodes = servers.Select(s => s.ProcessId).Distinct().Count();
var numMachines = servers.Select(s => s.MachineId).Distinct().Count();
var numDCs = servers.Select(s => s.DataCenterId).Distinct().Count();
var dcs = servers.GroupBy(x => x.DataCenterId).ToArray();
for (int dcIndex = 0; dcIndex < dcs.Length;dcIndex++)
{
var dc = dcs[dcIndex];
bool lastDc = dcIndex == dcs.Length - 1;
string dcId = dc.Key.EndsWith("0000000000000000") ? dc.Key.Substring(0, 16) : dc.Key;
log.WriteLine((lastDc ? "`- " : "|- ") + "[DataCenter] {0} (#{1})", dcId, dcIndex);
var machines = dc.GroupBy(x => x.MachineId).ToArray();
string dcPrefix = lastDc ? " " : "| ";
for (int machineIndex = 0; machineIndex < machines.Length; machineIndex++)
{
var machine = machines[machineIndex];
var lastMachine = machineIndex == machines.Length - 1;
string machineId = machine.Key.EndsWith("0000000000000000") ? machine.Key.Substring(0, 16) : machine.Key;
log.WriteLine(dcPrefix + (lastMachine ? "`- " : "|- ") + "[Machine] {0}, {1}", machine.First().Parts[0].Address, machineId);
var procs = machine.ToArray();
string machinePrefix = dcPrefix + (lastMachine ? " " : "| ");
for (int procIndex = 0; procIndex < procs.Length; procIndex++)
{
var proc = procs[procIndex];
bool lastProc = procIndex == procs.Length - 1;
log.WriteLine(machinePrefix + (lastProc ? "`- " : "|- ") + "[Process] {0}:{1}, {2}", proc.Parts[0].Address, proc.Parts[0].Port, proc.ProcessId);
//foreach (var part in proc.Parts)
//{
// log.WriteLine(machinePrefix + "| -> {0}, {1}, {2:X8}, {3:X8}, {4}, {5:X8}", part.Address, part.Port, part.Unknown1, part.Unknown2, part.Unknown3, part.Unknown4);
//}
}
}
}
log.WriteLine();
log.WriteLine("Found {0} process(es) on {1} machine(s) in {2} datacenter(s)", numNodes, numMachines, numDCs);
log.WriteLine();
}
public static async Task ChangeDirectoryLayer(string[] path, string layer, IFdbTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
var dir = await BasicCommands.TryOpenCurrentDirectoryAsync(path, db, ct);
if (dir == null)
{
log.WriteLine("# Directory {0} does not exist anymore", String.Join("/", path));
}
else
{
dir = await db.ReadWriteAsync((tr) => dir.ChangeLayerAsync(tr, Slice.FromString(layer)), ct);
log.WriteLine("# Directory {0} layer changed to {1}", String.Join("/", path), dir.Layer.ToAsciiOrHexaString());
}
}
/// <summary>Counts the number of keys inside a directory</summary>
public static async Task Count(string[] path, IFdbTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
// look if there is something under there
var folder = (await TryOpenCurrentDirectoryAsync(path, db, ct)) as FdbDirectorySubspace;
if (folder == null)
{
log.WriteLine("# Directory {0} does not exist", String.Join("/", path));
return;
}
var copy = folder.Copy();
log.WriteLine("# Counting keys under {0} ...", FdbKey.Dump(copy.Key));
var progress = new Progress<FdbTuple<long, Slice>>((state) =>
{
log.Write("\r# Found {0:N0} keys...", state.Item1);
});
long count = await Fdb.System.EstimateCountAsync(db, copy.ToRange(), progress, ct);
log.WriteLine("\r# Found {0:N0} keys in {1}", count, folder.FullName);
}
/// <summary>Run the worker loop</summary>
public async Task RunWorkerAsync(IFdbDatabase db, Func<FdbWorkerMessage, CancellationToken, Task> handler, CancellationToken ct)
{
int num = Interlocked.Increment(ref counter);
Slice workerId = Slice.Nil;
Slice previousTaskId = Slice.Nil;
FdbWatch watch = default(FdbWatch);
FdbWorkerMessage msg = null;
Interlocked.Increment(ref m_workers);
try
{
while (true)
{
//TODO: how do we clear the previousTaskId from the db in case of cancellation ?
ct.ThrowIfCancellationRequested();
Slice myId = Slice.Nil;
await db.ReadWriteAsync(
async (tr) =>
{
tr.Annotate("I'm worker #{0} with id {1}", num, workerId.ToAsciiOrHexaString());
myId = workerId;
watch = default(FdbWatch);
msg = new FdbWorkerMessage();
if (previousTaskId != null)
{ // we need to clean up the previous task
ClearTask(tr, previousTaskId);
}
else if (myId.IsPresent)
{ // look for an already assigned task
tr.Annotate("Look for already assigned task");
msg.Id = await tr.GetAsync(this.BusyRing.Pack(myId)).ConfigureAwait(false);
}
if (!msg.Id.IsPresent)
{ // We aren't already assigned a task, so get an item from a random queue
tr.Annotate("Look for next queued item");
// Find the next task on the queue
var item = await tr.GetRange(this.UnassignedTaskRing.ToRange()).FirstOrDefaultAsync().ConfigureAwait(false);
if (item.Key != null)
{ // pop the Task from the queue
msg.Id = item.Value;
tr.Clear(item.Key);
}
if (msg.Id.IsPresent)
{ // mark this worker as busy
// note: we need a random id so generate one if it is the first time...
if (!myId.IsPresent) myId = GetRandomId();
tr.Annotate("Found {0}, switch to busy with id {1}", msg.Id.ToAsciiOrHexaString(), myId.ToAsciiOrHexaString());
tr.Set(this.BusyRing.Pack(myId), msg.Id);
this.Counters.Increment(tr, COUNTER_BUSY);
}
else if (myId.IsPresent)
{ // remove ourselves from the busy ring
tr.Annotate("Found nothing, switch to idle with id {0}", myId.ToAsciiOrHexaString());
//tr.Clear(this.BusyRing.Pack(myId));
}
}
if (msg.Id.IsPresent)
{ // get the task body
tr.Annotate("Fetching body for task {0}", msg.Id.ToAsciiOrHexaString());
var prefix = this.TaskStore.Partition(msg.Id);
//TODO: replace this with a get_range ?
var data = await tr.GetValuesAsync(new [] {
prefix.Key,
prefix.Pack(TASK_META_SCHEDULED)
}).ConfigureAwait(false);
msg.Body = data[0];
msg.Scheduled = new DateTime(data[1].ToInt64(), DateTimeKind.Utc);
msg.Received = DateTime.UtcNow;
}
else
{ // There are no unassigned task, so enter the idle_worker_ring and wait for a task to be asssigned to us
// remove us from the busy ring
if (myId.IsPresent)
{
tr.Clear(this.BusyRing.Pack(myId));
this.Counters.Decrement(tr, COUNTER_BUSY);
}
// choose a new random position on the idle ring
myId = GetRandomId();
// the idle key will also be used as the watch key to wake us up
var watchKey = this.IdleRing.Pack(myId);
tr.Annotate("Will start watching on key {0} with id {1}", watchKey.ToAsciiOrHexaString(), myId.ToAsciiOrHexaString());
tr.Set(watchKey, Slice.Empty);
this.Counters.Increment(tr, COUNTER_IDLE);
watch = tr.Watch(watchKey, ct);
}
},
onDone: (tr) =>
{ // we have successfully acquired some work, or got a watch
previousTaskId = Slice.Nil;
workerId = myId;
},
cancellationToken: ct
).ConfigureAwait(false);
if (msg.Id.IsNullOrEmpty)
{ // wait for someone to wake us up...
Interlocked.Increment(ref m_idleWorkers);
try
{
await watch.Task;
//Console.WriteLine("Worker #" + num + " woken up!");
}
finally
{
Interlocked.Decrement(ref m_idleWorkers);
}
}
else
{
//Console.WriteLine("Got task " + taskId);
previousTaskId = msg.Id;
if (msg.Body.IsNull)
{ // the task has been dropped?
// TODO: loggin?
Console.WriteLine("[####] Task[" + msg.Id.ToAsciiOrHexaString() + "] has vanished?");
}
else
{
try
{
await RunTask(db, msg, handler, ct);
}
catch (Exception e)
{
//TODO: logging?
Console.Error.WriteLine("Task[" + msg.Id.ToAsciiOrHexaString() + "] failed: " + e.ToString());
}
}
}
}
}
finally
{
//TODO: we should ensure that the last processed task is properly removed from the db before leaving !
Interlocked.Decrement(ref m_workers);
}
}
/// <summary>Move/Rename a directory</summary>
public static async Task MoveDirectory(string[] srcPath, string[] dstPath, IFdbTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
var folder = await db.Directory.TryOpenAsync(srcPath, cancellationToken: ct);
if (folder == null)
{
log.WriteLine("# Source directory {0} does not exist!", string.Join("/", srcPath));
return;
}
folder = await db.Directory.TryOpenAsync(dstPath, cancellationToken: ct);
if (folder != null)
{
log.WriteLine("# Destination directory {0} already exists!", string.Join("/", dstPath));
return;
}
await db.Directory.MoveAsync(srcPath, dstPath, ct);
Console.WriteLine("Moved {0} to {1}", string.Join("/", srcPath), string.Join("/", dstPath));
}
public static async Task Sampling(string[] path, IFdbTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
double ratio = 0.1d;
bool auto = true;
if (extras.Count > 0)
{
double x = extras.Get<double>(0);
if (x > 0 && x <= 1) ratio = x;
auto = false;
}
var folder = await TryOpenCurrentDirectoryAsync(path, db, ct);
FdbKeyRange span;
if (folder is FdbDirectorySubspace)
{
span = FdbKeyRange.StartsWith((folder as FdbDirectorySubspace).Copy());
log.WriteLine("Reading list of shards for /{0} under {1} ...", String.Join("/", path), FdbKey.Dump(span.Begin));
}
else
{
log.WriteLine("Reading list of shards for the whole cluster ...");
span = FdbKeyRange.All;
}
// dump keyServers
var ranges = await Fdb.System.GetChunksAsync(db, span, ct);
log.WriteLine("> Found {0:N0} shard(s)", ranges.Count);
// take a sample
var samples = new List<FdbKeyRange>();
if (ranges.Count <= 32)
{ // small enough to scan it all
samples.AddRange(ranges);
log.WriteLine("Sampling all {0:N0} shards ...", samples.Count);
}
else
{ // need to take a random subset
var rnd = new Random();
int sz = Math.Max((int)Math.Ceiling(ratio * ranges.Count), 1);
if (auto)
{
if (sz > 100) sz = 100; //SAFETY
if (sz < 32) sz = Math.Max(sz, Math.Min(32, ranges.Count));
}
var population = new List<FdbKeyRange>(ranges);
for (int i = 0; i < sz; i++)
{
int p = rnd.Next(population.Count);
samples.Add(population[p]);
population.RemoveAt(p);
}
log.WriteLine("Sampling " + samples.Count + " out of " + ranges.Count + " shards (" + (100.0 * samples.Count / ranges.Count).ToString("N1") + "%) ...");
}
log.WriteLine();
const string FORMAT_STRING = "{0,9} ║{1,10}{6,6} {2,-29} ║{3,10}{7,7} {4,-37} ║{5,10}";
const string SCALE_KEY = "....--------========########M";
const string SCALE_VAL = "....--------========########@@@@@@@@M";
log.WriteLine(FORMAT_STRING, "Count", "Keys", SCALE_KEY, "Values", SCALE_VAL, "Total", "med.", "med.");
var rangeOptions = new FdbRangeOptions { Mode = FdbStreamingMode.WantAll };
samples = samples.OrderBy(x => x.Begin).ToList();
long globalSize = 0;
long globalCount = 0;
int workers = 8; // Math.Max(4, Environment.ProcessorCount);
var sw = Stopwatch.StartNew();
var tasks = new List<Task>();
int n = samples.Count;
while (samples.Count > 0)
{
while (tasks.Count < workers && samples.Count > 0)
{
var range = samples[0];
samples.RemoveAt(0);
tasks.Add(Task.Run(async () =>
{
var kk = new RobustHistogram(RobustHistogram.TimeScale.Ticks);
var vv = new RobustHistogram(RobustHistogram.TimeScale.Ticks);
#region Method 1: get_range everything...
using (var tr = db.BeginTransaction(ct))
{
long keySize = 0;
long valueSize = 0;
long count = 0;
int iter = 0;
var beginSelector = FdbKeySelector.FirstGreaterOrEqual(range.Begin);
var endSelector = FdbKeySelector.FirstGreaterOrEqual(range.End);
while (true)
{
FdbRangeChunk data = default(FdbRangeChunk);
FdbException error = null;
try
{
data = await tr.Snapshot.GetRangeAsync(
beginSelector,
endSelector,
rangeOptions,
iter
).ConfigureAwait(false);
}
catch (FdbException e)
{
error = e;
}
if (error != null)
{
await tr.OnErrorAsync(error.Code).ConfigureAwait(false);
continue;
}
if (data.Count == 0) break;
count += data.Count;
foreach (var kvp in data.Chunk)
{
keySize += kvp.Key.Count;
valueSize += kvp.Value.Count;
kk.Add(TimeSpan.FromTicks(kvp.Key.Count));
vv.Add(TimeSpan.FromTicks(kvp.Value.Count));
}
if (!data.HasMore) break;
beginSelector = FdbKeySelector.FirstGreaterThan(data.Last.Key);
++iter;
}
long totalSize = keySize + valueSize;
Interlocked.Add(ref globalSize, totalSize);
Interlocked.Add(ref globalCount, count);
lock (log)
{
log.WriteLine(FORMAT_STRING, count.ToString("N0"), FormatSize(keySize), kk.GetDistribution(begin: 1, end: 12000, fold: 2), FormatSize(valueSize), vv.GetDistribution(begin: 1, end: 120000, fold: 2), FormatSize(totalSize), FormatSize((int)Math.Ceiling(kk.Median)), FormatSize((int)Math.Ceiling(vv.Median)));
}
}
#endregion
#region Method 2: estimate the count using key selectors...
//long counter = await Fdb.System.EstimateCountAsync(db, range, ct);
//Console.WriteLine("COUNT = " + counter.ToString("N0"));
#endregion
}, ct));
}
var done = await Task.WhenAny(tasks);
tasks.Remove(done);
}
await Task.WhenAll(tasks);
sw.Stop();
log.WriteLine();
if (n != ranges.Count)
{
log.WriteLine("Sampled " + FormatSize(globalSize) + " (" + globalSize.ToString("N0") + " bytes) and " + globalCount.ToString("N0") + " keys in " + sw.Elapsed.TotalSeconds.ToString("N1") + " sec");
log.WriteLine("> Estimated total size is " + FormatSize(globalSize * ranges.Count / n));
}
else
{
log.WriteLine("Found " + FormatSize(globalSize) + " (" + globalSize.ToString("N0") + " bytes) and " + globalCount.ToString("N0") + " keys in " + sw.Elapsed.TotalSeconds.ToString("N1") + " sec");
// compare to the whole cluster
ranges = await Fdb.System.GetChunksAsync(db, FdbKey.MinValue, FdbKey.MaxValue, ct);
log.WriteLine("> This directory contains ~{0:N2}% of all data", (100.0 * n / ranges.Count));
}
log.WriteLine();
}
private static async Task TreeDirectoryWalk(FdbDirectorySubspace folder, List<bool> last, IFdbDatabase db, TextWriter stream, CancellationToken ct)
{
ct.ThrowIfCancellationRequested();
var sb = new StringBuilder(last.Count * 4);
if (last.Count > 0)
{
for (int i = 0; i < last.Count - 1; i++) sb.Append(last[i] ? " " : "| ");
sb.Append(last[last.Count - 1] ? "`-- " : "|-- ");
}
IFdbDirectory node;
if (folder == null)
{
stream.WriteLine(sb.ToString() + "<root>");
node = db.Directory;
}
else
{
stream.WriteLine(sb.ToString() + (folder.Layer.ToString() == "partition" ? ("<" + folder.Name + ">") : folder.Name) + (folder.Layer.IsNullOrEmpty ? String.Empty : (" [" + folder.Layer.ToString() + "]")));
node = folder;
}
var children = await Fdb.Directory.BrowseAsync(db, node, ct);
int n = children.Count;
foreach (var child in children)
{
last.Add((n--) == 1);
await TreeDirectoryWalk(child.Value, last, db, stream, ct);
last.RemoveAt(last.Count - 1);
}
}
/// <summary>Shows the first few keys of a directory</summary>
public static async Task Show(string[] path, IFdbTuple extras, bool reverse, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
int count = 20;
if (extras.Count > 0)
{
int x = extras.Get<int>(0);
if (x > 0) count = x;
}
// look if there is something under there
var folder = await db.Directory.TryOpenAsync(path, cancellationToken: ct);
if (folder != null)
{
log.WriteLine("# Content of {0} [{1}]", FdbKey.Dump(folder.Key), folder.Key.ToHexaString(' '));
var keys = await db.QueryAsync((tr) =>
{
var query = tr.GetRange(folder.ToRange());
return reverse
? query.Reverse().Take(count)
: query.Take(count + 1);
}, cancellationToken: ct);
if (keys.Count > 0)
{
if (reverse) keys.Reverse();
foreach (var key in keys.Take(count))
{
log.WriteLine("...{0} = {1}", FdbKey.Dump(folder.Extract(key.Key)), key.Value.ToAsciiOrHexaString());
}
if (!reverse && keys.Count == count + 1)
{
log.WriteLine("... more");
}
}
else
{
log.WriteLine(" no content found");
}
}
}
private async Task RunTask(IFdbDatabase db, FdbWorkerMessage msg, Func<FdbWorkerMessage, CancellationToken, Task> handler, CancellationToken ct)
{
var sw = Stopwatch.StartNew();
try
{
Interlocked.Increment(ref m_workerTasksReceived);
//TODO: custom TaskScheduler for task execution ?
await handler(msg, ct).ConfigureAwait(false);
Interlocked.Increment(ref m_workerTasksCompleted);
}
finally
{
sw.Stop();
Interlocked.Add(ref m_workerBusyTime, sw.ElapsedTicks);
}
}
public static async Task Map(string[] path, IFdbTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
// we want to merge the map of shards, with the map of directories from the Directory Layer, and count for each directory how many shards intersect
var folder = await TryOpenCurrentDirectoryAsync(path, db, ct);
if (folder == null)
{
log.WriteLine("# Directory not found");
return;
}
var span = folder.DirectoryLayer.ContentSubspace.ToRange();
// note: this may break in future versions of the DL! Maybe we need a custom API to get a flat list of all directories in a DL that span a specific range ?
var shards = await Fdb.System.GetChunksAsync(db, span, ct);
int totalShards = shards.Count;
log.WriteLine("Found {0} shard(s) in partition /{1}", totalShards, folder.DirectoryLayer.FullName);
log.WriteLine("Listing all directories...");
var map = new Dictionary<string, int>(StringComparer.Ordinal);
Action<string[], int> account = (p, c) =>
{
for (int i = 1; i <= p.Length; i++)
{
var s = "/" + String.Join("/", p, 0, i);
int x;
map[s] = map.TryGetValue(s, out x) ? (x + c) : c;
}
};
var work = new Stack<IFdbDirectory>();
work.Push(folder);
var dirs = new List<IFdbDirectory>();
int n = 0;
while(work.Count > 0)
{
var cur = work.Pop();
// skip sub partitions
var names = await cur.ListAsync(db, ct);
foreach(var name in names)
{
var sub = await cur.TryOpenAsync(db, name, ct);
if (sub != null)
{
var p = sub.FullName;
if (sub is FdbDirectoryPartition)
{
log.WriteLine("\r! Skipping partition {0} ", sub.Name);
n = 0;
continue;
}
log.Write("\r/{0}{1}", p, p.Length > n ? String.Empty : new string(' ', n - p.Length));
n = p.Length;
work.Push(sub);
dirs.Add(sub);
}
}
}
log.Write("\r" + new string(' ', n + 2));
log.WriteLine("\r> Found {0} sub-directories", dirs.Count);
log.WriteLine();
log.WriteLine("Estimating size of each directory...");
int foundShards = 0;
n = 0;
int max = 0;
IFdbDirectory bigBad = null;
foreach (var dir in dirs)
{
log.Write("\r> {0}{1}", dir.Name, dir.Name.Length > n ? String.Empty : new string(' ', n - dir.Name.Length));
n = dir.Name.Length;
var p = dir.Path.ToArray();
var key = ((FdbSubspace)dir).Key;
// verify that the subspace has at least one key inside
var bounds = await db.ReadAsync(async (tr) =>
{
var kvs = await Task.WhenAll(
tr.GetRange(FdbKeyRange.StartsWith(key)).FirstOrDefaultAsync(),
tr.GetRange(FdbKeyRange.StartsWith(key)).LastOrDefaultAsync()
);
return new { Min = kvs[0].Key, Max = kvs[1].Key };
}, ct);
if (bounds.Min.HasValue)
{ // folder is not empty
shards = await Fdb.System.GetChunksAsync(db, FdbKeyRange.StartsWith(key), ct);
//TODO: we still need to check if the first and last shard really intersect the subspace
// we need to check if the shards actually contain data
//Console.WriteLine("/{0} under {1} with {2} shard(s)", string.Join("/", p), FdbKey.Dump(key), shards.Count);
foundShards += shards.Count;
account(p, shards.Count);
if (shards.Count > max) { max = shards.Count; bigBad = dir; }
}
else
{
account(p, 0);
}
}
log.Write("\r" + new string(' ', n + 2));
log.WriteLine("\rFound a total of {0} shard(s) in {1} folder(s)", foundShards, dirs.Count);
log.WriteLine();
log.WriteLine("Shards %Total Path");
foreach(var kvp in map.OrderBy(x => x.Key))
{
log.WriteLine("{0,6} {1,-20} {2}", kvp.Value, RobustHistogram.FormatHistoBar((double)kvp.Value / foundShards, 20), kvp.Key);
}
log.WriteLine();
if (bigBad != null)
{
log.WriteLine("Biggest folder is /{0} with {1} shards ({2:N1}% total, {3:N1}% subtree)", bigBad.FullName, max, 100.0 * max / totalShards, 100.0 * max / foundShards);
log.WriteLine();
}
}
/// <summary>Remove a directory and all its data</summary>
public static async Task RemoveDirectory(string[] path, IFdbTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
if (log == null) log = Console.Out;
string layer = extras.Count > 0 ? extras.Get<string>(0) : null;
var folder = await db.Directory.TryOpenAsync(path, cancellationToken: ct);
if (folder == null)
{
log.WriteLine("# Directory {0} does not exist", string.Join("/", path));
return;
}
// are there any subdirectories ?
var subDirs = await folder.TryListAsync(db, ct);
if (subDirs != null && subDirs.Count > 0)
{
//TODO: "-r" flag ?
log.WriteLine("# Cannot remove {0} because it still contains {1} sub-directorie(s)", string.Join("/", path), subDirs.Count);
}
//TODO: ask for confirmation?
log.WriteLine("# Deleting directory {0} ...", String.Join("/", path));
await folder.RemoveAsync(db, ct);
log.WriteLine("# Gone!");
}
/// <summary>Display a tree of a directory's children</summary>
public static async Task Tree(string[] path, IFdbTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
if (log == null) log = Console.Out;
log.WriteLine("# Tree of {0}:", String.Join("/", path));
FdbDirectorySubspace root = null;
if (path.Length > 0) root = await db.Directory.TryOpenAsync(path, cancellationToken: ct);
await TreeDirectoryWalk(root, new List<bool>(), db, log, ct);
log.WriteLine("# done");
}
