private static async Task <string[]> AutoCompleteDirectories(FdbPath path, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
var parent = await db.ReadAsync(tr => BasicCommands.TryOpenCurrentDirectoryAsync(tr, path), ct);
if (parent == null)
{
return(null);
}
var paths = await db.ReadAsync(tr => parent.ListAsync(tr), ct);
return(paths.Select(p => p.Name).ToArray());
}
public static async Task <Dictionary <string, FdbDirectorySubspace> > BrowseAsync([NotNull] IFdbDatabase db, [NotNull] IFdbDirectory parent, CancellationToken cancellationToken)
{
if (db == null)
{
throw new ArgumentNullException("db");
}
if (parent == null)
{
throw new ArgumentNullException("parent");
}
return(await db.ReadAsync(async (tr) =>
{
// read the names of all the subdirectories
var names = await parent.ListAsync(tr).ConfigureAwait(false);
// open all the subdirectories
var folders = await names
.ToAsyncEnumerable()
.SelectAsync((name, ct) => parent.OpenAsync(tr, name))
.ToListAsync();
// map the result
return folders.ToDictionary(ds => ds.Name);
}, cancellationToken).ConfigureAwait(false));
}
/// <summary>Creates a new directory</summary>
public static async Task CreateDirectory(string[] path, IFdbTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
if (log == null)
{
log = Console.Out;
}
string layer = extras.Count > 0 ? extras.Get <string>(0) : null;
log.WriteLine("# Creating directory {0} with layer '{1}'", String.Join("/", path), layer);
var folder = await db.Directory.TryOpenAsync(path, cancellationToken : ct);
if (folder != null)
{
log.WriteLine("- Directory {0} already exists!", string.Join("/", path));
return;
}
folder = await db.Directory.TryCreateAsync(path, Slice.FromString(layer), cancellationToken : ct);
log.WriteLine("- Created under {0} [{1}]", FdbKey.Dump(folder.Key), folder.Key.ToHexaString(' '));
// look if there is already stuff under there
var stuff = await db.ReadAsync((tr) => tr.GetRange(folder.ToRange()).FirstOrDefaultAsync(), cancellationToken : ct);
if (stuff.Key.IsPresent)
{
log.WriteLine("CAUTION: There is already some data under {0} !");
log.WriteLine(" {0} = {1}", FdbKey.Dump(stuff.Key), stuff.Value.ToAsciiOrHexaString());
}
}
/// <summary>Creates a new directory</summary>
public static async Task CreateDirectory(string[] path, IVarTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
if (log == null)
{
log = Console.Out;
}
string layer = extras.Count > 0 ? extras.Get <string>(0) : null;
log.WriteLine($"# Creating directory {String.Join("/", path)} with layer '{layer}'");
var folder = await db.Directory.TryOpenAsync(db, path, ct : ct);
if (folder != null)
{
log.WriteLine($"- Directory {string.Join("/", path)} already exists!");
return;
}
folder = await db.Directory.TryCreateAsync(db, path, Slice.FromString(layer), ct : ct);
log.WriteLine($"- Created under {FdbKey.Dump(folder.GetPrefix())} [{folder.GetPrefix().ToHexaString(' ')}]");
// look if there is already stuff under there
var stuff = await db.ReadAsync((tr) => tr.GetRange(folder.Keys.ToRange()).FirstOrDefaultAsync(), ct : ct);
if (stuff.Key.IsPresent)
{
Program.Error(log, $"CAUTION: There is already some data under {string.Join("/", path)} !");
Program.Error(log, $" {FdbKey.Dump(stuff.Key)} = {stuff.Value:V}");
}
}
/// <summary>Return the current value of the metadata version of the database.</summary>
/// <remarks>
/// Please note that by the time the value has been read, it may have already changed in the database!
/// It is highly recommended to read the key as part as the same transaction that would read or update any metadata.
/// This method requires API version 610 or greater.
/// </remarks>
public static Task <Slice> GetMetadataVersionAsync([NotNull] IFdbDatabase db, CancellationToken ct)
{
Contract.NotNull(db, nameof(db));
if (Fdb.ApiVersion < 610)
{
throw new NotSupportedException($"The metadata version system key is only available on version 6.1 or greater. Your application has selected API version {Fdb.ApiVersion} which is too low. You will need to select API version 610 or greater.");
}
return(db.ReadAsync(tr => tr.GetAsync(System.MetadataVersion), ct));
}
public static Task <List <Slice> > GetBoundaryKeysAsync([NotNull] IFdbDatabase db, Slice beginInclusive, Slice endExclusive, CancellationToken ct)
{
if (db == null)
{
throw new ArgumentNullException(nameof(db));
}
return(db.ReadAsync((trans) => GetBoundaryKeysInternalAsync(trans, beginInclusive, endExclusive), ct));
}
public static Task <FdbSystemStatus?> GetStatusAsync(IFdbDatabase db, CancellationToken ct)
{
Contract.NotNull(db);
// we should not retry the read to the status key!
return(db.ReadAsync(tr =>
{
tr.WithPrioritySystemImmediate();
//note: in v3.x, the status key does not need the access to system key option.
//TODO: set a custom timeout?
return GetStatusAsync(tr);
}, ct));
}
/// <summary>Return the value of a configuration parameter (located under '\xFF/conf/')</summary>
/// <param name="db">Database to use for the operation</param>
/// <param name="name">Name of the configuration key (ex: "storage_engine")</param>
/// <param name="ct">Token used to cancel the operation</param>
/// <returns>Value of '\xFF/conf/storage_engine'</returns>
public static Task <Slice> GetConfigParameterAsync([NotNull] IFdbDatabase db, [NotNull] string name, CancellationToken ct)
{
Contract.NotNull(db, nameof(db));
Contract.NotNullOrEmpty(name, nameof(name), "Configuration parameter name cannot be null or empty.");
return(db.ReadAsync <Slice>((tr) =>
{
tr.WithReadAccessToSystemKeys();
tr.WithPrioritySystemImmediate();
//note: we ask for high priority, because this method maybe called by a monitoring system than has to run when the cluster is clogged up in requests
return tr.GetAsync(Fdb.System.ConfigKey(name));
}, ct));
}
/// <summary>Creates a new directory</summary>
public static async Task CreateDirectory(FdbPath path, IVarTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
if (log == null)
{
log = Console.Out;
}
string layer = extras.Count > 0 ? extras.Get <string>(0).Trim() : string.Empty;
if (path.LayerId != layer)
{
path.WithLayer(layer);
}
log.WriteLine($"# Creating directory {path}");
(var prefix, var created) = await db.ReadWriteAsync(async tr =>
{
var folder = await db.DirectoryLayer.TryOpenAsync(tr, path);
if (folder != null)
{
return(folder.GetPrefix(), false);
}
folder = await db.DirectoryLayer.TryCreateAsync(tr, path);
return(folder.Copy().GetPrefix(), true);
}, ct);
if (!created)
{
log.WriteLine($"- Directory {path} already exists at {FdbKey.Dump(prefix)} [{prefix.ToHexaString(' ')}]");
return;
}
log.WriteLine($"- Created under {FdbKey.Dump(prefix)} [{prefix.ToHexaString(' ')}]");
// look if there is already stuff under there
var stuff = await db.ReadAsync(async tr =>
{
var folder = await db.DirectoryLayer.TryOpenAsync(tr, path);
return(await tr.GetRange(folder.ToRange()).FirstOrDefaultAsync());
}, ct);
if (stuff.Key.IsPresent)
{
Program.Error(log, $"CAUTION: There is already some data under {path} !");
Program.Error(log, $" {FdbKey.Dump(stuff.Key)} = {stuff.Value:V}");
}
}
public static async Task <FdbClusterFile> GetCoordinatorsAsync([NotNull] IFdbDatabase db, CancellationToken ct)
{
Contract.NotNull(db, nameof(db));
var coordinators = await db.ReadAsync((tr) =>
{
tr.WithReadAccessToSystemKeys();
tr.WithPrioritySystemImmediate();
//note: we ask for high priority, because this method maybe called by a monitoring system than has to run when the cluster is clogged up in requests
return(tr.GetAsync(Fdb.System.Coordinators));
}, ct).ConfigureAwait(false);
if (coordinators.IsNull)
{
throw new InvalidOperationException("Failed to read the list of coordinators from the cluster's system keyspace.");
}
return(FdbClusterFile.Parse(coordinators.ToStringAscii()));
}
public static async Task <Dictionary <string, FdbDirectorySubspace> > BrowseAsync([NotNull] IFdbDatabase db, [NotNull] IFdbDirectory parent, CancellationToken ct)
{
Contract.NotNull(db, nameof(db));
Contract.NotNull(parent, nameof(parent));
return(await db.ReadAsync(async (tr) =>
{
// read the names of all the subdirectories
var names = await parent.ListAsync(tr).ConfigureAwait(false);
// open all the subdirectories
var folders = await names
.ToAsyncEnumerable()
.SelectAsync((name, _) => parent.OpenAsync(tr, name))
.ToListAsync(ct);
// map the result
return folders.ToDictionary(ds => ds.Name);
}, ct).ConfigureAwait(false));
}
/// <summary>Return the value of a configuration parameter (located under '\xFF/conf/')</summary>
/// <param name="db">Database to use for the operation</param>
/// <param name="name">Name of the configuration key (ex: "storage_engine")</param>
/// <param name="cancellationToken">Token used to cancel the operation</param>
/// <returns>Value of '\xFF/conf/storage_engine'</returns>
public static Task <Slice> GetConfigParameterAsync([NotNull] IFdbDatabase db, [NotNull] string name, CancellationToken cancellationToken)
{
if (db == null)
{
throw new ArgumentNullException("db");
}
if (string.IsNullOrEmpty(name))
{
throw new ArgumentException("Configuration parameter name cannot be null or empty", "name");
}
return(db.ReadAsync <Slice>((tr) =>
{
tr.WithReadAccessToSystemKeys();
tr.WithPrioritySystemImmediate();
//note: we ask for high priotity, because this method maybe called by a monitoring system than has to run when the cluster is clogged up in requests
return tr.GetAsync(Fdb.System.ConfigKey(name));
}, cancellationToken));
}
public static async Task ShowDirectoryLayer(FdbPath path, IVarTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
var dir = await db.ReadAsync(tr => TryOpenCurrentDirectoryAsync(tr, path), ct);
if (dir == null)
{
Program.Error(log, $"# Directory {path} does not exist anymore");
}
else
{
if (dir.Layer == FdbDirectoryPartition.LayerId)
{
log.WriteLine($"# Directory {path} is a partition");
}
else if (!string.IsNullOrEmpty(dir.Layer))
{
log.WriteLine($"# Directory {path} has layer '{dir.Layer}'");
}
else
{
log.WriteLine($"# Directory {path} does not have a layer defined");
}
}
}
private static async Task RunMultiClientTest(IFdbDatabase db, FdbSubspace location, bool highContention, string desc, int K, int NUM, CancellationToken ct)
{
Console.WriteLine("Starting {0} test with {1} threads and {2} iterations", desc, K, NUM);
var queue = new FdbQueue<string>(location, highContention);
await db.WriteAsync((tr) => queue.Clear(tr), ct);
// use a CTS to ensure that everything will stop in case of problems...
using (var go = new CancellationTokenSource(TimeSpan.FromSeconds(30)))
{
var tok = go.Token;
var pushLock = new AsyncCancelableMutex(tok);
var popLock = new AsyncCancelableMutex(tok);
int pushCount = 0;
int popCount = 0;
int stalls = 0;
var pushTreads = Enumerable.Range(0, K)
.Select(async id =>
{
// wait for the signal
await pushLock.Task.ConfigureAwait(false);
var res = new List<string>(NUM);
for (int i = 0; i < NUM; i++)
{
var item = id.ToString() + "." + i.ToString();
await db.ReadWriteAsync((tr) => queue.PushAsync(tr, item), tok).ConfigureAwait(false);
Interlocked.Increment(ref pushCount);
res.Add(item);
}
return res;
}).ToArray();
var popThreads = Enumerable.Range(0, K)
.Select(async id =>
{
// make everyone wait a bit, to ensure that they all start roughly at the same time
await popLock.Task.ConfigureAwait(false);
var res = new List<string>(NUM);
int i = 0;
while (i < NUM)
{
var item = await queue.PopAsync(db, tok).ConfigureAwait(false);
if (item.HasValue)
{
Interlocked.Increment(ref popCount);
res.Add(item.Value);
++i;
}
else
{
Interlocked.Increment(ref stalls);
await Task.Delay(10).ConfigureAwait(false);
}
}
return res;
}).ToArray();
var sw = Stopwatch.StartNew();
pushLock.Set(async: true);
await Task.Delay(100);
popLock.Set(async: true);
//using (var timer = new Timer((_) =>
//{
// var __ = TestHelpers.DumpSubspace(db, location);
//}, null, 1000, 4000))
{
await Task.WhenAll(pushTreads);
await Task.WhenAll(popThreads);
}
sw.Stop();
Console.WriteLine("> Finished {0} test in {1} seconds", desc, sw.Elapsed.TotalSeconds);
Console.WriteLine("> Pushed {0}, Popped {1} and Stalled {2}", pushCount, popCount, stalls);
var pushedItems = pushTreads.SelectMany(t => t.Result).ToList();
var poppedItems = popThreads.SelectMany(t => t.Result).ToList();
Assert.That(pushCount, Is.EqualTo(K * NUM));
Assert.That(popCount, Is.EqualTo(K * NUM));
// all pushed items should have been popped (with no duplicates)
Assert.That(poppedItems, Is.EquivalentTo(pushedItems));
// the queue should be empty
bool empty = await db.ReadAsync((tr) => queue.EmptyAsync(tr), ct);
Assert.That(empty, Is.True);
}
}
/// <summary>
/// Simulate a student that is really indecisive
/// </summary>
public async Task IndecisiveStudent(IFdbDatabase db, int id, int ops, CancellationToken ct)
{
string student = "s" + id.ToString("D04");
var allClasses = new List<string>(this.ClassNames);
var myClasses = new List<string>();
var rnd = new Random(id * 7);
for (int i = 0; i < ops && !ct.IsCancellationRequested; i++)
{
int classCount = myClasses.Count;
var moods = new List<string>();
if (classCount > 0) moods.AddRange(new[] { "drop", "switch" });
if (classCount < 5) moods.Add("add");
string mood = moods[rnd.Next(moods.Count)];
try
{
if (allClasses == null)
{
allClasses = await db.ReadAsync((tr) => AvailableClasses(tr), ct);
}
switch (mood)
{
case "add":
{
string @class = allClasses[rnd.Next(allClasses.Count)];
await db.ReadWriteAsync((tr) => Signup(tr, student, @class), ct);
myClasses.Add(@class);
break;
}
case "drop":
{
string @class = allClasses[rnd.Next(allClasses.Count)];
await db.ReadWriteAsync((tr) => Drop(tr, student, @class), ct);
myClasses.Remove(@class);
break;
}
case "switch":
{
string oldClass = allClasses[rnd.Next(allClasses.Count)];
string newClass = allClasses[rnd.Next(allClasses.Count)];
await db.ReadWriteAsync((tr) => Switch(tr, student, oldClass, newClass), ct);
myClasses.Remove(oldClass);
myClasses.Add(newClass);
break;
}
default:
{
throw new InvalidOperationException("Ooops");
}
}
}
catch (Exception e)
{
if (e is TaskCanceledException || e is OperationCanceledException) throw;
allClasses = null;
}
}
ct.ThrowIfCancellationRequested();
}
/// <summary>
/// Simulate a student that is really indecisive
/// </summary>
public async Task IndecisiveStudent(IFdbDatabase db, int id, int ops, CancellationToken ct)
{
string student = "s" + id.ToString("D04");
var allClasses = new List <string>(this.ClassNames);
var myClasses = new List <string>();
var rnd = new Random(id * 7);
for (int i = 0; i < ops && !ct.IsCancellationRequested; i++)
{
int classCount = myClasses.Count;
var moods = new List <string>();
if (classCount > 0)
{
moods.AddRange(new[] { "drop", "switch" });
}
if (classCount < 5)
{
moods.Add("add");
}
string mood = moods[rnd.Next(moods.Count)];
try
{
if (allClasses == null)
{
allClasses = await db.ReadAsync((tr) => AvailableClasses(tr), ct);
}
switch (mood)
{
case "add":
{
string @class = allClasses[rnd.Next(allClasses.Count)];
await db.ReadWriteAsync((tr) => Signup(tr, student, @class), ct);
myClasses.Add(@class);
break;
}
case "drop":
{
string @class = allClasses[rnd.Next(allClasses.Count)];
await db.ReadWriteAsync((tr) => Drop(tr, student, @class), ct);
myClasses.Remove(@class);
break;
}
case "switch":
{
string oldClass = allClasses[rnd.Next(allClasses.Count)];
string newClass = allClasses[rnd.Next(allClasses.Count)];
await db.ReadWriteAsync((tr) => Switch(tr, student, oldClass, newClass), ct);
myClasses.Remove(oldClass);
myClasses.Add(newClass);
break;
}
default:
{
throw new InvalidOperationException("Ooops");
}
}
}
catch (Exception e)
{
if (e is TaskCanceledException || e is OperationCanceledException)
{
throw;
}
allClasses = null;
}
}
ct.ThrowIfCancellationRequested();
}
public static async Task Map(string[] path, IFdbTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
// we want to merge the map of shards, with the map of directories from the Directory Layer, and count for each directory how many shards intersect
var folder = await TryOpenCurrentDirectoryAsync(path, db, ct);
if (folder == null)
{
log.WriteLine("# Directory not found");
return;
}
var span = folder.DirectoryLayer.ContentSubspace.ToRange();
// note: this may break in future versions of the DL! Maybe we need a custom API to get a flat list of all directories in a DL that span a specific range ?
var shards = await Fdb.System.GetChunksAsync(db, span, ct);
int totalShards = shards.Count;
log.WriteLine("Found {0} shard(s) in partition /{1}", totalShards, folder.DirectoryLayer.FullName);
log.WriteLine("Listing all directories...");
var map = new Dictionary<string, int>(StringComparer.Ordinal);
Action<string[], int> account = (p, c) =>
{
for (int i = 1; i <= p.Length; i++)
{
var s = "/" + String.Join("/", p, 0, i);
int x;
map[s] = map.TryGetValue(s, out x) ? (x + c) : c;
}
};
var work = new Stack<IFdbDirectory>();
work.Push(folder);
var dirs = new List<IFdbDirectory>();
int n = 0;
while(work.Count > 0)
{
var cur = work.Pop();
// skip sub partitions
var names = await cur.ListAsync(db, ct);
foreach(var name in names)
{
var sub = await cur.TryOpenAsync(db, name, ct);
if (sub != null)
{
var p = sub.FullName;
if (sub is FdbDirectoryPartition)
{
log.WriteLine("\r! Skipping partition {0} ", sub.Name);
n = 0;
continue;
}
log.Write("\r/{0}{1}", p, p.Length > n ? String.Empty : new string(' ', n - p.Length));
n = p.Length;
work.Push(sub);
dirs.Add(sub);
}
}
}
log.Write("\r" + new string(' ', n + 2));
log.WriteLine("\r> Found {0} sub-directories", dirs.Count);
log.WriteLine();
log.WriteLine("Estimating size of each directory...");
int foundShards = 0;
n = 0;
int max = 0;
IFdbDirectory bigBad = null;
foreach (var dir in dirs)
{
log.Write("\r> {0}{1}", dir.Name, dir.Name.Length > n ? String.Empty : new string(' ', n - dir.Name.Length));
n = dir.Name.Length;
var p = dir.Path.ToArray();
var key = ((FdbSubspace)dir).Key;
// verify that the subspace has at least one key inside
var bounds = await db.ReadAsync(async (tr) =>
{
var kvs = await Task.WhenAll(
tr.GetRange(FdbKeyRange.StartsWith(key)).FirstOrDefaultAsync(),
tr.GetRange(FdbKeyRange.StartsWith(key)).LastOrDefaultAsync()
);
return new { Min = kvs[0].Key, Max = kvs[1].Key };
}, ct);
if (bounds.Min.HasValue)
{ // folder is not empty
shards = await Fdb.System.GetChunksAsync(db, FdbKeyRange.StartsWith(key), ct);
//TODO: we still need to check if the first and last shard really intersect the subspace
// we need to check if the shards actually contain data
//Console.WriteLine("/{0} under {1} with {2} shard(s)", string.Join("/", p), FdbKey.Dump(key), shards.Count);
foundShards += shards.Count;
account(p, shards.Count);
if (shards.Count > max) { max = shards.Count; bigBad = dir; }
}
else
{
account(p, 0);
}
}
log.Write("\r" + new string(' ', n + 2));
log.WriteLine("\rFound a total of {0} shard(s) in {1} folder(s)", foundShards, dirs.Count);
log.WriteLine();
log.WriteLine("Shards %Total Path");
foreach(var kvp in map.OrderBy(x => x.Key))
{
log.WriteLine("{0,6} {1,-20} {2}", kvp.Value, RobustHistogram.FormatHistoBar((double)kvp.Value / foundShards, 20), kvp.Key);
}
log.WriteLine();
if (bigBad != null)
{
log.WriteLine("Biggest folder is /{0} with {1} shards ({2:N1}% total, {3:N1}% subtree)", bigBad.FullName, max, 100.0 * max / totalShards, 100.0 * max / foundShards);
log.WriteLine();
}
}
/// <summary>Creates a new directory</summary>
public static async Task CreateDirectory(string[] path, IFdbTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
if (log == null) log = Console.Out;
string layer = extras.Count > 0 ? extras.Get<string>(0) : null;
log.WriteLine("# Creating directory {0} with layer '{1}'", String.Join("/", path), layer);
var folder = await db.Directory.TryOpenAsync(path, cancellationToken: ct);
if (folder != null)
{
log.WriteLine("- Directory {0} already exists!", string.Join("/", path));
return;
}
folder = await db.Directory.TryCreateAsync(path, Slice.FromString(layer), cancellationToken: ct);
log.WriteLine("- Created under {0} [{1}]", FdbKey.Dump(folder.Key), folder.Key.ToHexaString(' '));
// look if there is already stuff under there
var stuff = await db.ReadAsync((tr) => tr.GetRange(folder.ToRange()).FirstOrDefaultAsync(), cancellationToken: ct);
if (stuff.Key.IsPresent)
{
log.WriteLine("CAUTION: There is already some data under {0} !");
log.WriteLine(" {0} = {1}", FdbKey.Dump(stuff.Key), stuff.Value.ToAsciiOrHexaString());
}
}
private static async Task RunMultiClientTest(IFdbDatabase db, KeySubspace location, bool highContention, string desc, int K, int NUM, CancellationToken ct)
{
Log("Starting {0} test with {1} threads and {2} iterations", desc, K, NUM);
var queue = new FdbQueue <string>(location, highContention);
await db.WriteAsync((tr) => queue.Clear(tr), ct);
// use a CTS to ensure that everything will stop in case of problems...
using (var go = new CancellationTokenSource(TimeSpan.FromSeconds(30)))
{
var tok = go.Token;
var pushLock = new AsyncCancelableMutex(tok);
var popLock = new AsyncCancelableMutex(tok);
int pushCount = 0;
int popCount = 0;
int stalls = 0;
var pushTreads = Enumerable.Range(0, K)
.Select(async id =>
{
try
{
// wait for the signal
await pushLock.Task.ConfigureAwait(false);
var res = new List <string>(NUM);
for (int i = 0; i < NUM; i++)
{
var item = id.ToString() + "." + i.ToString();
await db.ReadWriteAsync((tr) => queue.PushAsync(tr, item), tok).ConfigureAwait(false);
Interlocked.Increment(ref pushCount);
res.Add(item);
}
return(res);
}
catch (Exception e)
{
Log("PushThread[" + id + "] failed: " + e);
Assert.Fail("PushThread[" + id + "] failed: " + e.Message);
throw;
}
}).ToArray();
var popThreads = Enumerable.Range(0, K)
.Select(async id =>
{
try
{
// make everyone wait a bit, to ensure that they all start roughly at the same time
await popLock.Task.ConfigureAwait(false);
var res = new List <string>(NUM);
int i = 0;
while (i < NUM)
{
var item = await queue.PopAsync(db, tok).ConfigureAwait(false);
if (item.HasValue)
{
Interlocked.Increment(ref popCount);
res.Add(item.Value);
++i;
}
else
{
Interlocked.Increment(ref stalls);
await Task.Delay(10).ConfigureAwait(false);
}
}
return(res);
}
catch (Exception e)
{
Log("PopThread[" + id + "] failed: " + e);
Assert.Fail("PopThread[" + id + "] failed: " + e.Message);
throw;
}
}).ToArray();
var sw = Stopwatch.StartNew();
pushLock.Set(async: true);
await Task.Delay(100);
popLock.Set(async: true);
//using (var timer = new Timer((_) =>
//{
// var __ = TestHelpers.DumpSubspace(db, location);
//}, null, 1000, 4000))
{
await Task.WhenAll(pushTreads);
await Task.WhenAll(popThreads);
}
sw.Stop();
Log("> Finished {0} test in {1} seconds", desc, sw.Elapsed.TotalSeconds);
Log("> Pushed {0}, Popped {1} and Stalled {2}", pushCount, popCount, stalls);
var pushedItems = pushTreads.SelectMany(t => t.Result).ToList();
var poppedItems = popThreads.SelectMany(t => t.Result).ToList();
Assert.That(pushCount, Is.EqualTo(K * NUM));
Assert.That(popCount, Is.EqualTo(K * NUM));
// all pushed items should have been popped (with no duplicates)
Assert.That(poppedItems, Is.EquivalentTo(pushedItems));
// the queue should be empty
bool empty = await db.ReadAsync((tr) => queue.EmptyAsync(tr), ct);
Assert.That(empty, Is.True);
}
}
public async Task Run(IFdbDatabase db, TextWriter log, CancellationToken ct)
{
const int WORKERS = 1;
const int RUN_IN_SECONDS = 100;
await Init(db, ct);
log.WriteLine("Initialized for " + this.Mode.ToString());
var timeline = new RobustTimeLine(
TimeSpan.FromSeconds(1),
RobustHistogram.TimeScale.Milliseconds,
(histo, idx) =>
{
if (idx == 0)
{
Console.WriteLine("T+s | " + RobustHistogram.GetDistributionScale(RobustHistogram.HorizontalScale, 1, 5000 - 1) + " | ");
}
Console.WriteLine(String.Format(CultureInfo.InvariantCulture, "{0,3} | {1} | {2,6:#,##0.0} ms (+/- {3:#0.000})", idx, histo.GetDistribution(1, 5000 - 1), histo.Median, histo.MedianAbsoluteDeviation()));
if (log != Console.Out)
{
log.WriteLine(histo.GetReport(false));
}
return(false);
}
);
var duration = Stopwatch.StartNew();
var foo = this.Subspace.Keys.Encode("foo");
var bar = Slice.FromString("bar");
var barf = Slice.FromString("barf");
long total = 0;
timeline.Start();
var elapsed = await Program.RunConcurrentWorkersAsync(
WORKERS,
async (i, _ct) =>
{
var dur = Stopwatch.StartNew();
int k = 0;
while (dur.Elapsed.TotalSeconds < RUN_IN_SECONDS)
{
var sw = Stopwatch.StartNew();
switch (this.Mode)
{
case BenchMode.GetReadVersion:
{
await db.ReadAsync(tr => tr.GetReadVersionAsync(), ct);
break;
}
case BenchMode.Get:
{
if (this.Value <= 1)
{
await db.ReadAsync(tr => tr.GetAsync(foo), ct);
}
else
{
var foos = STuple.EncodePrefixedKeys(foo, Enumerable.Range(1, this.Value).ToArray());
await db.ReadAsync(tr => tr.GetValuesAsync(foos), ct);
}
break;
}
case BenchMode.Set:
{
await db.WriteAsync(tr => tr.Set(foo, bar), ct);
break;
}
case BenchMode.Watch:
{
var w = await db.GetAndWatch(foo, ct);
var v = w.Value;
// swap
v = v == bar ? barf : bar;
await db.WriteAsync((tr) => tr.Set(foo, v), ct);
await w;
break;
}
}
sw.Stop();
timeline.Add(sw.Elapsed.TotalMilliseconds);
Console.Write(k.ToString() + "\r");
++k;
Interlocked.Increment(ref total);
}
},
ct
);
timeline.Stop();
Console.WriteLine("Done ");
Console.WriteLine("# Ran {0} transactions in {1:0.0##} sec", total, elapsed.TotalSeconds);
var global = timeline.MergeResults();
log.WriteLine("# Merged results:");
log.WriteLine(global.GetReport(true));
if (log != Console.Out)
{
Console.WriteLine("# Merged results:");
Console.WriteLine(global.GetReport(true));
}
}
public static async Task Dir(FdbPath path, IVarTuple extras, DirectoryBrowseOptions options, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
if (log == null)
{
log = Console.Out;
}
Program.Comment(log, $"# Listing {path}:");
await db.ReadAsync(async tr =>
{
var parent = await TryOpenCurrentDirectoryAsync(tr, path);
if (parent == null)
{
Program.Error(log, "Directory not found.");
return;
}
if (!string.IsNullOrEmpty(parent.Layer))
{
log.WriteLine($"# Layer: {parent.Layer}");
}
var folders = await Fdb.Directory.BrowseAsync(tr, parent);
if (folders.Count > 0)
{
// to better align the names, we allow between 16 to 40 chars for the first column.
// if there is a larger name, it will stick out!
var maxLen = Math.Min(Math.Max(folders.Keys.Max(n => n.Length), 16), 40);
foreach (var kvp in folders)
{
var name = kvp.Key;
var subfolder = kvp.Value;
if (subfolder != null)
{
if ((options & DirectoryBrowseOptions.ShowCount) != 0)
{
if (!(subfolder is FdbDirectoryPartition))
{
long count = await Fdb.System.EstimateCountAsync(db, subfolder.ToRange(), ct);
Program.StdOut(log, $" {name.PadRight(maxLen)} {FdbKey.Dump(subfolder.Copy().GetPrefix()),-12} {(string.IsNullOrEmpty(subfolder.Layer) ? "-" : ("[" + subfolder.Layer + "]")),-12} {count,9:N0}", ConsoleColor.White);
}
else
{
Program.StdOut(log, $" {name.PadRight(maxLen)} {FdbKey.Dump(subfolder.Copy().GetPrefix()),-12} {(string.IsNullOrEmpty(subfolder.Layer) ? "-" : ("[" + subfolder.Layer + "]")),-12} {"-",9}", ConsoleColor.White);
}
}
else
{
Program.StdOut(log, $" {name.PadRight(maxLen)} {FdbKey.Dump(subfolder.Copy().GetPrefix()),-12} {(string.IsNullOrEmpty(subfolder.Layer) ? "-" : ("[" + subfolder.Layer + "]")),-12}", ConsoleColor.White);
}
}
else
{
Program.Error(log, $" WARNING: {name} seems to be missing!");
}
}
log.WriteLine($" {folders.Count} sub-directories.");
}
else
{
//TODO: test if it contains data?
log.WriteLine("No sub-directories.");
}
//TODO: check if there is at least one key?
}, ct);
}
/// <summary>Opens a subdirectory with the given <paramref name="name"/>.
/// An exception is thrown if the subdirectory does not exist, or if a layer is specified and a different layer was specified when the subdirectory was created.
/// </summary>
/// <param name="name">Name of the subdirectory to open</param>
public Task <FdbDirectorySubspace> OpenAsync([NotNull] string name, CancellationToken cancellationToken)
{
return(m_database.ReadAsync((tr) => m_directory.OpenAsync(tr, new [] { name }, Slice.Nil), cancellationToken));
}
public async Task Run(IFdbDatabase db, TextWriter log, CancellationToken ct)
{
const int WORKERS = 1;
const int RUN_IN_SECONDS = 100;
await Init(db, ct);
log.WriteLine("Initialized for " + this.Mode.ToString());
var timeline = new RobustTimeLine(
TimeSpan.FromSeconds(1),
RobustHistogram.TimeScale.Milliseconds,
(histo, idx) =>
{
if (idx == 0)
{
Console.WriteLine("T+s | " + RobustHistogram.GetDistributionScale(RobustHistogram.HorizontalScale, 1, 5000 - 1) + " | ");
}
Console.WriteLine(String.Format(CultureInfo.InvariantCulture, "{0,3} | {1} | {2,6:#,##0.0} ms (+/- {3:#0.000})", idx, histo.GetDistribution(1, 5000 - 1), histo.Median, histo.MedianAbsoluteDeviation()));
if (log != Console.Out) log.WriteLine(histo.GetReport(false));
return false;
}
);
var duration = Stopwatch.StartNew();
var foo = this.Subspace.Pack("foo");
var bar = Slice.FromString("bar");
var barf = Slice.FromString("barf");
long total = 0;
timeline.Start();
var elapsed = await Program.RunConcurrentWorkersAsync(
WORKERS,
async (i, _ct) =>
{
var dur = Stopwatch.StartNew();
int k = 0;
while (dur.Elapsed.TotalSeconds < RUN_IN_SECONDS)
{
var sw = Stopwatch.StartNew();
switch(this.Mode)
{
case BenchMode.GetReadVersion:
{
await db.ReadAsync(tr => tr.GetReadVersionAsync(), ct);
break;
}
case BenchMode.Get:
{
if (this.Value <= 1)
{
await db.ReadAsync(tr => tr.GetAsync(foo), ct);
}
else
{
var foos = FdbTuple.PackRange(foo, Enumerable.Range(1, this.Value).ToArray());
await db.ReadAsync(tr => tr.GetValuesAsync(foos), ct);
}
break;
}
case BenchMode.Set:
{
await db.WriteAsync(tr => tr.Set(foo, bar), ct);
break;
}
case BenchMode.Watch:
{
var w = await db.GetAndWatch(foo, ct);
var v = w.Value;
if (v == bar)
v = barf;
else
v = bar;
await db.WriteAsync((tr) => tr.Set(foo, v), ct);
await w;
break;
}
}
sw.Stop();
timeline.Add(sw.Elapsed.TotalMilliseconds);
Console.Write(k.ToString() + "\r");
++k;
Interlocked.Increment(ref total);
}
},
ct
);
timeline.Stop();
Console.WriteLine("Done ");
Console.WriteLine("# Ran {0} transactions in {1:0.0##} sec", total, elapsed.TotalSeconds);
var global = timeline.MergeResults();
log.WriteLine("# Merged results:");
log.WriteLine(global.GetReport(true));
if (log != Console.Out)
{
Console.WriteLine("# Merged results:");
Console.WriteLine(global.GetReport(true));
}
}
public static async Task Map(string[] path, IVarTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
// we want to merge the map of shards, with the map of directories from the Directory Layer, and count for each directory how many shards intersect
bool progress = log == Console.Out;
var folder = await TryOpenCurrentDirectoryAsync(path, db, ct);
if (folder == null)
{
Program.Error(log, "# Directory not found");
return;
}
Program.StdOut(log, "Listing all shards...");
// note: this may break in future versions of the DL! Maybe we need a custom API to get a flat list of all directories in a DL that span a specific range ?
var span = folder.DirectoryLayer.ContentSubspace.Keys.ToRange();
var shards = await Fdb.System.GetChunksAsync(db, span, ct);
int totalShards = shards.Count;
Program.StdOut(log, $"> Found {totalShards} shard(s) in partition /{folder.DirectoryLayer.FullName}", ConsoleColor.Gray);
Program.StdOut(log, "Listing all directories...");
var map = new Dictionary <string, int>(StringComparer.Ordinal);
void Account(string[] p, int c)
{
for (int i = 1; i <= p.Length; i++)
{
var s = "/" + string.Join("/", p, 0, i);
map[s] = map.TryGetValue(s, out int x) ? (x + c) : c;
}
}
var work = new Stack <IFdbDirectory>();
work.Push(folder);
var dirs = new List <IFdbDirectory>();
int n = 0;
while (work.Count > 0)
{
var cur = work.Pop();
// skip sub partitions
var names = await cur.ListAsync(db, ct);
foreach (var name in names)
{
var sub = await cur.TryOpenAsync(db, name, ct);
if (sub != null)
{
var p = sub.FullName;
if (sub is FdbDirectoryPartition)
{
if (progress)
{
log.Write("\r");
}
Program.StdOut(log, $"! Skipping partition {sub.Name} ", ConsoleColor.DarkRed);
n = 0;
continue;
}
if (progress)
{
log.Write($"\r/{p}{(p.Length > n ? String.Empty : new string(' ', n - p.Length))}");
}
n = p.Length;
work.Push(sub);
dirs.Add(sub);
}
}
}
if (progress)
{
log.Write("\r" + new string(' ', n + 2) + "\r");
}
Program.StdOut(log, $"> Found {dirs.Count} sub-directories", ConsoleColor.Gray);
log.WriteLine();
Program.StdOut(log, "Estimating size of each directory...");
int foundShards = 0;
n = 0;
int max = 0;
IFdbDirectory bigBad = null;
foreach (var dir in dirs)
{
if (progress)
{
log.Write($"\r> {dir.Name}{(dir.Name.Length > n ? String.Empty : new string(' ', n - dir.Name.Length))}");
}
n = dir.Name.Length;
var p = dir.Path.ToArray();
var key = ((KeySubspace)dir).GetPrefix();
// verify that the subspace has at least one key inside
var bounds = await db.ReadAsync(async (tr) =>
{
var kvs = await Task.WhenAll(
tr.GetRange(KeyRange.StartsWith(key)).FirstOrDefaultAsync(),
tr.GetRange(KeyRange.StartsWith(key)).LastOrDefaultAsync()
);
return(new { Min = kvs[0].Key, Max = kvs[1].Key });
}, ct);
if (bounds.Min.HasValue)
{ // folder is not empty
shards = await Fdb.System.GetChunksAsync(db, KeyRange.StartsWith(key), ct);
//TODO: we still need to check if the first and last shard really intersect the subspace
// we need to check if the shards actually contain data
//Console.WriteLine("/{0} under {1} with {2} shard(s)", string.Join("/", p), FdbKey.Dump(key), shards.Count);
foundShards += shards.Count;
Account(p, shards.Count);
if (shards.Count > max)
{
max = shards.Count; bigBad = dir;
}
}
else
{
Account(p, 0);
}
}
if (progress)
{
log.Write("\r" + new string(' ', n + 2) + "\r");
}
Program.StdOut(log, $"> Found a total of {foundShards:N0} shard(s) in {dirs.Count:N0} folder(s)", ConsoleColor.Gray);
log.WriteLine();
Program.StdOut(log, "Shards %Total Path");
foreach (var kvp in map.OrderBy(x => x.Key))
{
Program.StdOut(log, $"{kvp.Value,6} {RobustHistogram.FormatHistoBar((double)kvp.Value / foundShards, 20),-20} {kvp.Key}", ConsoleColor.Gray);
}
log.WriteLine();
if (bigBad != null)
{
Program.StdOut(log, $"Biggest folder is /{bigBad.FullName} with {max} shards ({100.0 * max / totalShards:N1}% total, {100.0 * max / foundShards:N1}% subtree)");
log.WriteLine();
}
}
public static Task <List <Slice> > GetBoundaryKeysAsync([NotNull] IFdbDatabase db, Slice beginInclusive, Slice endExclusive, CancellationToken ct)
{
Contract.NotNull(db, nameof(db));
return(db.ReadAsync((trans) => GetBoundaryKeysInternalAsync(trans, beginInclusive, endExclusive), ct));
}
