private static async Task <string[]> AutoCompleteDirectories(FdbPath path, IFdbDatabase db, TextWriter log, CancellationToken ct) { var parent = await db.ReadAsync(tr => BasicCommands.TryOpenCurrentDirectoryAsync(tr, path), ct); if (parent == null) { return(null); } var paths = await db.ReadAsync(tr => parent.ListAsync(tr), ct); return(paths.Select(p => p.Name).ToArray()); }
public static async Task <Dictionary <string, FdbDirectorySubspace> > BrowseAsync([NotNull] IFdbDatabase db, [NotNull] IFdbDirectory parent, CancellationToken cancellationToken) { if (db == null) { throw new ArgumentNullException("db"); } if (parent == null) { throw new ArgumentNullException("parent"); } return(await db.ReadAsync(async (tr) => { // read the names of all the subdirectories var names = await parent.ListAsync(tr).ConfigureAwait(false); // open all the subdirectories var folders = await names .ToAsyncEnumerable() .SelectAsync((name, ct) => parent.OpenAsync(tr, name)) .ToListAsync(); // map the result return folders.ToDictionary(ds => ds.Name); }, cancellationToken).ConfigureAwait(false)); }
/// <summary>Creates a new directory</summary> public static async Task CreateDirectory(string[] path, IFdbTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct) { if (log == null) { log = Console.Out; } string layer = extras.Count > 0 ? extras.Get <string>(0) : null; log.WriteLine("# Creating directory {0} with layer '{1}'", String.Join("/", path), layer); var folder = await db.Directory.TryOpenAsync(path, cancellationToken : ct); if (folder != null) { log.WriteLine("- Directory {0} already exists!", string.Join("/", path)); return; } folder = await db.Directory.TryCreateAsync(path, Slice.FromString(layer), cancellationToken : ct); log.WriteLine("- Created under {0} [{1}]", FdbKey.Dump(folder.Key), folder.Key.ToHexaString(' ')); // look if there is already stuff under there var stuff = await db.ReadAsync((tr) => tr.GetRange(folder.ToRange()).FirstOrDefaultAsync(), cancellationToken : ct); if (stuff.Key.IsPresent) { log.WriteLine("CAUTION: There is already some data under {0} !"); log.WriteLine(" {0} = {1}", FdbKey.Dump(stuff.Key), stuff.Value.ToAsciiOrHexaString()); } }
/// <summary>Creates a new directory</summary> public static async Task CreateDirectory(string[] path, IVarTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct) { if (log == null) { log = Console.Out; } string layer = extras.Count > 0 ? extras.Get <string>(0) : null; log.WriteLine($"# Creating directory {String.Join("/", path)} with layer '{layer}'"); var folder = await db.Directory.TryOpenAsync(db, path, ct : ct); if (folder != null) { log.WriteLine($"- Directory {string.Join("/", path)} already exists!"); return; } folder = await db.Directory.TryCreateAsync(db, path, Slice.FromString(layer), ct : ct); log.WriteLine($"- Created under {FdbKey.Dump(folder.GetPrefix())} [{folder.GetPrefix().ToHexaString(' ')}]"); // look if there is already stuff under there var stuff = await db.ReadAsync((tr) => tr.GetRange(folder.Keys.ToRange()).FirstOrDefaultAsync(), ct : ct); if (stuff.Key.IsPresent) { Program.Error(log, $"CAUTION: There is already some data under {string.Join("/", path)} !"); Program.Error(log, $" {FdbKey.Dump(stuff.Key)} = {stuff.Value:V}"); } }
/// <summary>Return the current value of the metadata version of the database.</summary> /// <remarks> /// Please note that by the time the value has been read, it may have already changed in the database! /// It is highly recommended to read the key as part as the same transaction that would read or update any metadata. /// This method requires API version 610 or greater. /// </remarks> public static Task <Slice> GetMetadataVersionAsync([NotNull] IFdbDatabase db, CancellationToken ct) { Contract.NotNull(db, nameof(db)); if (Fdb.ApiVersion < 610) { throw new NotSupportedException($"The metadata version system key is only available on version 6.1 or greater. Your application has selected API version {Fdb.ApiVersion} which is too low. You will need to select API version 610 or greater."); } return(db.ReadAsync(tr => tr.GetAsync(System.MetadataVersion), ct)); }
public static Task <List <Slice> > GetBoundaryKeysAsync([NotNull] IFdbDatabase db, Slice beginInclusive, Slice endExclusive, CancellationToken ct) { if (db == null) { throw new ArgumentNullException(nameof(db)); } return(db.ReadAsync((trans) => GetBoundaryKeysInternalAsync(trans, beginInclusive, endExclusive), ct)); }
public static Task <FdbSystemStatus?> GetStatusAsync(IFdbDatabase db, CancellationToken ct) { Contract.NotNull(db); // we should not retry the read to the status key! return(db.ReadAsync(tr => { tr.WithPrioritySystemImmediate(); //note: in v3.x, the status key does not need the access to system key option. //TODO: set a custom timeout? return GetStatusAsync(tr); }, ct)); }
/// <summary>Return the value of a configuration parameter (located under '\xFF/conf/')</summary> /// <param name="db">Database to use for the operation</param> /// <param name="name">Name of the configuration key (ex: "storage_engine")</param> /// <param name="ct">Token used to cancel the operation</param> /// <returns>Value of '\xFF/conf/storage_engine'</returns> public static Task <Slice> GetConfigParameterAsync([NotNull] IFdbDatabase db, [NotNull] string name, CancellationToken ct) { Contract.NotNull(db, nameof(db)); Contract.NotNullOrEmpty(name, nameof(name), "Configuration parameter name cannot be null or empty."); return(db.ReadAsync <Slice>((tr) => { tr.WithReadAccessToSystemKeys(); tr.WithPrioritySystemImmediate(); //note: we ask for high priority, because this method maybe called by a monitoring system than has to run when the cluster is clogged up in requests return tr.GetAsync(Fdb.System.ConfigKey(name)); }, ct)); }
/// <summary>Creates a new directory</summary> public static async Task CreateDirectory(FdbPath path, IVarTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct) { if (log == null) { log = Console.Out; } string layer = extras.Count > 0 ? extras.Get <string>(0).Trim() : string.Empty; if (path.LayerId != layer) { path.WithLayer(layer); } log.WriteLine($"# Creating directory {path}"); (var prefix, var created) = await db.ReadWriteAsync(async tr => { var folder = await db.DirectoryLayer.TryOpenAsync(tr, path); if (folder != null) { return(folder.GetPrefix(), false); } folder = await db.DirectoryLayer.TryCreateAsync(tr, path); return(folder.Copy().GetPrefix(), true); }, ct); if (!created) { log.WriteLine($"- Directory {path} already exists at {FdbKey.Dump(prefix)} [{prefix.ToHexaString(' ')}]"); return; } log.WriteLine($"- Created under {FdbKey.Dump(prefix)} [{prefix.ToHexaString(' ')}]"); // look if there is already stuff under there var stuff = await db.ReadAsync(async tr => { var folder = await db.DirectoryLayer.TryOpenAsync(tr, path); return(await tr.GetRange(folder.ToRange()).FirstOrDefaultAsync()); }, ct); if (stuff.Key.IsPresent) { Program.Error(log, $"CAUTION: There is already some data under {path} !"); Program.Error(log, $" {FdbKey.Dump(stuff.Key)} = {stuff.Value:V}"); } }
public static async Task <FdbClusterFile> GetCoordinatorsAsync([NotNull] IFdbDatabase db, CancellationToken ct) { Contract.NotNull(db, nameof(db)); var coordinators = await db.ReadAsync((tr) => { tr.WithReadAccessToSystemKeys(); tr.WithPrioritySystemImmediate(); //note: we ask for high priority, because this method maybe called by a monitoring system than has to run when the cluster is clogged up in requests return(tr.GetAsync(Fdb.System.Coordinators)); }, ct).ConfigureAwait(false); if (coordinators.IsNull) { throw new InvalidOperationException("Failed to read the list of coordinators from the cluster's system keyspace."); } return(FdbClusterFile.Parse(coordinators.ToStringAscii())); }
public static async Task <Dictionary <string, FdbDirectorySubspace> > BrowseAsync([NotNull] IFdbDatabase db, [NotNull] IFdbDirectory parent, CancellationToken ct) { Contract.NotNull(db, nameof(db)); Contract.NotNull(parent, nameof(parent)); return(await db.ReadAsync(async (tr) => { // read the names of all the subdirectories var names = await parent.ListAsync(tr).ConfigureAwait(false); // open all the subdirectories var folders = await names .ToAsyncEnumerable() .SelectAsync((name, _) => parent.OpenAsync(tr, name)) .ToListAsync(ct); // map the result return folders.ToDictionary(ds => ds.Name); }, ct).ConfigureAwait(false)); }
/// <summary>Return the value of a configuration parameter (located under '\xFF/conf/')</summary> /// <param name="db">Database to use for the operation</param> /// <param name="name">Name of the configuration key (ex: "storage_engine")</param> /// <param name="cancellationToken">Token used to cancel the operation</param> /// <returns>Value of '\xFF/conf/storage_engine'</returns> public static Task <Slice> GetConfigParameterAsync([NotNull] IFdbDatabase db, [NotNull] string name, CancellationToken cancellationToken) { if (db == null) { throw new ArgumentNullException("db"); } if (string.IsNullOrEmpty(name)) { throw new ArgumentException("Configuration parameter name cannot be null or empty", "name"); } return(db.ReadAsync <Slice>((tr) => { tr.WithReadAccessToSystemKeys(); tr.WithPrioritySystemImmediate(); //note: we ask for high priotity, because this method maybe called by a monitoring system than has to run when the cluster is clogged up in requests return tr.GetAsync(Fdb.System.ConfigKey(name)); }, cancellationToken)); }
public static async Task ShowDirectoryLayer(FdbPath path, IVarTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct) { var dir = await db.ReadAsync(tr => TryOpenCurrentDirectoryAsync(tr, path), ct); if (dir == null) { Program.Error(log, $"# Directory {path} does not exist anymore"); } else { if (dir.Layer == FdbDirectoryPartition.LayerId) { log.WriteLine($"# Directory {path} is a partition"); } else if (!string.IsNullOrEmpty(dir.Layer)) { log.WriteLine($"# Directory {path} has layer '{dir.Layer}'"); } else { log.WriteLine($"# Directory {path} does not have a layer defined"); } } }
private static async Task RunMultiClientTest(IFdbDatabase db, FdbSubspace location, bool highContention, string desc, int K, int NUM, CancellationToken ct) { Console.WriteLine("Starting {0} test with {1} threads and {2} iterations", desc, K, NUM); var queue = new FdbQueue<string>(location, highContention); await db.WriteAsync((tr) => queue.Clear(tr), ct); // use a CTS to ensure that everything will stop in case of problems... using (var go = new CancellationTokenSource(TimeSpan.FromSeconds(30))) { var tok = go.Token; var pushLock = new AsyncCancelableMutex(tok); var popLock = new AsyncCancelableMutex(tok); int pushCount = 0; int popCount = 0; int stalls = 0; var pushTreads = Enumerable.Range(0, K) .Select(async id => { // wait for the signal await pushLock.Task.ConfigureAwait(false); var res = new List<string>(NUM); for (int i = 0; i < NUM; i++) { var item = id.ToString() + "." + i.ToString(); await db.ReadWriteAsync((tr) => queue.PushAsync(tr, item), tok).ConfigureAwait(false); Interlocked.Increment(ref pushCount); res.Add(item); } return res; }).ToArray(); var popThreads = Enumerable.Range(0, K) .Select(async id => { // make everyone wait a bit, to ensure that they all start roughly at the same time await popLock.Task.ConfigureAwait(false); var res = new List<string>(NUM); int i = 0; while (i < NUM) { var item = await queue.PopAsync(db, tok).ConfigureAwait(false); if (item.HasValue) { Interlocked.Increment(ref popCount); res.Add(item.Value); ++i; } else { Interlocked.Increment(ref stalls); await Task.Delay(10).ConfigureAwait(false); } } return res; }).ToArray(); var sw = Stopwatch.StartNew(); pushLock.Set(async: true); await Task.Delay(100); popLock.Set(async: true); //using (var timer = new Timer((_) => //{ // var __ = TestHelpers.DumpSubspace(db, location); //}, null, 1000, 4000)) { await Task.WhenAll(pushTreads); await Task.WhenAll(popThreads); } sw.Stop(); Console.WriteLine("> Finished {0} test in {1} seconds", desc, sw.Elapsed.TotalSeconds); Console.WriteLine("> Pushed {0}, Popped {1} and Stalled {2}", pushCount, popCount, stalls); var pushedItems = pushTreads.SelectMany(t => t.Result).ToList(); var poppedItems = popThreads.SelectMany(t => t.Result).ToList(); Assert.That(pushCount, Is.EqualTo(K * NUM)); Assert.That(popCount, Is.EqualTo(K * NUM)); // all pushed items should have been popped (with no duplicates) Assert.That(poppedItems, Is.EquivalentTo(pushedItems)); // the queue should be empty bool empty = await db.ReadAsync((tr) => queue.EmptyAsync(tr), ct); Assert.That(empty, Is.True); } }
/// <summary> /// Simulate a student that is really indecisive /// </summary> public async Task IndecisiveStudent(IFdbDatabase db, int id, int ops, CancellationToken ct) { string student = "s" + id.ToString("D04"); var allClasses = new List<string>(this.ClassNames); var myClasses = new List<string>(); var rnd = new Random(id * 7); for (int i = 0; i < ops && !ct.IsCancellationRequested; i++) { int classCount = myClasses.Count; var moods = new List<string>(); if (classCount > 0) moods.AddRange(new[] { "drop", "switch" }); if (classCount < 5) moods.Add("add"); string mood = moods[rnd.Next(moods.Count)]; try { if (allClasses == null) { allClasses = await db.ReadAsync((tr) => AvailableClasses(tr), ct); } switch (mood) { case "add": { string @class = allClasses[rnd.Next(allClasses.Count)]; await db.ReadWriteAsync((tr) => Signup(tr, student, @class), ct); myClasses.Add(@class); break; } case "drop": { string @class = allClasses[rnd.Next(allClasses.Count)]; await db.ReadWriteAsync((tr) => Drop(tr, student, @class), ct); myClasses.Remove(@class); break; } case "switch": { string oldClass = allClasses[rnd.Next(allClasses.Count)]; string newClass = allClasses[rnd.Next(allClasses.Count)]; await db.ReadWriteAsync((tr) => Switch(tr, student, oldClass, newClass), ct); myClasses.Remove(oldClass); myClasses.Add(newClass); break; } default: { throw new InvalidOperationException("Ooops"); } } } catch (Exception e) { if (e is TaskCanceledException || e is OperationCanceledException) throw; allClasses = null; } } ct.ThrowIfCancellationRequested(); }
/// <summary> /// Simulate a student that is really indecisive /// </summary> public async Task IndecisiveStudent(IFdbDatabase db, int id, int ops, CancellationToken ct) { string student = "s" + id.ToString("D04"); var allClasses = new List <string>(this.ClassNames); var myClasses = new List <string>(); var rnd = new Random(id * 7); for (int i = 0; i < ops && !ct.IsCancellationRequested; i++) { int classCount = myClasses.Count; var moods = new List <string>(); if (classCount > 0) { moods.AddRange(new[] { "drop", "switch" }); } if (classCount < 5) { moods.Add("add"); } string mood = moods[rnd.Next(moods.Count)]; try { if (allClasses == null) { allClasses = await db.ReadAsync((tr) => AvailableClasses(tr), ct); } switch (mood) { case "add": { string @class = allClasses[rnd.Next(allClasses.Count)]; await db.ReadWriteAsync((tr) => Signup(tr, student, @class), ct); myClasses.Add(@class); break; } case "drop": { string @class = allClasses[rnd.Next(allClasses.Count)]; await db.ReadWriteAsync((tr) => Drop(tr, student, @class), ct); myClasses.Remove(@class); break; } case "switch": { string oldClass = allClasses[rnd.Next(allClasses.Count)]; string newClass = allClasses[rnd.Next(allClasses.Count)]; await db.ReadWriteAsync((tr) => Switch(tr, student, oldClass, newClass), ct); myClasses.Remove(oldClass); myClasses.Add(newClass); break; } default: { throw new InvalidOperationException("Ooops"); } } } catch (Exception e) { if (e is TaskCanceledException || e is OperationCanceledException) { throw; } allClasses = null; } } ct.ThrowIfCancellationRequested(); }
public static async Task Map(string[] path, IFdbTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct) { // we want to merge the map of shards, with the map of directories from the Directory Layer, and count for each directory how many shards intersect var folder = await TryOpenCurrentDirectoryAsync(path, db, ct); if (folder == null) { log.WriteLine("# Directory not found"); return; } var span = folder.DirectoryLayer.ContentSubspace.ToRange(); // note: this may break in future versions of the DL! Maybe we need a custom API to get a flat list of all directories in a DL that span a specific range ? var shards = await Fdb.System.GetChunksAsync(db, span, ct); int totalShards = shards.Count; log.WriteLine("Found {0} shard(s) in partition /{1}", totalShards, folder.DirectoryLayer.FullName); log.WriteLine("Listing all directories..."); var map = new Dictionary<string, int>(StringComparer.Ordinal); Action<string[], int> account = (p, c) => { for (int i = 1; i <= p.Length; i++) { var s = "/" + String.Join("/", p, 0, i); int x; map[s] = map.TryGetValue(s, out x) ? (x + c) : c; } }; var work = new Stack<IFdbDirectory>(); work.Push(folder); var dirs = new List<IFdbDirectory>(); int n = 0; while(work.Count > 0) { var cur = work.Pop(); // skip sub partitions var names = await cur.ListAsync(db, ct); foreach(var name in names) { var sub = await cur.TryOpenAsync(db, name, ct); if (sub != null) { var p = sub.FullName; if (sub is FdbDirectoryPartition) { log.WriteLine("\r! Skipping partition {0} ", sub.Name); n = 0; continue; } log.Write("\r/{0}{1}", p, p.Length > n ? String.Empty : new string(' ', n - p.Length)); n = p.Length; work.Push(sub); dirs.Add(sub); } } } log.Write("\r" + new string(' ', n + 2)); log.WriteLine("\r> Found {0} sub-directories", dirs.Count); log.WriteLine(); log.WriteLine("Estimating size of each directory..."); int foundShards = 0; n = 0; int max = 0; IFdbDirectory bigBad = null; foreach (var dir in dirs) { log.Write("\r> {0}{1}", dir.Name, dir.Name.Length > n ? String.Empty : new string(' ', n - dir.Name.Length)); n = dir.Name.Length; var p = dir.Path.ToArray(); var key = ((FdbSubspace)dir).Key; // verify that the subspace has at least one key inside var bounds = await db.ReadAsync(async (tr) => { var kvs = await Task.WhenAll( tr.GetRange(FdbKeyRange.StartsWith(key)).FirstOrDefaultAsync(), tr.GetRange(FdbKeyRange.StartsWith(key)).LastOrDefaultAsync() ); return new { Min = kvs[0].Key, Max = kvs[1].Key }; }, ct); if (bounds.Min.HasValue) { // folder is not empty shards = await Fdb.System.GetChunksAsync(db, FdbKeyRange.StartsWith(key), ct); //TODO: we still need to check if the first and last shard really intersect the subspace // we need to check if the shards actually contain data //Console.WriteLine("/{0} under {1} with {2} shard(s)", string.Join("/", p), FdbKey.Dump(key), shards.Count); foundShards += shards.Count; account(p, shards.Count); if (shards.Count > max) { max = shards.Count; bigBad = dir; } } else { account(p, 0); } } log.Write("\r" + new string(' ', n + 2)); log.WriteLine("\rFound a total of {0} shard(s) in {1} folder(s)", foundShards, dirs.Count); log.WriteLine(); log.WriteLine("Shards %Total Path"); foreach(var kvp in map.OrderBy(x => x.Key)) { log.WriteLine("{0,6} {1,-20} {2}", kvp.Value, RobustHistogram.FormatHistoBar((double)kvp.Value / foundShards, 20), kvp.Key); } log.WriteLine(); if (bigBad != null) { log.WriteLine("Biggest folder is /{0} with {1} shards ({2:N1}% total, {3:N1}% subtree)", bigBad.FullName, max, 100.0 * max / totalShards, 100.0 * max / foundShards); log.WriteLine(); } }
/// <summary>Creates a new directory</summary> public static async Task CreateDirectory(string[] path, IFdbTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct) { if (log == null) log = Console.Out; string layer = extras.Count > 0 ? extras.Get<string>(0) : null; log.WriteLine("# Creating directory {0} with layer '{1}'", String.Join("/", path), layer); var folder = await db.Directory.TryOpenAsync(path, cancellationToken: ct); if (folder != null) { log.WriteLine("- Directory {0} already exists!", string.Join("/", path)); return; } folder = await db.Directory.TryCreateAsync(path, Slice.FromString(layer), cancellationToken: ct); log.WriteLine("- Created under {0} [{1}]", FdbKey.Dump(folder.Key), folder.Key.ToHexaString(' ')); // look if there is already stuff under there var stuff = await db.ReadAsync((tr) => tr.GetRange(folder.ToRange()).FirstOrDefaultAsync(), cancellationToken: ct); if (stuff.Key.IsPresent) { log.WriteLine("CAUTION: There is already some data under {0} !"); log.WriteLine(" {0} = {1}", FdbKey.Dump(stuff.Key), stuff.Value.ToAsciiOrHexaString()); } }
private static async Task RunMultiClientTest(IFdbDatabase db, KeySubspace location, bool highContention, string desc, int K, int NUM, CancellationToken ct) { Log("Starting {0} test with {1} threads and {2} iterations", desc, K, NUM); var queue = new FdbQueue <string>(location, highContention); await db.WriteAsync((tr) => queue.Clear(tr), ct); // use a CTS to ensure that everything will stop in case of problems... using (var go = new CancellationTokenSource(TimeSpan.FromSeconds(30))) { var tok = go.Token; var pushLock = new AsyncCancelableMutex(tok); var popLock = new AsyncCancelableMutex(tok); int pushCount = 0; int popCount = 0; int stalls = 0; var pushTreads = Enumerable.Range(0, K) .Select(async id => { try { // wait for the signal await pushLock.Task.ConfigureAwait(false); var res = new List <string>(NUM); for (int i = 0; i < NUM; i++) { var item = id.ToString() + "." + i.ToString(); await db.ReadWriteAsync((tr) => queue.PushAsync(tr, item), tok).ConfigureAwait(false); Interlocked.Increment(ref pushCount); res.Add(item); } return(res); } catch (Exception e) { Log("PushThread[" + id + "] failed: " + e); Assert.Fail("PushThread[" + id + "] failed: " + e.Message); throw; } }).ToArray(); var popThreads = Enumerable.Range(0, K) .Select(async id => { try { // make everyone wait a bit, to ensure that they all start roughly at the same time await popLock.Task.ConfigureAwait(false); var res = new List <string>(NUM); int i = 0; while (i < NUM) { var item = await queue.PopAsync(db, tok).ConfigureAwait(false); if (item.HasValue) { Interlocked.Increment(ref popCount); res.Add(item.Value); ++i; } else { Interlocked.Increment(ref stalls); await Task.Delay(10).ConfigureAwait(false); } } return(res); } catch (Exception e) { Log("PopThread[" + id + "] failed: " + e); Assert.Fail("PopThread[" + id + "] failed: " + e.Message); throw; } }).ToArray(); var sw = Stopwatch.StartNew(); pushLock.Set(async: true); await Task.Delay(100); popLock.Set(async: true); //using (var timer = new Timer((_) => //{ // var __ = TestHelpers.DumpSubspace(db, location); //}, null, 1000, 4000)) { await Task.WhenAll(pushTreads); await Task.WhenAll(popThreads); } sw.Stop(); Log("> Finished {0} test in {1} seconds", desc, sw.Elapsed.TotalSeconds); Log("> Pushed {0}, Popped {1} and Stalled {2}", pushCount, popCount, stalls); var pushedItems = pushTreads.SelectMany(t => t.Result).ToList(); var poppedItems = popThreads.SelectMany(t => t.Result).ToList(); Assert.That(pushCount, Is.EqualTo(K * NUM)); Assert.That(popCount, Is.EqualTo(K * NUM)); // all pushed items should have been popped (with no duplicates) Assert.That(poppedItems, Is.EquivalentTo(pushedItems)); // the queue should be empty bool empty = await db.ReadAsync((tr) => queue.EmptyAsync(tr), ct); Assert.That(empty, Is.True); } }
public async Task Run(IFdbDatabase db, TextWriter log, CancellationToken ct) { const int WORKERS = 1; const int RUN_IN_SECONDS = 100; await Init(db, ct); log.WriteLine("Initialized for " + this.Mode.ToString()); var timeline = new RobustTimeLine( TimeSpan.FromSeconds(1), RobustHistogram.TimeScale.Milliseconds, (histo, idx) => { if (idx == 0) { Console.WriteLine("T+s | " + RobustHistogram.GetDistributionScale(RobustHistogram.HorizontalScale, 1, 5000 - 1) + " | "); } Console.WriteLine(String.Format(CultureInfo.InvariantCulture, "{0,3} | {1} | {2,6:#,##0.0} ms (+/- {3:#0.000})", idx, histo.GetDistribution(1, 5000 - 1), histo.Median, histo.MedianAbsoluteDeviation())); if (log != Console.Out) { log.WriteLine(histo.GetReport(false)); } return(false); } ); var duration = Stopwatch.StartNew(); var foo = this.Subspace.Keys.Encode("foo"); var bar = Slice.FromString("bar"); var barf = Slice.FromString("barf"); long total = 0; timeline.Start(); var elapsed = await Program.RunConcurrentWorkersAsync( WORKERS, async (i, _ct) => { var dur = Stopwatch.StartNew(); int k = 0; while (dur.Elapsed.TotalSeconds < RUN_IN_SECONDS) { var sw = Stopwatch.StartNew(); switch (this.Mode) { case BenchMode.GetReadVersion: { await db.ReadAsync(tr => tr.GetReadVersionAsync(), ct); break; } case BenchMode.Get: { if (this.Value <= 1) { await db.ReadAsync(tr => tr.GetAsync(foo), ct); } else { var foos = STuple.EncodePrefixedKeys(foo, Enumerable.Range(1, this.Value).ToArray()); await db.ReadAsync(tr => tr.GetValuesAsync(foos), ct); } break; } case BenchMode.Set: { await db.WriteAsync(tr => tr.Set(foo, bar), ct); break; } case BenchMode.Watch: { var w = await db.GetAndWatch(foo, ct); var v = w.Value; // swap v = v == bar ? barf : bar; await db.WriteAsync((tr) => tr.Set(foo, v), ct); await w; break; } } sw.Stop(); timeline.Add(sw.Elapsed.TotalMilliseconds); Console.Write(k.ToString() + "\r"); ++k; Interlocked.Increment(ref total); } }, ct ); timeline.Stop(); Console.WriteLine("Done "); Console.WriteLine("# Ran {0} transactions in {1:0.0##} sec", total, elapsed.TotalSeconds); var global = timeline.MergeResults(); log.WriteLine("# Merged results:"); log.WriteLine(global.GetReport(true)); if (log != Console.Out) { Console.WriteLine("# Merged results:"); Console.WriteLine(global.GetReport(true)); } }
public static async Task Dir(FdbPath path, IVarTuple extras, DirectoryBrowseOptions options, IFdbDatabase db, TextWriter log, CancellationToken ct) { if (log == null) { log = Console.Out; } Program.Comment(log, $"# Listing {path}:"); await db.ReadAsync(async tr => { var parent = await TryOpenCurrentDirectoryAsync(tr, path); if (parent == null) { Program.Error(log, "Directory not found."); return; } if (!string.IsNullOrEmpty(parent.Layer)) { log.WriteLine($"# Layer: {parent.Layer}"); } var folders = await Fdb.Directory.BrowseAsync(tr, parent); if (folders.Count > 0) { // to better align the names, we allow between 16 to 40 chars for the first column. // if there is a larger name, it will stick out! var maxLen = Math.Min(Math.Max(folders.Keys.Max(n => n.Length), 16), 40); foreach (var kvp in folders) { var name = kvp.Key; var subfolder = kvp.Value; if (subfolder != null) { if ((options & DirectoryBrowseOptions.ShowCount) != 0) { if (!(subfolder is FdbDirectoryPartition)) { long count = await Fdb.System.EstimateCountAsync(db, subfolder.ToRange(), ct); Program.StdOut(log, $" {name.PadRight(maxLen)} {FdbKey.Dump(subfolder.Copy().GetPrefix()),-12} {(string.IsNullOrEmpty(subfolder.Layer) ? "-" : ("[" + subfolder.Layer + "]")),-12} {count,9:N0}", ConsoleColor.White); } else { Program.StdOut(log, $" {name.PadRight(maxLen)} {FdbKey.Dump(subfolder.Copy().GetPrefix()),-12} {(string.IsNullOrEmpty(subfolder.Layer) ? "-" : ("[" + subfolder.Layer + "]")),-12} {"-",9}", ConsoleColor.White); } } else { Program.StdOut(log, $" {name.PadRight(maxLen)} {FdbKey.Dump(subfolder.Copy().GetPrefix()),-12} {(string.IsNullOrEmpty(subfolder.Layer) ? "-" : ("[" + subfolder.Layer + "]")),-12}", ConsoleColor.White); } } else { Program.Error(log, $" WARNING: {name} seems to be missing!"); } } log.WriteLine($" {folders.Count} sub-directories."); } else { //TODO: test if it contains data? log.WriteLine("No sub-directories."); } //TODO: check if there is at least one key? }, ct); }
/// <summary>Opens a subdirectory with the given <paramref name="name"/>. /// An exception is thrown if the subdirectory does not exist, or if a layer is specified and a different layer was specified when the subdirectory was created. /// </summary> /// <param name="name">Name of the subdirectory to open</param> public Task <FdbDirectorySubspace> OpenAsync([NotNull] string name, CancellationToken cancellationToken) { return(m_database.ReadAsync((tr) => m_directory.OpenAsync(tr, new [] { name }, Slice.Nil), cancellationToken)); }
public async Task Run(IFdbDatabase db, TextWriter log, CancellationToken ct) { const int WORKERS = 1; const int RUN_IN_SECONDS = 100; await Init(db, ct); log.WriteLine("Initialized for " + this.Mode.ToString()); var timeline = new RobustTimeLine( TimeSpan.FromSeconds(1), RobustHistogram.TimeScale.Milliseconds, (histo, idx) => { if (idx == 0) { Console.WriteLine("T+s | " + RobustHistogram.GetDistributionScale(RobustHistogram.HorizontalScale, 1, 5000 - 1) + " | "); } Console.WriteLine(String.Format(CultureInfo.InvariantCulture, "{0,3} | {1} | {2,6:#,##0.0} ms (+/- {3:#0.000})", idx, histo.GetDistribution(1, 5000 - 1), histo.Median, histo.MedianAbsoluteDeviation())); if (log != Console.Out) log.WriteLine(histo.GetReport(false)); return false; } ); var duration = Stopwatch.StartNew(); var foo = this.Subspace.Pack("foo"); var bar = Slice.FromString("bar"); var barf = Slice.FromString("barf"); long total = 0; timeline.Start(); var elapsed = await Program.RunConcurrentWorkersAsync( WORKERS, async (i, _ct) => { var dur = Stopwatch.StartNew(); int k = 0; while (dur.Elapsed.TotalSeconds < RUN_IN_SECONDS) { var sw = Stopwatch.StartNew(); switch(this.Mode) { case BenchMode.GetReadVersion: { await db.ReadAsync(tr => tr.GetReadVersionAsync(), ct); break; } case BenchMode.Get: { if (this.Value <= 1) { await db.ReadAsync(tr => tr.GetAsync(foo), ct); } else { var foos = FdbTuple.PackRange(foo, Enumerable.Range(1, this.Value).ToArray()); await db.ReadAsync(tr => tr.GetValuesAsync(foos), ct); } break; } case BenchMode.Set: { await db.WriteAsync(tr => tr.Set(foo, bar), ct); break; } case BenchMode.Watch: { var w = await db.GetAndWatch(foo, ct); var v = w.Value; if (v == bar) v = barf; else v = bar; await db.WriteAsync((tr) => tr.Set(foo, v), ct); await w; break; } } sw.Stop(); timeline.Add(sw.Elapsed.TotalMilliseconds); Console.Write(k.ToString() + "\r"); ++k; Interlocked.Increment(ref total); } }, ct ); timeline.Stop(); Console.WriteLine("Done "); Console.WriteLine("# Ran {0} transactions in {1:0.0##} sec", total, elapsed.TotalSeconds); var global = timeline.MergeResults(); log.WriteLine("# Merged results:"); log.WriteLine(global.GetReport(true)); if (log != Console.Out) { Console.WriteLine("# Merged results:"); Console.WriteLine(global.GetReport(true)); } }
public static async Task Map(string[] path, IVarTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct) { // we want to merge the map of shards, with the map of directories from the Directory Layer, and count for each directory how many shards intersect bool progress = log == Console.Out; var folder = await TryOpenCurrentDirectoryAsync(path, db, ct); if (folder == null) { Program.Error(log, "# Directory not found"); return; } Program.StdOut(log, "Listing all shards..."); // note: this may break in future versions of the DL! Maybe we need a custom API to get a flat list of all directories in a DL that span a specific range ? var span = folder.DirectoryLayer.ContentSubspace.Keys.ToRange(); var shards = await Fdb.System.GetChunksAsync(db, span, ct); int totalShards = shards.Count; Program.StdOut(log, $"> Found {totalShards} shard(s) in partition /{folder.DirectoryLayer.FullName}", ConsoleColor.Gray); Program.StdOut(log, "Listing all directories..."); var map = new Dictionary <string, int>(StringComparer.Ordinal); void Account(string[] p, int c) { for (int i = 1; i <= p.Length; i++) { var s = "/" + string.Join("/", p, 0, i); map[s] = map.TryGetValue(s, out int x) ? (x + c) : c; } } var work = new Stack <IFdbDirectory>(); work.Push(folder); var dirs = new List <IFdbDirectory>(); int n = 0; while (work.Count > 0) { var cur = work.Pop(); // skip sub partitions var names = await cur.ListAsync(db, ct); foreach (var name in names) { var sub = await cur.TryOpenAsync(db, name, ct); if (sub != null) { var p = sub.FullName; if (sub is FdbDirectoryPartition) { if (progress) { log.Write("\r"); } Program.StdOut(log, $"! Skipping partition {sub.Name} ", ConsoleColor.DarkRed); n = 0; continue; } if (progress) { log.Write($"\r/{p}{(p.Length > n ? String.Empty : new string(' ', n - p.Length))}"); } n = p.Length; work.Push(sub); dirs.Add(sub); } } } if (progress) { log.Write("\r" + new string(' ', n + 2) + "\r"); } Program.StdOut(log, $"> Found {dirs.Count} sub-directories", ConsoleColor.Gray); log.WriteLine(); Program.StdOut(log, "Estimating size of each directory..."); int foundShards = 0; n = 0; int max = 0; IFdbDirectory bigBad = null; foreach (var dir in dirs) { if (progress) { log.Write($"\r> {dir.Name}{(dir.Name.Length > n ? String.Empty : new string(' ', n - dir.Name.Length))}"); } n = dir.Name.Length; var p = dir.Path.ToArray(); var key = ((KeySubspace)dir).GetPrefix(); // verify that the subspace has at least one key inside var bounds = await db.ReadAsync(async (tr) => { var kvs = await Task.WhenAll( tr.GetRange(KeyRange.StartsWith(key)).FirstOrDefaultAsync(), tr.GetRange(KeyRange.StartsWith(key)).LastOrDefaultAsync() ); return(new { Min = kvs[0].Key, Max = kvs[1].Key }); }, ct); if (bounds.Min.HasValue) { // folder is not empty shards = await Fdb.System.GetChunksAsync(db, KeyRange.StartsWith(key), ct); //TODO: we still need to check if the first and last shard really intersect the subspace // we need to check if the shards actually contain data //Console.WriteLine("/{0} under {1} with {2} shard(s)", string.Join("/", p), FdbKey.Dump(key), shards.Count); foundShards += shards.Count; Account(p, shards.Count); if (shards.Count > max) { max = shards.Count; bigBad = dir; } } else { Account(p, 0); } } if (progress) { log.Write("\r" + new string(' ', n + 2) + "\r"); } Program.StdOut(log, $"> Found a total of {foundShards:N0} shard(s) in {dirs.Count:N0} folder(s)", ConsoleColor.Gray); log.WriteLine(); Program.StdOut(log, "Shards %Total Path"); foreach (var kvp in map.OrderBy(x => x.Key)) { Program.StdOut(log, $"{kvp.Value,6} {RobustHistogram.FormatHistoBar((double)kvp.Value / foundShards, 20),-20} {kvp.Key}", ConsoleColor.Gray); } log.WriteLine(); if (bigBad != null) { Program.StdOut(log, $"Biggest folder is /{bigBad.FullName} with {max} shards ({100.0 * max / totalShards:N1}% total, {100.0 * max / foundShards:N1}% subtree)"); log.WriteLine(); } }
public static Task <List <Slice> > GetBoundaryKeysAsync([NotNull] IFdbDatabase db, Slice beginInclusive, Slice endExclusive, CancellationToken ct) { Contract.NotNull(db, nameof(db)); return(db.ReadAsync((trans) => GetBoundaryKeysInternalAsync(trans, beginInclusive, endExclusive), ct)); }