public static async Task DumpSubspace(IFdbReadOnlyTransaction tr, IKeySubspace subspace)
{
Assert.That(tr, Is.Not.Null);
Assert.That(subspace, Is.Not.Null);
FdbTest.Log($"Dumping content of {subspace} at {subspace.GetPrefix():K}:");
int count = 0;
await tr
.GetRange(KeyRange.StartsWith(subspace.GetPrefix()))
.ForEachAsync((kvp) =>
{
var key = subspace.ExtractKey(kvp.Key, boundCheck: true);
++count;
string keyDump;
try
{
// attempts decoding it as a tuple
keyDump = TuPack.Unpack(key).ToString() !;
}
catch (Exception)
{
// not a tuple, dump as bytes
keyDump = "'" + key.ToString() + "'";
}
FdbTest.Log("- " + keyDump + " = " + kvp.Value.ToString());
});
if (count == 0)
{
FdbTest.Log("> empty !");
}
else
{
FdbTest.Log("> Found " + count + " values");
}
}
/// <summary>Delete a document from the collection</summary>
/// <param name="trans"></param>
/// <param name="ids"></param>
public async Task DeleteMultipleAsync(IFdbTransaction trans, IEnumerable <TId> ids)
{
if (trans == null)
{
throw new ArgumentNullException(nameof(trans));
}
if (ids == null)
{
throw new ArgumentNullException(nameof(ids));
}
var subspace = await this.Location.Resolve(trans);
if (subspace == null)
{
throw new InvalidOperationException($"Location '{this.Location}' referenced by Document Collection Layer was not found.");
}
foreach (var id in ids)
{
var key = subspace.EncodePartial(id);
trans.ClearRange(KeyRange.StartsWith(key));
}
}
public static async Task DumpSubspace([NotNull] IFdbReadOnlyTransaction tr, [NotNull] IFdbSubspace subspace)
{
Assert.That(tr, Is.Not.Null);
Console.WriteLine("Dumping content of subspace " + subspace.ToString() + " :");
int count = 0;
await tr
.GetRange(KeyRange.StartsWith(subspace.Key))
.ForEachAsync((kvp) =>
{
var key = subspace.ExtractKey(kvp.Key, boundCheck: true);
++count;
string keyDump = null;
try
{
// attemps decoding it as a tuple
keyDump = key.ToTuple().ToString();
}
catch (Exception)
{
// not a tuple, dump as bytes
keyDump = "'" + key.ToString() + "'";
}
Console.WriteLine("- " + keyDump + " = " + kvp.Value.ToString());
});
if (count == 0)
{
Console.WriteLine("> empty !");
}
else
{
Console.WriteLine("> Found " + count + " values");
}
}
/// <summary>Find the DCs, machines and processes in the cluster</summary>
public static async Task Topology(string[] path, IVarTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
var coords = await Fdb.System.GetCoordinatorsAsync(db, ct);
log.WriteLine($"[Cluster] {coords.Id}");
var servers = await db.QueryAsync(tr => tr
.WithReadAccessToSystemKeys()
.GetRange(KeyRange.StartsWith(Fdb.System.ServerList))
.Select(kvp => new
{
// Offsets Size Type Name Description
// 0 2 Word Version? 0100 (1.0 ?)
// 2 4 DWord ??? 0x00 0x20 0xA2 0x00
// 6 2 Word FDBMagic 0xDB 0x0F "FDB"
// 8 16 Guid NodeId Unique Process ID
// 24 16 Guid Machine "machine_id" field in foundationdb.conf (ends with 8x0 if manually specified)
// 40 16 Guid DataCenter "datacenter_id" field in foundationdb.conf (ends with 8x0 if manually specified)
// 56 4 ??? ?? 4 x 0
// 60 12 x24 ARRAY[] ?? array of 12x the same 24-byte struct defined below
// ...0 4 DWord IPAddress 01 00 00 7F => 127.0.0.1
// ...4 4 DWord Port 94 11 00 00 -> 4500
// ...8 4 DWord ?? randomish, changes every reboot
// ..12 4 DWord ?? randomish, changes every reboot
// ..16 4 DWord Size? small L-E integer, usually between 0x20 and 0x40...
// ..20 4 DWord ?? randmoish, changes every reboot
ProcessId = kvp.Value.Substring(8, 16).ToHexaString(),
MachineId = kvp.Value.Substring(24, 16).ToHexaString(),
DataCenterId = kvp.Value.Substring(40, 16).ToHexaString(),
Parts = Enumerable.Range(0, 12).Select(i =>
{
int p = 60 + 24 * i;
return(new
{
Address = new IPAddress(kvp.Value.Substring(p, 4).GetBytes().Reverse().ToArray()),
Port = kvp.Value.Substring(p + 4, 4).ToInt32(),
Unknown1 = kvp.Value.Substring(p + 8, 4).ToInt32(),
Unknown2 = kvp.Value.Substring(p + 12, 4).ToInt32(),
Unknown3 = kvp.Value.Substring(p + 16, 4).ToInt32(),
Unknown4 = kvp.Value.Substring(p + 20, 4).ToInt32(),
});
}).ToList(),
Raw = kvp.Value,
}),
ct
);
var numNodes = servers.Select(s => s.ProcessId).Distinct().Count();
var numMachines = servers.Select(s => s.MachineId).Distinct().Count();
var numDCs = servers.Select(s => s.DataCenterId).Distinct().Count();
var dcs = servers.GroupBy(x => x.DataCenterId).ToArray();
for (int dcIndex = 0; dcIndex < dcs.Length; dcIndex++)
{
var dc = dcs[dcIndex];
bool lastDc = dcIndex == dcs.Length - 1;
string dcId = dc.Key.EndsWith("0000000000000000") ? dc.Key.Substring(0, 16) : dc.Key;
log.WriteLine((lastDc ? "`- " : "|- ") + "[DataCenter] {0} (#{1})", dcId, dcIndex);
var machines = dc.GroupBy(x => x.MachineId).ToArray();
string dcPrefix = lastDc ? " " : "| ";
for (int machineIndex = 0; machineIndex < machines.Length; machineIndex++)
{
var machine = machines[machineIndex];
var lastMachine = machineIndex == machines.Length - 1;
string machineId = machine.Key.EndsWith("0000000000000000") ? machine.Key.Substring(0, 16) : machine.Key;
log.WriteLine(dcPrefix + (lastMachine ? "`- " : "|- ") + "[Machine] {0}, {1}", machine.First().Parts[0].Address, machineId);
var procs = machine.ToArray();
string machinePrefix = dcPrefix + (lastMachine ? " " : "| ");
for (int procIndex = 0; procIndex < procs.Length; procIndex++)
{
var proc = procs[procIndex];
bool lastProc = procIndex == procs.Length - 1;
log.WriteLine(machinePrefix + (lastProc ? "`- " : "|- ") + "[Process] {0}:{1}, {2}", proc.Parts[0].Address, proc.Parts[0].Port, proc.ProcessId);
//foreach (var part in proc.Parts)
//{
// log.WriteLine(machinePrefix + "| -> {0}, {1}, {2:X8}, {3:X8}, {4}, {5:X8}", part.Address, part.Port, part.Unknown1, part.Unknown2, part.Unknown3, part.Unknown4);
//}
}
}
}
log.WriteLine();
Program.StdOut(log, $"Found {numNodes:N0} process(es) on {numMachines:N0} machine(s) in {numDCs:N0} datacenter(s)", ConsoleColor.White);
log.WriteLine();
}
public static async Task Map(string[] path, IVarTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
// we want to merge the map of shards, with the map of directories from the Directory Layer, and count for each directory how many shards intersect
bool progress = log == Console.Out;
var folder = await TryOpenCurrentDirectoryAsync(path, db, ct);
if (folder == null)
{
Program.Error(log, "# Directory not found");
return;
}
Program.StdOut(log, "Listing all shards...");
// note: this may break in future versions of the DL! Maybe we need a custom API to get a flat list of all directories in a DL that span a specific range ?
var span = folder.DirectoryLayer.ContentSubspace.Keys.ToRange();
var shards = await Fdb.System.GetChunksAsync(db, span, ct);
int totalShards = shards.Count;
Program.StdOut(log, $"> Found {totalShards} shard(s) in partition /{folder.DirectoryLayer.FullName}", ConsoleColor.Gray);
Program.StdOut(log, "Listing all directories...");
var map = new Dictionary <string, int>(StringComparer.Ordinal);
void Account(string[] p, int c)
{
for (int i = 1; i <= p.Length; i++)
{
var s = "/" + string.Join("/", p, 0, i);
map[s] = map.TryGetValue(s, out int x) ? (x + c) : c;
}
}
var work = new Stack <IFdbDirectory>();
work.Push(folder);
var dirs = new List <IFdbDirectory>();
int n = 0;
while (work.Count > 0)
{
var cur = work.Pop();
// skip sub partitions
var names = await cur.ListAsync(db, ct);
foreach (var name in names)
{
var sub = await cur.TryOpenAsync(db, name, ct);
if (sub != null)
{
var p = sub.FullName;
if (sub is FdbDirectoryPartition)
{
if (progress)
{
log.Write("\r");
}
Program.StdOut(log, $"! Skipping partition {sub.Name} ", ConsoleColor.DarkRed);
n = 0;
continue;
}
if (progress)
{
log.Write($"\r/{p}{(p.Length > n ? String.Empty : new string(' ', n - p.Length))}");
}
n = p.Length;
work.Push(sub);
dirs.Add(sub);
}
}
}
if (progress)
{
log.Write("\r" + new string(' ', n + 2) + "\r");
}
Program.StdOut(log, $"> Found {dirs.Count} sub-directories", ConsoleColor.Gray);
log.WriteLine();
Program.StdOut(log, "Estimating size of each directory...");
int foundShards = 0;
n = 0;
int max = 0;
IFdbDirectory bigBad = null;
foreach (var dir in dirs)
{
if (progress)
{
log.Write($"\r> {dir.Name}{(dir.Name.Length > n ? String.Empty : new string(' ', n - dir.Name.Length))}");
}
n = dir.Name.Length;
var p = dir.Path.ToArray();
var key = ((KeySubspace)dir).GetPrefix();
// verify that the subspace has at least one key inside
var bounds = await db.ReadAsync(async (tr) =>
{
var kvs = await Task.WhenAll(
tr.GetRange(KeyRange.StartsWith(key)).FirstOrDefaultAsync(),
tr.GetRange(KeyRange.StartsWith(key)).LastOrDefaultAsync()
);
return(new { Min = kvs[0].Key, Max = kvs[1].Key });
}, ct);
if (bounds.Min.HasValue)
{ // folder is not empty
shards = await Fdb.System.GetChunksAsync(db, KeyRange.StartsWith(key), ct);
//TODO: we still need to check if the first and last shard really intersect the subspace
// we need to check if the shards actually contain data
//Console.WriteLine("/{0} under {1} with {2} shard(s)", string.Join("/", p), FdbKey.Dump(key), shards.Count);
foundShards += shards.Count;
Account(p, shards.Count);
if (shards.Count > max)
{
max = shards.Count; bigBad = dir;
}
}
else
{
Account(p, 0);
}
}
if (progress)
{
log.Write("\r" + new string(' ', n + 2) + "\r");
}
Program.StdOut(log, $"> Found a total of {foundShards:N0} shard(s) in {dirs.Count:N0} folder(s)", ConsoleColor.Gray);
log.WriteLine();
Program.StdOut(log, "Shards %Total Path");
foreach (var kvp in map.OrderBy(x => x.Key))
{
Program.StdOut(log, $"{kvp.Value,6} {RobustHistogram.FormatHistoBar((double)kvp.Value / foundShards, 20),-20} {kvp.Key}", ConsoleColor.Gray);
}
log.WriteLine();
if (bigBad != null)
{
Program.StdOut(log, $"Biggest folder is /{bigBad.FullName} with {max} shards ({100.0 * max / totalShards:N1}% total, {100.0 * max / foundShards:N1}% subtree)");
log.WriteLine();
}
}
public static async Task ClearRange(string[] path, IVarTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
if (path == null || path.Length == 0)
{
Program.Error(log, "Cannot directory list the content of Root Partition.");
return;
}
var folder = await db.Directory.TryOpenAsync(db, path, ct : ct);
if (folder == null)
{
Program.Error(log, "The directory does not exist anymore");
return;
}
if (folder.Layer == FdbDirectoryPartition.LayerId)
{
Program.Error(log, "Cannot clear the content of a Directory Partition!");
return;
}
if (extras.Count == 0)
{
Program.Error(log, "You must specify a key of range of keys!");
return;
}
KeyRange range;
if (extras.Get <string>(0) == "*")
{ // clear all!
range = folder.ToRange();
}
else
{
object from = extras[0];
object to = extras.Count > 1 ? extras[1] : null;
if (to == null)
{
range = KeyRange.StartsWith(MakeKey(folder, from));
}
else
{
range = new KeyRange(
MakeKey(folder, from),
MakeKey(folder, to)
);
}
}
Program.Comment(log, "Clearing range " + range.ToString());
bool empty = await db.ReadWriteAsync(async tr =>
{
var any = await tr.GetRangeAsync(range, new FdbRangeOptions {
Limit = 1
});
if (any.Count == 0)
{
return(true);
}
tr.ClearRange(folder.ToRange());
return(false);
}, ct);
if (empty)
{
Program.StdOut(log, $"Directory {String.Join("/", path)} was already empty.", ConsoleColor.Cyan);
}
else
{
Program.Success(log, $"Cleared all content of Directory {String.Join("/", path)}.");
}
}
public static async Task Sampling(string[] path, IVarTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
double ratio = 0.1d;
bool auto = true;
if (extras.Count > 0)
{
double x = extras.Get <double>(0);
if (x > 0 && x <= 1)
{
ratio = x;
}
auto = false;
}
var folder = await TryOpenCurrentDirectoryAsync(path, db, ct);
KeyRange span;
if (folder is FdbDirectorySubspace)
{
span = KeyRange.StartsWith((folder as FdbDirectorySubspace).Copy().GetPrefix());
log.WriteLine($"Reading list of shards for /{String.Join("/", path)} under {FdbKey.Dump(span.Begin)} ...");
}
else
{
log.WriteLine("Reading list of shards for the whole cluster ...");
span = KeyRange.All;
}
// dump keyServers
var ranges = await Fdb.System.GetChunksAsync(db, span, ct);
log.WriteLine($"> Found {ranges.Count:N0} shard(s)");
// take a sample
var samples = new List <KeyRange>();
if (ranges.Count <= 32)
{ // small enough to scan it all
samples.AddRange(ranges);
log.WriteLine($"Sampling all {samples.Count:N0} shards ...");
}
else
{ // need to take a random subset
var rnd = new Random();
int sz = Math.Max((int)Math.Ceiling(ratio * ranges.Count), 1);
if (auto)
{
if (sz > 100)
{
sz = 100; //SAFETY
}
if (sz < 32)
{
sz = Math.Max(sz, Math.Min(32, ranges.Count));
}
}
var population = new List <KeyRange>(ranges);
for (int i = 0; i < sz; i++)
{
int p = rnd.Next(population.Count);
samples.Add(population[p]);
population.RemoveAt(p);
}
log.WriteLine($"Sampling {samples.Count:N0} out of {ranges.Count:N0} shards ({(100.0 * samples.Count / ranges.Count):N1}%) ...");
}
log.WriteLine();
const string FORMAT_STRING = "{0,9} ║{1,10}{6,6} {2,-29} ║{3,10}{7,7} {4,-37} ║{5,10}";
const string SCALE_KEY = "....--------========########M";
const string SCALE_VAL = "....--------========########@@@@@@@@M";
log.WriteLine(FORMAT_STRING, "Count", "Keys", SCALE_KEY, "Values", SCALE_VAL, "Total", "med.", "med.");
var rangeOptions = new FdbRangeOptions {
Mode = FdbStreamingMode.WantAll
};
samples = samples.OrderBy(x => x.Begin).ToList();
long globalSize = 0;
long globalCount = 0;
int workers = 8; // Math.Max(4, Environment.ProcessorCount);
var sw = Stopwatch.StartNew();
var tasks = new List <Task>();
int n = samples.Count;
while (samples.Count > 0)
{
while (tasks.Count < workers && samples.Count > 0)
{
var range = samples[0];
samples.RemoveAt(0);
tasks.Add(Task.Run(async() =>
{
var kk = new RobustHistogram(RobustHistogram.TimeScale.Ticks);
var vv = new RobustHistogram(RobustHistogram.TimeScale.Ticks);
#region Method 1: get_range everything...
using (var tr = db.BeginTransaction(ct))
{
long keySize = 0;
long valueSize = 0;
long count = 0;
int iter = 0;
var beginSelector = KeySelector.FirstGreaterOrEqual(range.Begin);
var endSelector = KeySelector.FirstGreaterOrEqual(range.End);
while (true)
{
FdbRangeChunk data = default(FdbRangeChunk);
FdbException error = null;
try
{
data = await tr.Snapshot.GetRangeAsync(
beginSelector,
endSelector,
rangeOptions,
iter
).ConfigureAwait(false);
}
catch (FdbException e)
{
error = e;
}
if (error != null)
{
await tr.OnErrorAsync(error.Code).ConfigureAwait(false);
continue;
}
if (data.Count == 0)
{
break;
}
count += data.Count;
foreach (var kvp in data)
{
keySize += kvp.Key.Count;
valueSize += kvp.Value.Count;
kk.Add(TimeSpan.FromTicks(kvp.Key.Count));
vv.Add(TimeSpan.FromTicks(kvp.Value.Count));
}
if (!data.HasMore)
{
break;
}
beginSelector = KeySelector.FirstGreaterThan(data.Last);
++iter;
}
long totalSize = keySize + valueSize;
Interlocked.Add(ref globalSize, totalSize);
Interlocked.Add(ref globalCount, count);
lock (log)
{
log.WriteLine(FORMAT_STRING, count.ToString("N0"), FormatSize(keySize), kk.GetDistribution(begin: 1, end: 12000, fold: 2), FormatSize(valueSize), vv.GetDistribution(begin: 1, end: 120000, fold: 2), FormatSize(totalSize), FormatSize((int)Math.Ceiling(kk.Median)), FormatSize((int)Math.Ceiling(vv.Median)));
}
}
#endregion
#region Method 2: estimate the count using key selectors...
//long counter = await Fdb.System.EstimateCountAsync(db, range, ct);
//Console.WriteLine("COUNT = " + counter.ToString("N0"));
#endregion
}, ct));
}
var done = await Task.WhenAny(tasks);
tasks.Remove(done);
}
await Task.WhenAll(tasks);
sw.Stop();
log.WriteLine();
if (n != ranges.Count)
{
log.WriteLine($"Sampled {FormatSize(globalSize)} ({globalSize:N0} bytes) and {globalCount:N0} keys in {sw.Elapsed.TotalSeconds:N1} sec");
log.WriteLine($"> Estimated total size is {FormatSize(globalSize * ranges.Count / n)}");
}
else
{
log.WriteLine($"Found {FormatSize(globalSize)} ({globalSize:N0} bytes) and {globalCount:N0} keys in {sw.Elapsed.TotalSeconds:N1} sec");
// compare to the whole cluster
ranges = await Fdb.System.GetChunksAsync(db, FdbKey.MinValue, FdbKey.MaxValue, ct);
log.WriteLine($"> This directory contains ~{(100.0 * n / ranges.Count):N2}% of all data");
}
log.WriteLine();
}
private async Task RunAsync(IFdbDatabase db, IDynamicKeySubspace location, CancellationToken ct, Action done, int N, int K, int W)
{
if (db == null)
{
throw new ArgumentNullException(nameof(db));
}
StringBuilder sb = new StringBuilder();
db = new FdbLoggedDatabase(db, false, false, (log) =>
{
sb.AppendLine(log.Log.GetTimingsReport(true));
//Console.WriteLine(log.Log.GetTimingsReport(true));
});
try
{
var workerPool = new FdbWorkerPool(location);
Console.WriteLine($"workerPool at {location.GetPrefix():P}");
var workerSignal = new AsyncCancelableMutex(ct);
var clientSignal = new AsyncCancelableMutex(ct);
int taskCounter = 0;
int msgSent = 0;
int msgReceived = 0;
Func <FdbWorkerMessage, CancellationToken, Task> handler = async(msg, _ct) =>
{
Interlocked.Increment(ref msgReceived);
//await Task.Delay(10 + Math.Abs(msg.Id.GetHashCode()) % 50);
await Task.Delay(10).ConfigureAwait(false);
};
Func <int, Task> worker = async(id) =>
{
await workerSignal.Task.ConfigureAwait(false);
Console.WriteLine("Worker #" + id + " is starting");
try
{
await workerPool.RunWorkerAsync(db, handler, ct).ConfigureAwait(false);
}
finally
{
Console.WriteLine("Worker #" + id + " has stopped");
}
};
Func <int, Task> client = async(id) =>
{
await clientSignal.Task.ConfigureAwait(false);
await Task.Delay(10).ConfigureAwait(false);
var rnd = new Random(id * 111);
for (int i = 0; i < N; i++)
{
var taskId = Slice.FromString("T" + Interlocked.Increment(ref taskCounter));
var taskBody = Slice.FromString("Message " + (i + 1) + " of " + N + " from client #" + id);
await workerPool.ScheduleTaskAsync(db, taskId, taskBody, ct).ConfigureAwait(false);
Interlocked.Increment(ref msgSent);
//if (i > 0 && i % 10 == 0) Console.WriteLine("@@@ Client#" + id + " pushed " + (i + 1) + " / " + N + " messages");
switch (rnd.Next(5))
{
case 0: await Task.Delay(10).ConfigureAwait(false); break;
case 1: await Task.Delay(100).ConfigureAwait(false); break;
case 2: await Task.Delay(500).ConfigureAwait(false); break;
}
}
Console.WriteLine("@@@ Client#" + id + " has finished!");
};
Func <string, Task> dump = async(label) =>
{
Console.WriteLine($"<dump label=\'{label}\' key=\'{location.GetPrefix():P}\'>");
using (var tr = db.BeginTransaction(ct))
{
await tr.Snapshot
.GetRange(KeyRange.StartsWith(location.GetPrefix()))
.ForEachAsync((kvp) =>
{
Console.WriteLine($" - {location.Keys.Unpack(kvp.Key)} = {kvp.Value:V}");
}).ConfigureAwait(false);
}
Console.WriteLine("</dump>");
};
var workers = Enumerable.Range(0, W).Select((i) => worker(i)).ToArray();
var clients = Enumerable.Range(0, K).Select((i) => client(i)).ToArray();
DateTime start = DateTime.Now;
DateTime last = start;
int lastHandled = -1;
using (var timer = new Timer((_) =>
{
var now = DateTime.Now;
Console.WriteLine("@@@ T=" + now.Subtract(start) + ", sent: " + msgSent.ToString("N0") + ", recv: " + msgReceived.ToString("N0"));
Console.WriteLine("### Workers: " + workerPool.IdleWorkers + " / " + workerPool.ActiveWorkers + " (" + new string('#', workerPool.IdleWorkers) + new string('.', workerPool.ActiveWorkers - workerPool.IdleWorkers) + "), sent: " + workerPool.MessageScheduled.ToString("N0") + ", recv: " + workerPool.MessageReceived.ToString("N0") + ", delta: " + (workerPool.MessageScheduled - workerPool.MessageReceived).ToString("N0") + ", busy: " + workerPool.WorkerBusyTime + " (avg " + workerPool.WorkerAverageBusyDuration.TotalMilliseconds.ToString("N3") + " ms)");
if (now.Subtract(last).TotalSeconds >= 10)
{
//dump("timer").GetAwaiter().GetResult();
last = now;
if (lastHandled == msgReceived)
{ // STALL ?
Console.WriteLine("STALL! ");
done();
}
lastHandled = msgReceived;
}
if (msgReceived >= K * N)
{
dump("complete").GetAwaiter().GetResult();
done();
}
}, null, 1000, 1000))
{
var sw = Stopwatch.StartNew();
// start the workers
workerSignal.Set(async: true);
await Task.Delay(500);
await dump("workers started");
// start the clients
clientSignal.Set(async: true);
await Task.WhenAll(clients);
Console.WriteLine("Clients completed after " + sw.Elapsed);
await Task.WhenAll(workers);
Console.WriteLine("Workers completed after " + sw.Elapsed);
}
}
finally
{
Console.WriteLine("---------------------------------------------------------------------------");
Console.WriteLine("Transaction logs:");
Console.WriteLine();
Console.WriteLine(sb.ToString());
}
}
public async Task Run(IFdbDatabase db, TextWriter log, CancellationToken ct)
{
// estimate the number of machines...
Console.WriteLine("# Detecting cluster topology...");
var servers = await db.QueryAsync(tr => tr
.WithReadAccessToSystemKeys()
.GetRange(KeyRange.StartsWith(Fdb.System.ServerList))
.Select(kvp => new
{
Node = kvp.Value.Substring(8, 16).ToHexaString(),
Machine = kvp.Value.Substring(24, 16).ToHexaString(),
DataCenter = kvp.Value.Substring(40, 16).ToHexaString()
}),
ct
);
var numNodes = servers.Select(s => s.Node).Distinct().Count();
var numMachines = servers.Select(s => s.Machine).Distinct().Count();
var numDCs = servers.Select(s => s.DataCenter).Distinct().Count();
Console.WriteLine("# > Found " + numNodes + " process(es) on " + numMachines + " machine(s) in " + numDCs + " datacenter(s)");
Console.WriteLine("# Reading list of shards...");
// dump keyServers
var ranges = await Fdb.System.GetChunksAsync(db, FdbKey.MinValue, FdbKey.MaxValue, ct);
Console.WriteLine("# > Found " + ranges.Count + " shards:");
// take a sample
var rnd = new Random(1234);
int sz = Math.Max((int)Math.Ceiling(this.Ratio * ranges.Count), 1);
if (sz > 500)
{
sz = 500; //SAFETY
}
if (sz < 50)
{
sz = Math.Max(sz, Math.Min(50, ranges.Count));
}
var samples = new List <KeyRange>();
for (int i = 0; i < sz; i++)
{
int p = rnd.Next(ranges.Count);
samples.Add(ranges[p]);
ranges.RemoveAt(p);
}
Console.WriteLine("# Sampling " + sz + " out of " + ranges.Count + " shards (" + (100.0 * sz / ranges.Count).ToString("N1") + "%) ...");
Console.WriteLine("{0,9}{1,10}{2,10}{3,10} : K+V size distribution", "Count", "Keys", "Values", "Total");
var rangeOptions = new FdbRangeOptions {
Mode = FdbStreamingMode.WantAll
};
samples = samples.OrderBy(x => x.Begin).ToList();
long total = 0;
int workers = Math.Min(numMachines, 8);
var sw = Stopwatch.StartNew();
var tasks = new List <Task>();
while (samples.Count > 0)
{
while (tasks.Count < workers && samples.Count > 0)
{
var range = samples[0];
samples.RemoveAt(0);
tasks.Add(Task.Run(async() =>
{
var hh = new RobustHistogram(RobustHistogram.TimeScale.Ticks);
#region Method 1: get_range everything...
using (var tr = await db.BeginTransactionAsync(ct))
{
long keySize = 0;
long valueSize = 0;
long count = 0;
int iter = 0;
var beginSelector = KeySelector.FirstGreaterOrEqual(range.Begin);
var endSelector = KeySelector.FirstGreaterOrEqual(range.End);
while (true)
{
FdbRangeChunk data = default(FdbRangeChunk);
FdbException error = null;
try
{
data = await tr.Snapshot.GetRangeAsync(
beginSelector,
endSelector,
rangeOptions,
iter
).ConfigureAwait(false);
}
catch (FdbException e)
{
error = e;
}
if (error != null)
{
await tr.OnErrorAsync(error.Code).ConfigureAwait(false);
continue;
}
if (data.Count == 0)
{
break;
}
count += data.Count;
foreach (var kvp in data)
{
keySize += kvp.Key.Count;
valueSize += kvp.Value.Count;
hh.Add(TimeSpan.FromTicks(kvp.Key.Count + kvp.Value.Count));
}
if (!data.HasMore)
{
break;
}
beginSelector = KeySelector.FirstGreaterThan(data.Last);
++iter;
}
long totalSize = keySize + valueSize;
Interlocked.Add(ref total, totalSize);
Console.WriteLine("{0,9}{1,10}{2,10}{3,10} : {4}", count.ToString("N0"), FormatSize(keySize), FormatSize(valueSize), FormatSize(totalSize), hh.GetDistribution(begin: 1, end: 10000, fold: 2));
}
#endregion
#region Method 2: estimate the count using key selectors...
//long counter = await Fdb.System.EstimateCountAsync(db, range, ct);
//Console.WriteLine("COUNT = " + counter.ToString("N0"));
#endregion
}, ct));
}
var done = await Task.WhenAny(tasks);
tasks.Remove(done);
}
await Task.WhenAll(tasks);
sw.Stop();
Console.WriteLine("> Sampled " + FormatSize(total) + " (" + total.ToString("N0") + " bytes) in " + sw.Elapsed.TotalSeconds.ToString("N1") + " sec");
Console.WriteLine("> Estimated total size is " + FormatSize(total * ranges.Count / sz));
}
public async Task Test_Can_Log_A_Transaction()
{
const int N = 10;
using (var db = await OpenTestPartitionAsync())
{
// get a tuple view of the directory
var location = (await GetCleanDirectory(db, "Logging")).Keys;
// note: ensure that all methods are JITed
await db.ReadWriteAsync(async (tr) =>
{
await tr.GetReadVersionAsync();
tr.Set(location.Encode("Warmup", 0), Slice.FromInt32(1));
tr.Clear(location.Encode("Warmup", 1));
await tr.GetAsync(location.Encode("Warmup", 2));
await tr.GetRange(KeyRange.StartsWith(location.Encode("Warmup", 3))).ToListAsync();
tr.ClearRange(location.Encode("Warmup", 4), location.Encode("Warmup", 5));
}, this.Cancellation);
await db.WriteAsync((tr) =>
{
var rnd = new Random();
tr.Set(location.Encode("One"), Slice.FromString("111111"));
tr.Set(location.Encode("Two"), Slice.FromString("222222"));
for (int j = 0; j < 4; j++)
{
for (int i = 0; i < 100; i++)
{
tr.Set(location.Encode("Range", j, rnd.Next(1000)), Slice.Empty);
}
}
for (int j = 0; j < N; j++)
{
tr.Set(location.Encode("X", j), Slice.FromInt32(j));
tr.Set(location.Encode("Y", j), Slice.FromInt32(j));
tr.Set(location.Encode("Z", j), Slice.FromInt32(j));
tr.Set(location.Encode("W", j), Slice.FromInt32(j));
}
}, this.Cancellation);
bool first = true;
Action <FdbLoggedTransaction> logHandler = (tr) =>
{
if (first)
{
Log(tr.Log.GetCommandsReport());
first = false;
}
Log(tr.Log.GetTimingsReport(true));
};
// create a logged version of the database
var logged = new FdbLoggedDatabase(db, false, false, logHandler);
for (int k = 0; k < N; k++)
{
Log("==== " + k + " ==== ");
Log();
await logged.ReadWriteAsync(async (tr) =>
{
Assert.That(tr, Is.InstanceOf <FdbLoggedTransaction>());
//tr.SetOption(FdbTransactionOption.CausalReadRisky);
long ver = await tr.GetReadVersionAsync().ConfigureAwait(false);
await tr.GetAsync(location.Encode("One")).ConfigureAwait(false);
await tr.GetAsync(location.Encode("NotFound")).ConfigureAwait(false);
tr.Set(location.Encode("Write"), Slice.FromString("abcdef" + k.ToString()));
//tr.Annotate("BEFORE");
//await Task.Delay(TimeSpan.FromMilliseconds(10));
//tr.Annotate("AFTER");
//await tr.Snapshot.GetAsync(location.Pack("Snap")).ConfigureAwait(false);
tr.Annotate("This is a comment");
//await tr.GetRangeAsync(KeySelector.LastLessOrEqual(location.Pack("A")), KeySelector.FirstGreaterThan(location.Pack("Z"))).ConfigureAwait(false);
await Task.WhenAll(
tr.GetRange(KeyRange.StartsWith(location.Encode("Range", 0))).ToListAsync(),
tr.GetRange(location.Encode("Range", 1, 0), location.Encode("Range", 1, 200)).ToListAsync(),
tr.GetRange(location.Encode("Range", 2, 400), location.Encode("Range", 2, 600)).ToListAsync(),
tr.GetRange(location.Encode("Range", 3, 800), location.Encode("Range", 3, 1000)).ToListAsync()
).ConfigureAwait(false);
await tr.GetAsync(location.Encode("Two")).ConfigureAwait(false);
await tr.GetValuesAsync(Enumerable.Range(0, N).Select(x => location.Encode("X", x))).ConfigureAwait(false);
for (int i = 0; i < N; i++)
{
await tr.GetAsync(location.Encode("Z", i)).ConfigureAwait(false);
}
await Task.WhenAll(Enumerable.Range(0, N / 2).Select(x => tr.GetAsync(location.Encode("Y", x)))).ConfigureAwait(false);
await Task.WhenAll(Enumerable.Range(N / 2, N / 2).Select(x => tr.GetAsync(location.Encode("Y", x)))).ConfigureAwait(false);
await Task.WhenAll(
tr.GetAsync(location.Encode("W", 1)),
tr.GetAsync(location.Encode("W", 2)),
tr.GetAsync(location.Encode("W", 3))
).ConfigureAwait(false);
tr.Set(location.Encode("Write2"), Slice.FromString("ghijkl" + k.ToString()));
tr.Clear(location.Encode("Clear", "0"));
tr.ClearRange(location.Encode("Clear", "A"), location.Encode("Clear", "Z"));
if (tr.Context.Retries == 0)
{
// make it fail
//throw new FdbException(FdbError.PastVersion, "fake timeout");
}
}, this.Cancellation);
}
}
}
/// <summary>Remove all the values for a specific key</summary>
/// <param name="trans"></param>
/// <param name="key"></param>
/// <returns></returns>
public void Remove([NotNull] IFdbTransaction trans, TKey key)
{
if (trans == null) throw new ArgumentNullException(nameof(trans));
trans.ClearRange(KeyRange.StartsWith(this.Subspace.Keys.EncodePartial(key)));
}