/// <summary>Shows the first few keys of a directory</summary>
public static async Task Show(string[] path, IVarTuple extras, bool reverse, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
int count = 20;
if (extras.Count > 0)
{
int x = extras.Get <int>(0);
if (x > 0)
{
count = x;
}
}
if (path == null || path.Length == 0)
{
Program.Error(log, "Cannot directory list the content of Root Partition.");
return;
}
// look if there is something under there
var folder = await db.Directory.TryOpenAsync(db, path, ct : ct);
if (folder != null)
{
if (folder.Layer == FdbDirectoryPartition.LayerId)
{
Program.Error(log, "Cannot list the content of a Directory Partition!");
return;
}
Program.Comment(log, $"# Content of {FdbKey.Dump(folder.GetPrefix())} [{folder.GetPrefix().ToHexaString(' ')}]");
var keys = await db.QueryAsync((tr) =>
{
var query = tr.GetRange(folder.Keys.ToRange());
return(reverse
? query.Reverse().Take(count)
: query.Take(count + 1));
}, ct : ct);
if (keys.Count > 0)
{
if (reverse)
{
keys.Reverse();
}
foreach (var key in keys.Take(count))
{
log.WriteLine($"...{FdbKey.Dump(folder.ExtractKey(key.Key))} = {key.Value:V}");
}
if (!reverse && keys.Count == count + 1)
{
log.WriteLine("... more");
}
}
else
{
log.WriteLine("Folder is empty");
}
}
}
/// <summary>Shows the first few keys of a directory</summary>
public static async Task Show(string[] path, IFdbTuple extras, bool reverse, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
int count = 20;
if (extras.Count > 0)
{
int x = extras.Get <int>(0);
if (x > 0)
{
count = x;
}
}
// look if there is something under there
var folder = await db.Directory.TryOpenAsync(path, cancellationToken : ct);
if (folder != null)
{
log.WriteLine("# Content of {0} [{1}]", FdbKey.Dump(folder.Key), folder.Key.ToHexaString(' '));
var keys = await db.QueryAsync((tr) =>
{
var query = tr.GetRange(folder.ToRange());
return(reverse
? query.Reverse().Take(count)
: query.Take(count + 1));
}, cancellationToken : ct);
if (keys.Count > 0)
{
if (reverse)
{
keys.Reverse();
}
foreach (var key in keys.Take(count))
{
log.WriteLine("...{0} = {1}", FdbKey.Dump(folder.Extract(key.Key)), key.Value.ToAsciiOrHexaString());
}
if (!reverse && keys.Count == count + 1)
{
log.WriteLine("... more");
}
}
else
{
log.WriteLine(" no content found");
}
}
}
/// <summary>Find the DCs, machines and processes in the cluster</summary>
public static async Task Topology(string[] path, IVarTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
var coords = await Fdb.System.GetCoordinatorsAsync(db, ct);
log.WriteLine($"[Cluster] {coords.Id}");
var servers = await db.QueryAsync(tr => tr
.WithReadAccessToSystemKeys()
.GetRange(KeyRange.StartsWith(Fdb.System.ServerList))
.Select(kvp => new
{
// Offsets Size Type Name Description
// 0 2 Word Version? 0100 (1.0 ?)
// 2 4 DWord ??? 0x00 0x20 0xA2 0x00
// 6 2 Word FDBMagic 0xDB 0x0F "FDB"
// 8 16 Guid NodeId Unique Process ID
// 24 16 Guid Machine "machine_id" field in foundationdb.conf (ends with 8x0 if manually specified)
// 40 16 Guid DataCenter "datacenter_id" field in foundationdb.conf (ends with 8x0 if manually specified)
// 56 4 ??? ?? 4 x 0
// 60 12 x24 ARRAY[] ?? array of 12x the same 24-byte struct defined below
// ...0 4 DWord IPAddress 01 00 00 7F => 127.0.0.1
// ...4 4 DWord Port 94 11 00 00 -> 4500
// ...8 4 DWord ?? randomish, changes every reboot
// ..12 4 DWord ?? randomish, changes every reboot
// ..16 4 DWord Size? small L-E integer, usually between 0x20 and 0x40...
// ..20 4 DWord ?? randmoish, changes every reboot
ProcessId = kvp.Value.Substring(8, 16).ToHexaString(),
MachineId = kvp.Value.Substring(24, 16).ToHexaString(),
DataCenterId = kvp.Value.Substring(40, 16).ToHexaString(),
Parts = Enumerable.Range(0, 12).Select(i =>
{
int p = 60 + 24 * i;
return(new
{
Address = new IPAddress(kvp.Value.Substring(p, 4).GetBytes().Reverse().ToArray()),
Port = kvp.Value.Substring(p + 4, 4).ToInt32(),
Unknown1 = kvp.Value.Substring(p + 8, 4).ToInt32(),
Unknown2 = kvp.Value.Substring(p + 12, 4).ToInt32(),
Unknown3 = kvp.Value.Substring(p + 16, 4).ToInt32(),
Unknown4 = kvp.Value.Substring(p + 20, 4).ToInt32(),
});
}).ToList(),
Raw = kvp.Value,
}),
ct
);
var numNodes = servers.Select(s => s.ProcessId).Distinct().Count();
var numMachines = servers.Select(s => s.MachineId).Distinct().Count();
var numDCs = servers.Select(s => s.DataCenterId).Distinct().Count();
var dcs = servers.GroupBy(x => x.DataCenterId).ToArray();
for (int dcIndex = 0; dcIndex < dcs.Length; dcIndex++)
{
var dc = dcs[dcIndex];
bool lastDc = dcIndex == dcs.Length - 1;
string dcId = dc.Key.EndsWith("0000000000000000") ? dc.Key.Substring(0, 16) : dc.Key;
log.WriteLine((lastDc ? "`- " : "|- ") + "[DataCenter] {0} (#{1})", dcId, dcIndex);
var machines = dc.GroupBy(x => x.MachineId).ToArray();
string dcPrefix = lastDc ? " " : "| ";
for (int machineIndex = 0; machineIndex < machines.Length; machineIndex++)
{
var machine = machines[machineIndex];
var lastMachine = machineIndex == machines.Length - 1;
string machineId = machine.Key.EndsWith("0000000000000000") ? machine.Key.Substring(0, 16) : machine.Key;
log.WriteLine(dcPrefix + (lastMachine ? "`- " : "|- ") + "[Machine] {0}, {1}", machine.First().Parts[0].Address, machineId);
var procs = machine.ToArray();
string machinePrefix = dcPrefix + (lastMachine ? " " : "| ");
for (int procIndex = 0; procIndex < procs.Length; procIndex++)
{
var proc = procs[procIndex];
bool lastProc = procIndex == procs.Length - 1;
log.WriteLine(machinePrefix + (lastProc ? "`- " : "|- ") + "[Process] {0}:{1}, {2}", proc.Parts[0].Address, proc.Parts[0].Port, proc.ProcessId);
//foreach (var part in proc.Parts)
//{
// log.WriteLine(machinePrefix + "| -> {0}, {1}, {2:X8}, {3:X8}, {4}, {5:X8}", part.Address, part.Port, part.Unknown1, part.Unknown2, part.Unknown3, part.Unknown4);
//}
}
}
}
log.WriteLine();
Program.StdOut(log, $"Found {numNodes:N0} process(es) on {numMachines:N0} machine(s) in {numDCs:N0} datacenter(s)", ConsoleColor.White);
log.WriteLine();
}
public async Task Run(IFdbDatabase db, TextWriter log, CancellationToken ct)
{
// estimate the number of machines...
Console.WriteLine("# Detecting cluster topology...");
var servers = await db.QueryAsync(tr => tr
.WithReadAccessToSystemKeys()
.GetRange(FdbKeyRange.StartsWith(Fdb.System.ServerList))
.Select(kvp => new
{
Node = kvp.Value.Substring(8, 16).ToHexaString(),
Machine = kvp.Value.Substring(24, 16).ToHexaString(),
DataCenter = kvp.Value.Substring(40, 16).ToHexaString()
}),
ct
);
var numNodes = servers.Select(s => s.Node).Distinct().Count();
var numMachines = servers.Select(s => s.Machine).Distinct().Count();
var numDCs = servers.Select(s => s.DataCenter).Distinct().Count();
Console.WriteLine("# > Found " + numNodes + " process(es) on " + numMachines + " machine(s) in " + numDCs + " datacenter(s)");
Console.WriteLine("# Reading list of shards...");
// dump keyServers
var ranges = await Fdb.System.GetChunksAsync(db, FdbKey.MinValue, FdbKey.MaxValue, ct);
Console.WriteLine("# > Found " + ranges.Count + " shards:");
// take a sample
var rnd = new Random(1234);
int sz = Math.Max((int)Math.Ceiling(this.Ratio * ranges.Count), 1);
if (sz > 500)
{
sz = 500; //SAFETY
}
if (sz < 50)
{
sz = Math.Max(sz, Math.Min(50, ranges.Count));
}
var samples = new List <FdbKeyRange>();
for (int i = 0; i < sz; i++)
{
int p = rnd.Next(ranges.Count);
samples.Add(ranges[p]);
ranges.RemoveAt(p);
}
Console.WriteLine("# Sampling " + sz + " out of " + ranges.Count + " shards (" + (100.0 * sz / ranges.Count).ToString("N1") + "%) ...");
Console.WriteLine("{0,9}{1,10}{2,10}{3,10} : K+V size distribution", "Count", "Keys", "Values", "Total");
var rangeOptions = new FdbRangeOptions {
Mode = FdbStreamingMode.WantAll
};
samples = samples.OrderBy(x => x.Begin).ToList();
long total = 0;
int workers = Math.Min(numMachines, 8);
var sw = Stopwatch.StartNew();
var tasks = new List <Task>();
while (samples.Count > 0)
{
while (tasks.Count < workers && samples.Count > 0)
{
var range = samples[0];
samples.RemoveAt(0);
tasks.Add(Task.Run(async() =>
{
var hh = new RobustHistogram(RobustHistogram.TimeScale.Ticks);
#region Method 1: get_range everything...
using (var tr = db.BeginTransaction(ct))
{
long keySize = 0;
long valueSize = 0;
long count = 0;
int iter = 0;
var beginSelector = FdbKeySelector.FirstGreaterOrEqual(range.Begin);
var endSelector = FdbKeySelector.FirstGreaterOrEqual(range.End);
while (true)
{
FdbRangeChunk data = default(FdbRangeChunk);
FdbException error = null;
try
{
data = await tr.Snapshot.GetRangeAsync(
beginSelector,
endSelector,
rangeOptions,
iter
).ConfigureAwait(false);
}
catch (FdbException e)
{
error = e;
}
if (error != null)
{
await tr.OnErrorAsync(error.Code).ConfigureAwait(false);
continue;
}
if (data.Count == 0)
{
break;
}
count += data.Count;
foreach (var kvp in data.Chunk)
{
keySize += kvp.Key.Count;
valueSize += kvp.Value.Count;
hh.Add(TimeSpan.FromTicks(kvp.Key.Count + kvp.Value.Count));
}
if (!data.HasMore)
{
break;
}
beginSelector = FdbKeySelector.FirstGreaterThan(data.Last.Key);
++iter;
}
long totalSize = keySize + valueSize;
Interlocked.Add(ref total, totalSize);
Console.WriteLine("{0,9}{1,10}{2,10}{3,10} : {4}", count.ToString("N0"), FormatSize(keySize), FormatSize(valueSize), FormatSize(totalSize), hh.GetDistribution(begin: 1, end: 10000, fold: 2));
}
#endregion
#region Method 2: estimate the count using key selectors...
//long counter = await Fdb.System.EstimateCountAsync(db, range, ct);
//Console.WriteLine("COUNT = " + counter.ToString("N0"));
#endregion
}, ct));
}
var done = await Task.WhenAny(tasks);
tasks.Remove(done);
}
await Task.WhenAll(tasks);
sw.Stop();
Console.WriteLine("> Sampled " + FormatSize(total) + " (" + total.ToString("N0") + " bytes) in " + sw.Elapsed.TotalSeconds.ToString("N1") + " sec");
Console.WriteLine("> Estimated total size is " + FormatSize(total * ranges.Count / sz));
}
/// <summary>Find the DCs, machines and processes in the cluster</summary>
public static async Task Topology(string[] path, IFdbTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
var coords = await Fdb.System.GetCoordinatorsAsync(db, ct);
log.WriteLine("[Cluster] {0}", coords.Id);
var servers = await db.QueryAsync(tr => tr
.WithReadAccessToSystemKeys()
.GetRange(FdbKeyRange.StartsWith(Fdb.System.ServerList))
.Select(kvp => new
{
// Offsets Size Type Name Description
// 0 2 Word Version? 0100 (1.0 ?)
// 2 4 DWord ??? 0x00 0x20 0xA2 0x00
// 6 2 Word FDBMagic 0xDB 0x0F "FDB"
// 8 16 Guid NodeId Unique Process ID
// 24 16 Guid Machine "machine_id" field in foundationdb.conf (ends with 8x0 if manually specified)
// 40 16 Guid DataCenter "datacenter_id" field in foundationdb.conf (ends with 8x0 if manually specified)
// 56 4 ??? ?? 4 x 0
// 60 12 x24 ARRAY[] ?? array of 12x the same 24-byte struct defined below
// ...0 4 DWord IPAddress 01 00 00 7F => 127.0.0.1
// ...4 4 DWord Port 94 11 00 00 -> 4500
// ...8 4 DWord ?? randomish, changes every reboot
// ..12 4 DWord ?? randomish, changes every reboot
// ..16 4 DWord Size? small L-E integer, usually between 0x20 and 0x40...
// ..20 4 DWord ?? randmoish, changes every reboot
ProcessId = kvp.Value.Substring(8, 16).ToHexaString(),
MachineId = kvp.Value.Substring(24, 16).ToHexaString(),
DataCenterId = kvp.Value.Substring(40, 16).ToHexaString(),
Parts = Enumerable.Range(0, 12).Select(i =>
{
int p = 60 + 24 * i;
return new
{
Address = new IPAddress(kvp.Value.Substring(p, 4).GetBytes().Reverse().ToArray()),
Port = kvp.Value.Substring(p + 4, 4).ToInt32(),
Unknown1 = kvp.Value.Substring(p + 8, 4).ToInt32(),
Unknown2 = kvp.Value.Substring(p + 12, 4).ToInt32(),
Unknown3 = kvp.Value.Substring(p + 16, 4).ToInt32(),
Unknown4 = kvp.Value.Substring(p + 20, 4).ToInt32(),
};
}).ToList(),
Raw = kvp.Value,
}),
ct
);
var numNodes = servers.Select(s => s.ProcessId).Distinct().Count();
var numMachines = servers.Select(s => s.MachineId).Distinct().Count();
var numDCs = servers.Select(s => s.DataCenterId).Distinct().Count();
var dcs = servers.GroupBy(x => x.DataCenterId).ToArray();
for (int dcIndex = 0; dcIndex < dcs.Length;dcIndex++)
{
var dc = dcs[dcIndex];
bool lastDc = dcIndex == dcs.Length - 1;
string dcId = dc.Key.EndsWith("0000000000000000") ? dc.Key.Substring(0, 16) : dc.Key;
log.WriteLine((lastDc ? "`- " : "|- ") + "[DataCenter] {0} (#{1})", dcId, dcIndex);
var machines = dc.GroupBy(x => x.MachineId).ToArray();
string dcPrefix = lastDc ? " " : "| ";
for (int machineIndex = 0; machineIndex < machines.Length; machineIndex++)
{
var machine = machines[machineIndex];
var lastMachine = machineIndex == machines.Length - 1;
string machineId = machine.Key.EndsWith("0000000000000000") ? machine.Key.Substring(0, 16) : machine.Key;
log.WriteLine(dcPrefix + (lastMachine ? "`- " : "|- ") + "[Machine] {0}, {1}", machine.First().Parts[0].Address, machineId);
var procs = machine.ToArray();
string machinePrefix = dcPrefix + (lastMachine ? " " : "| ");
for (int procIndex = 0; procIndex < procs.Length; procIndex++)
{
var proc = procs[procIndex];
bool lastProc = procIndex == procs.Length - 1;
log.WriteLine(machinePrefix + (lastProc ? "`- " : "|- ") + "[Process] {0}:{1}, {2}", proc.Parts[0].Address, proc.Parts[0].Port, proc.ProcessId);
//foreach (var part in proc.Parts)
//{
// log.WriteLine(machinePrefix + "| -> {0}, {1}, {2:X8}, {3:X8}, {4}, {5:X8}", part.Address, part.Port, part.Unknown1, part.Unknown2, part.Unknown3, part.Unknown4);
//}
}
}
}
log.WriteLine();
log.WriteLine("Found {0} process(es) on {1} machine(s) in {2} datacenter(s)", numNodes, numMachines, numDCs);
log.WriteLine();
}
/// <summary>Shows the first few keys of a directory</summary>
public static async Task Show(string[] path, IFdbTuple extras, bool reverse, IFdbDatabase db, TextWriter log, CancellationToken ct)
{
int count = 20;
if (extras.Count > 0)
{
int x = extras.Get<int>(0);
if (x > 0) count = x;
}
// look if there is something under there
var folder = await db.Directory.TryOpenAsync(path, cancellationToken: ct);
if (folder != null)
{
log.WriteLine("# Content of {0} [{1}]", FdbKey.Dump(folder.Key), folder.Key.ToHexaString(' '));
var keys = await db.QueryAsync((tr) =>
{
var query = tr.GetRange(folder.ToRange());
return reverse
? query.Reverse().Take(count)
: query.Take(count + 1);
}, cancellationToken: ct);
if (keys.Count > 0)
{
if (reverse) keys.Reverse();
foreach (var key in keys.Take(count))
{
log.WriteLine("...{0} = {1}", FdbKey.Dump(folder.Extract(key.Key)), key.Value.ToAsciiOrHexaString());
}
if (!reverse && keys.Count == count + 1)
{
log.WriteLine("... more");
}
}
else
{
log.WriteLine(" no content found");
}
}
}
public async Task Run(IFdbDatabase db, TextWriter log, CancellationToken ct)
{
// estimate the number of machines...
Console.WriteLine("# Detecting cluster topology...");
var servers = await db.QueryAsync(tr => tr
.WithReadAccessToSystemKeys()
.GetRange(FdbKeyRange.StartsWith(Fdb.System.ServerList))
.Select(kvp => new
{
Node = kvp.Value.Substring(8, 16).ToHexaString(),
Machine = kvp.Value.Substring(24, 16).ToHexaString(),
DataCenter = kvp.Value.Substring(40, 16).ToHexaString()
}),
ct
);
var numNodes = servers.Select(s => s.Node).Distinct().Count();
var numMachines = servers.Select(s => s.Machine).Distinct().Count();
var numDCs = servers.Select(s => s.DataCenter).Distinct().Count();
Console.WriteLine("# > Found " + numNodes + " process(es) on " + numMachines + " machine(s) in " + numDCs + " datacenter(s)");
Console.WriteLine("# Reading list of shards...");
// dump keyServers
var ranges = await Fdb.System.GetChunksAsync(db, FdbKey.MinValue, FdbKey.MaxValue, ct);
Console.WriteLine("# > Found " + ranges.Count + " shards:");
// take a sample
var rnd = new Random(1234);
int sz = Math.Max((int)Math.Ceiling(this.Ratio * ranges.Count), 1);
if (sz > 500) sz = 500; //SAFETY
if (sz < 50) sz = Math.Max(sz, Math.Min(50, ranges.Count));
var samples = new List<FdbKeyRange>();
for (int i = 0; i < sz; i++)
{
int p = rnd.Next(ranges.Count);
samples.Add(ranges[p]);
ranges.RemoveAt(p);
}
Console.WriteLine("# Sampling " + sz + " out of " + ranges.Count + " shards (" + (100.0 * sz / ranges.Count).ToString("N1") + "%) ...");
Console.WriteLine("{0,9}{1,10}{2,10}{3,10} : K+V size distribution", "Count", "Keys", "Values", "Total");
var rangeOptions = new FdbRangeOptions { Mode = FdbStreamingMode.WantAll };
samples = samples.OrderBy(x => x.Begin).ToList();
long total = 0;
int workers = Math.Min(numMachines, 8);
var sw = Stopwatch.StartNew();
var tasks = new List<Task>();
while(samples.Count > 0)
{
while (tasks.Count < workers && samples.Count > 0)
{
var range = samples[0];
samples.RemoveAt(0);
tasks.Add(Task.Run(async () =>
{
var hh = new RobustHistogram(RobustHistogram.TimeScale.Ticks);
#region Method 1: get_range everything...
using (var tr = db.BeginTransaction(ct))
{
long keySize = 0;
long valueSize = 0;
long count = 0;
int iter = 0;
var beginSelector = FdbKeySelector.FirstGreaterOrEqual(range.Begin);
var endSelector = FdbKeySelector.FirstGreaterOrEqual(range.End);
while (true)
{
FdbRangeChunk data = default(FdbRangeChunk);
FdbException error = null;
try
{
data = await tr.Snapshot.GetRangeAsync(
beginSelector,
endSelector,
rangeOptions,
iter
).ConfigureAwait(false);
}
catch (FdbException e)
{
error = e;
}
if (error != null)
{
await tr.OnErrorAsync(error.Code).ConfigureAwait(false);
continue;
}
if (data.Count == 0) break;
count += data.Count;
foreach (var kvp in data.Chunk)
{
keySize += kvp.Key.Count;
valueSize += kvp.Value.Count;
hh.Add(TimeSpan.FromTicks(kvp.Key.Count + kvp.Value.Count));
}
if (!data.HasMore) break;
beginSelector = FdbKeySelector.FirstGreaterThan(data.Last.Key);
++iter;
}
long totalSize = keySize + valueSize;
Interlocked.Add(ref total, totalSize);
Console.WriteLine("{0,9}{1,10}{2,10}{3,10} : {4}", count.ToString("N0"), FormatSize(keySize), FormatSize(valueSize), FormatSize(totalSize), hh.GetDistribution(begin: 1, end: 10000, fold:2));
}
#endregion
#region Method 2: estimate the count using key selectors...
//long counter = await Fdb.System.EstimateCountAsync(db, range, ct);
//Console.WriteLine("COUNT = " + counter.ToString("N0"));
#endregion
}, ct));
}
var done = await Task.WhenAny(tasks);
tasks.Remove(done);
}
await Task.WhenAll(tasks);
sw.Stop();
Console.WriteLine("> Sampled " + FormatSize(total) + " (" + total.ToString("N0") + " bytes) in " + sw.Elapsed.TotalSeconds.ToString("N1") + " sec");
Console.WriteLine("> Estimated total size is " + FormatSize(total * ranges.Count / sz));
}
